def test_avatar_url(self):
# user object doesn't have an email or an avatar by default
avatar_url = filters.avatar_url(self.user)
self.assertEqual(
avatar_url,
'//www.gravatar.com/avatar/00000000000000000000000000000000?d=mm',
)
# testing what if the user has an avatar already
self.user.set_avatar(self.avatar_url)
avatar_url = filters.avatar_url(self.user, self.avatar_size)
self.assertEqual(
avatar_url,
self.avatar_url + '?size=' + 'x'.join(self.avatar_size))
# what if the user doesn't have an avatar but has an email
self.user.set_avatar(None)
self.user.set_email('*****@*****.**')
avatar_url = filters.avatar_url(self.user, self.avatar_size)
ehash = md5sum(self.user.email)
self.assertEqual(
avatar_url,
'//www.gravatar.com/avatar/' + ehash + '?d=mm&s=' +
'x'.join(self.avatar_size),
)
def avatar_url(user: Any,
size: Union[str, List[int], Tuple[int, int]] = None) -> str:
"""Generate an avatar for the given user."""
if isinstance(size, (list, tuple)):
size = 'x'.join(str(s) for s in size)
if user.avatar:
if size:
# TODO: Use a URL parser
if '?' in user.avatar:
return user.avatar + '&size=' + str(size)
else:
return user.avatar + '?size=' + str(size)
else:
return user.avatar
email = user.email
if email:
if isinstance(email, str):
# Flask-Lastuser's User model has email as a string
ehash = md5sum(user.email)
else:
# Lastuser's User model has email as a UserEmail object
ehash = email.md5sum
gravatar = '//www.gravatar.com/avatar/' + ehash + '?d=mm'
if size:
gravatar += '&s=' + str(size)
return gravatar
# Return Gravatar's missing man image
return '//www.gravatar.com/avatar/00000000000000000000000000000000?d=mm'
def avatar_url(user, size=None):
if isinstance(size, (list, tuple)):
size = u'x'.join(size)
if user.avatar:
if size:
# TODO: Use a URL parser
if u'?' in user.avatar:
return user.avatar + u'&size=' + unicode(size)
else:
return user.avatar + u'?size=' + unicode(size)
else:
return user.avatar
email = user.email
if email:
if isinstance(email, basestring):
hash = md5sum(
user.email
) # Flask-Lastuser's User model has email as a string
else:
hash = email.md5sum # Lastuser's User model has email as a UserEmail object
gravatar = u'//www.gravatar.com/avatar/' + hash + u'?d=mm'
if size:
gravatar += u'&s=' + unicode(size)
return gravatar
# Return Gravatar's missing man image
return u'//www.gravatar.com/avatar/00000000000000000000000000000000?d=mm'
def avatar_url(user, size=None):
if isinstance(size, (list, tuple)):
size = 'x'.join(size)
if user.avatar:
if size:
# TODO: Use a URL parser
if '?' in user.avatar:
return user.avatar + '&size=' + six.text_type(size)
else:
return user.avatar + '?size=' + six.text_type(size)
else:
return user.avatar
email = user.email
if email:
if isinstance(email, six.string_types):
# Flask-Lastuser's User model has email as a string
ehash = md5sum(user.email)
else:
# Lastuser's User model has email as a UserEmail object
ehash = email.md5sum
gravatar = '//www.gravatar.com/avatar/' + ehash + '?d=mm'
if size:
gravatar += '&s=' + six.text_type(size)
return gravatar
# Return Gravatar's missing man image
return '//www.gravatar.com/avatar/00000000000000000000000000000000?d=mm'
def is_public_email_domain(email_or_domain, default=None, timeout=30):
"""
Checks if the given domain (or domain of given email) is known to offer public email accounts.
All MX lookup results are cached for one day. An exception is raised if timeout happens
before the check is completed or domain lookup fails, and no default is provided.
:param email_or_domain: Email address or domain name to check
:param default: Default value to return in case domain lookup fails
:param timeout: Lookup timeout
:raises MXLookupException: If a DNS lookup error happens and no default is specified
"""
if six.PY2:
cache_key = b'mxrecord/' + md5sum(
email_or_domain.encode('utf-8')
if isinstance(email_or_domain, six.text_type)
else email_or_domain
)
else:
cache_key = 'mxrecord/' + md5sum(email_or_domain)
try:
sniffedmx = asset_cache.get(cache_key)
except ValueError: # Possible error from Py2 vs Py3 pickle mismatch
sniffedmx = None
if sniffedmx is None or not isinstance(sniffedmx, dict):
# Cache entry missing or corrupted; fetch a new result and update cache
try:
sniffedmx = mxsniff(email_or_domain, timeout=timeout)
asset_cache.set(cache_key, sniffedmx, timeout=86400) # cache for a day
except MXLookupException as e:
# Domain lookup failed
if default is None:
raise e
else:
return default
if any(p['public'] for p in sniffedmx['providers']):
return True
else:
return False
def test_useremailclaim(self):
crusoe = self.fixtures.crusoe
domain = 'batdogs.ca'
new_email = 'crusoe@' + domain
emd5sum = md5sum(new_email)
result = models.UserEmailClaim(email=new_email, user=crusoe)
db.session.add(result)
db.session.commit()
self.assertIsInstance(result, models.UserEmailClaim)
self.assertEqual(emd5sum, result.md5sum)
self.assertEqual(domain, result.domain)
self.assertEqual(crusoe, result.user)
assert '<UserEmailClaim {email} of {user}>'.format(
email=new_email, user=repr(crusoe)[1:-1]) in (repr(result))
def test_useremail_get(self):
"""
Test for verifying UserEmail's get that should return a UserEmail object with matching email or md5sum
"""
crusoe = self.fixtures.crusoe
email = crusoe.email.email
email_md5 = md5sum(email)
# scenario 1: when both email and md5sum are not passed
with self.assertRaises(TypeError):
models.UserEmail.get()
# scenario 2: when email is passed
get_by_email = models.UserEmail.get(email=email)
self.assertIsInstance(get_by_email, models.UserEmail)
self.assertEqual(get_by_email.user, crusoe)
# scenario 3: when md5sum is passed
get_by_md5sum = models.UserEmail.get(md5sum=email_md5)
self.assertIsInstance(get_by_md5sum, models.UserEmail)
self.assertEqual(get_by_md5sum.user, crusoe)
def is_public_email_domain(
email_or_domain: str, default: Optional[bool] = None, timeout: int = 30
) -> bool:
"""
Return True if the given email domain is known to offer public email accounts.
Looks up a list of known public email providers, both directly via their domains
(for popular domains like gmail.com) and via their MX records for services offering
email on multiple domains. All MX lookup results are cached for one day. An
exception is raised if timeout happens before the check is completed or domain
lookup fails, and no default is provided.
:param email_or_domain: Email address or domain name to check
:param default: Default value to return in case domain lookup fails
:param timeout: Lookup timeout in seconds
:raises MxLookupError: If a DNS lookup error happens and no default is specified
"""
cache_key = 'mxrecord/' + md5sum(email_or_domain)
try:
sniffedmx = asset_cache.get(cache_key)
except ValueError: # Possible error from Py2 vs Py3 pickle mismatch
sniffedmx = None
if sniffedmx is None or not isinstance(sniffedmx, dict):
# Cache entry missing or corrupted; fetch a new result and update cache
try:
sniffedmx = mxsniff(email_or_domain, timeout=timeout)
asset_cache.set(cache_key, sniffedmx, timeout=86400) # cache for a day
except MxLookupError as e:
# Domain lookup failed
if default is None:
raise e
return default
if any(p['public'] for p in sniffedmx['providers']):
return True
else:
return False
def avatar_url(user, size=None):
if isinstance(size, (list, tuple)):
size = u'x'.join(size)
if user.avatar:
if size:
# TODO: Use a URL parser
if u'?' in user.avatar:
return user.avatar + u'&size=' + unicode(size)
else:
return user.avatar + u'?size=' + unicode(size)
else:
return user.avatar
email = user.email
if email:
if isinstance(email, basestring):
hash = md5sum(user.email) # Flask-Lastuser's User model has email as a string
else:
hash = email.md5sum # Lastuser's User model has email as a UserEmail object
gravatar = u'//www.gravatar.com/avatar/' + hash + u'?d=mm'
if size:
gravatar += u'&s=' + unicode(size)
return gravatar
# Return Gravatar's missing man image
return u'//www.gravatar.com/avatar/00000000000000000000000000000000?d=mm'
def editjob(hashid, key, domain=None, form=None, validated=False, newpost=None):
if form is None:
form = forms.ListingForm(request.form)
form.job_type.choices = JobType.choices(g.board)
form.job_category.choices = JobCategory.choices(g.board)
if g.board and not g.board.require_pay:
form.job_pay_type.choices = [(-1, u'Confidential')] + PAY_TYPE.items()
post = None
no_email = False
if not newpost:
post = JobPost.query.filter_by(hashid=hashid).first_or_404()
if not ((key is None and g.user is not None and post.admin_is(g.user)) or (key == post.edit_key)):
abort(403)
# Once this post is published, require editing at /domain/<hashid>/edit
if not key and post.status not in POSTSTATUS.UNPUBLISHED and post.email_domain != domain:
return redirect(post.url_for('edit'), code=301)
# Don't allow editing jobs that aren't on this board as that may be a loophole when
# the board allows no pay (except in the 'www' root board, where editing is always allowed)
with db.session.no_autoflush:
if g.board and g.board.not_root and post.link_to_board(g.board) is None and request.method == 'GET':
blink = post.postboards.first()
if blink:
return redirect(post.url_for('edit', subdomain=blink.board.name, _external=True))
else:
return redirect(post.url_for('edit', subdomain=None, _external=True))
# Don't allow email address to be changed once it's confirmed
if post.status in POSTSTATUS.POSTPENDING:
no_email = True
if request.method == 'POST' and post and post.status in POSTSTATUS.POSTPENDING:
# del form.poster_name # Deprecated 2013-11-20
form.poster_email.data = post.email
if request.method == 'POST' and (validated or form.validate()):
form_description = bleach.linkify(bleach.clean(form.job_description.data, tags=ALLOWED_TAGS))
form_perks = bleach.linkify(bleach.clean(form.job_perks_description.data, tags=ALLOWED_TAGS)) if form.job_perks.data else ''
form_how_to_apply = form.job_how_to_apply.data
form_email_domain = get_email_domain(form.poster_email.data)
form_words = get_word_bag(u' '.join((form_description, form_perks, form_how_to_apply)))
similar = False
with db.session.no_autoflush:
for oldpost in JobPost.query.filter(db.or_(
db.and_(
JobPost.email_domain == form_email_domain,
JobPost.status.in_(POSTSTATUS.POSTPENDING)),
JobPost.status == POSTSTATUS.SPAM)).filter(
JobPost.datetime > datetime.utcnow() - agelimit).all():
if not post or (oldpost.id != post.id):
if oldpost.words:
s = SequenceMatcher(None, form_words, oldpost.words)
if s.ratio() > 0.6:
similar = True
break
if similar:
flash("This post is very similar to an earlier post. You may not repost the same job "
"in less than %d days." % agelimit.days, category='interactive')
else:
if newpost:
post = JobPost(**newpost)
db.session.add(post)
if g.board:
post.add_to(g.board)
if g.board.not_root:
post.add_to('www')
post.headline = form.job_headline.data
post.headlineb = form.job_headlineb.data
post.type_id = form.job_type.data
post.category_id = form.job_category.data
post.location = form.job_location.data
post.relocation_assist = form.job_relocation_assist.data
post.description = form_description
post.perks = form_perks
post.how_to_apply = form_how_to_apply
post.company_name = form.company_name.data
post.company_url = form.company_url.data
post.hr_contact = form.hr_contact.data
post.twitter = form.twitter.data
post.pay_type = form.job_pay_type.data
if post.pay_type == -1:
post.pay_type = None
if post.pay_type is not None and post.pay_type != PAY_TYPE.NOCASH:
post.pay_currency = form.job_pay_currency.data
post.pay_cash_min = form.job_pay_cash_min.data
post.pay_cash_max = form.job_pay_cash_max.data
else:
post.pay_currency = None
post.pay_cash_min = None
post.pay_cash_max = None
if form.job_pay_equity.data:
post.pay_equity_min = form.job_pay_equity_min.data
post.pay_equity_max = form.job_pay_equity_max.data
else:
post.pay_equity_min = None
post.pay_equity_max = None
post.admins = form.collaborators.data
# Allow name and email to be set only on non-confirmed posts
if not no_email:
# post.fullname = form.poster_name.data # Deprecated 2013-11-20
if post.email != form.poster_email.data:
# Change the email_verify_key if the email changes
post.email_verify_key = random_long_key()
post.email = form.poster_email.data
post.email_domain = form_email_domain
post.md5sum = md5sum(post.email)
with db.session.no_autoflush:
# This is dependent on the domain's DNS validity already being confirmed
# by the form's email validator
post.domain = Domain.get(post.email_domain, create=True)
# To protect from gaming, don't allow words to be removed in edited posts once the post
# has been confirmed. Just add the new words.
if post.status in POSTSTATUS.POSTPENDING:
prev_words = post.words or u''
else:
prev_words = u''
post.words = get_word_bag(u' '.join((prev_words, form_description, form_perks, form_how_to_apply)))
post.language, post.language_confidence = identify_language(post)
if post.status == POSTSTATUS.MODERATED:
post.status = POSTSTATUS.CONFIRMED
if request.files['company_logo']:
# The form's validator saved the processed logo in g.company_logo.
thumbnail = g.company_logo
logofilename = uploaded_logos.save(thumbnail, name='%s.' % post.hashid)
post.company_logo = logofilename
else:
if form.company_logo_remove.data:
post.company_logo = None
db.session.commit()
tag_jobpost.delay(post.id) # Keywords
tag_locations.delay(post.id) # Locations
post.uncache_viewcounts('pay_label')
session.pop('userkeys', None) # Remove legacy userkeys dict
session.permanent = True
return redirect(post.url_for(), code=303)
elif request.method == 'POST':
flash("Please review the indicated issues", category='interactive')
elif request.method == 'GET':
form.populate_from(post)
return render_template('postjob.html', form=form, no_email=no_email)
def editjob(hashid, key, form=None, post=None, validated=False):
if form is None:
form = forms.ListingForm(request.form)
form.job_type.choices = JobType.choices(g.board)
form.job_category.choices = JobCategory.choices(g.board)
if g.board and not g.board.require_pay:
form.job_pay_type.choices = [(-1, u'Confidential')] + PAY_TYPE.items()
if post is None:
post = JobPost.query.filter_by(hashid=hashid).first_or_404()
if not ((key is None and g.user is not None and post.admin_is(g.user)) or (key == post.edit_key)):
abort(403)
# Don't allow editing jobs that aren't on this board as that may be a loophole when
# the board allows no pay.
if g.board and post.link_to_board(g.board) is None and request.method == 'GET':
blink = post.postboards.first()
if blink:
return redirect(url_for('editjob', hashid=post.hashid, subdomain=blink.board.name, _external=True))
else:
return redirect(url_for('editjob', hashid=post.hashid, subdomain=None, _external=True))
# Don't allow email address to be changed once it's confirmed
if post.status in POSTSTATUS.POSTPENDING:
no_email = True
else:
no_email = False
if request.method == 'POST' and post.status in POSTSTATUS.POSTPENDING:
# del form.poster_name # Deprecated 2013-11-20
form.poster_email.data = post.email
if request.method == 'POST' and (validated or form.validate()):
form_description = bleach.linkify(bleach.clean(form.job_description.data, tags=ALLOWED_TAGS))
form_perks = bleach.linkify(bleach.clean(form.job_perks_description.data, tags=ALLOWED_TAGS)) if form.job_perks.data else ''
form_how_to_apply = form.job_how_to_apply.data
form_email_domain = get_email_domain(form.poster_email.data)
form_words = get_word_bag(u' '.join((form_description, form_perks, form_how_to_apply)))
similar = False
for oldpost in JobPost.query.filter(db.or_(
db.and_(
JobPost.email_domain == form_email_domain,
JobPost.status.in_(POSTSTATUS.POSTPENDING)),
JobPost.status == POSTSTATUS.SPAM)).filter(
JobPost.datetime > datetime.utcnow() - agelimit).all():
if oldpost.id != post.id:
if oldpost.words:
s = SequenceMatcher(None, form_words, oldpost.words)
if s.ratio() > 0.6:
similar = True
break
if similar:
flash("This listing is very similar to an earlier listing. You may not relist the same job "
"in less than %d days." % agelimit.days, category='interactive')
else:
post.headline = form.job_headline.data
post.type_id = form.job_type.data
post.category_id = form.job_category.data
post.location = form.job_location.data
post.relocation_assist = form.job_relocation_assist.data
post.description = form_description
post.perks = form_perks
post.how_to_apply = form_how_to_apply
post.company_name = form.company_name.data
post.company_url = form.company_url.data
post.hr_contact = form.hr_contact.data
post.pay_type = form.job_pay_type.data
if post.pay_type == -1:
post.pay_type = None
if post.pay_type is not None and post.pay_type != PAY_TYPE.NOCASH:
post.pay_currency = form.job_pay_currency.data
post.pay_cash_min = form.job_pay_cash_min.data
post.pay_cash_max = form.job_pay_cash_max.data
else:
post.pay_currency = None
post.pay_cash_min = None
post.pay_cash_max = None
if form.job_pay_equity.data:
post.pay_equity_min = form.job_pay_equity_min.data
post.pay_equity_max = form.job_pay_equity_max.data
else:
post.pay_equity_min = None
post.pay_equity_max = None
post.admins = form.collaborators.data
# Allow name and email to be set only on non-confirmed posts
if not no_email:
# post.fullname = form.poster_name.data # Deprecated 2013-11-20
post.email = form.poster_email.data
post.email_domain = form_email_domain
post.md5sum = md5sum(post.email)
# To protect from gaming, don't allow words to be removed in edited listings once the post
# has been confirmed. Just add the new words.
if post.status in POSTSTATUS.POSTPENDING:
prev_words = post.words or u''
else:
prev_words = u''
post.words = get_word_bag(u' '.join((prev_words, form_description, form_perks, form_how_to_apply)))
post.language = identify_language(post)
if post.status == POSTSTATUS.MODERATED:
post.status = POSTSTATUS.CONFIRMED
if request.files['company_logo']:
# The form's validator saved the processed logo in g.company_logo.
thumbnail = g.company_logo
logofilename = uploaded_logos.save(thumbnail, name='%s.' % post.hashid)
post.company_logo = logofilename
else:
if form.company_logo_remove.data:
post.company_logo = None
db.session.commit()
tag_locations.delay(post.id)
userkeys = session.get('userkeys', [])
userkeys.append(post.edit_key)
session['userkeys'] = userkeys
session.permanent = True
return redirect(url_for('jobdetail', hashid=post.hashid), code=303)
elif request.method == 'POST':
flash("Please correct the indicated errors", category='interactive')
elif request.method == 'GET':
# Populate form from model
form.job_headline.data = post.headline
form.job_type.data = post.type_id
form.job_category.data = post.category_id
form.job_location.data = post.location
form.job_relocation_assist.data = post.relocation_assist
form.job_description.data = post.description
form.job_perks.data = True if post.perks else False
form.job_perks_description.data = post.perks
form.job_how_to_apply.data = post.how_to_apply
form.company_name.data = post.company_name
form.company_url.data = post.company_url
# form.poster_name.data = post.fullname # Deprecated 2013-11-20
form.poster_email.data = post.email
form.hr_contact.data = int(post.hr_contact or False)
form.collaborators.data = post.admins
form.job_pay_type.data = post.pay_type
if post.pay_type is None:
# This kludge required because WTForms doesn't know how to handle None in forms
form.job_pay_type.data = -1
form.job_pay_currency.data = post.pay_currency
form.job_pay_cash_min.data = post.pay_cash_min
form.job_pay_cash_max.data = post.pay_cash_max
form.job_pay_equity.data = bool(post.pay_equity_min and post.pay_equity_max)
form.job_pay_equity_min.data = post.pay_equity_min
form.job_pay_equity_max.data = post.pay_equity_max
return render_template('postjob.html', form=form, no_email=no_email)
def editjob(hashid, key, form=None, post=None, validated=False):
if form is None:
form = forms.ListingForm(request.form)
form.job_type.choices = [(ob.id, ob.title) for ob in JobType.query.filter_by(public=True).order_by('seq')]
form.job_category.choices = [(ob.id, ob.title) for ob in JobCategory.query.filter_by(public=True).order_by('seq')]
if post is None:
post = JobPost.query.filter_by(hashid=hashid).first_or_404()
if not ((key is None and g.user is not None and post.admin_is(g.user)) or (key == post.edit_key)):
abort(403)
# Don't allow email address to be changed once its confirmed
if post.status in POSTSTATUS.POSTPENDING:
no_email = True
else:
no_email = False
if request.method == 'POST' and post.status in POSTSTATUS.POSTPENDING:
# del form.poster_name # Deprecated 2013-11-20
form.poster_email.data = post.email
if request.method == 'POST' and (validated or form.validate()):
form_description = bleach.linkify(bleach.clean(form.job_description.data, tags=ALLOWED_TAGS))
form_perks = bleach.linkify(bleach.clean(form.job_perks_description.data, tags=ALLOWED_TAGS)) if form.job_perks.data else ''
form_how_to_apply = form.job_how_to_apply.data
form_email_domain = get_email_domain(form.poster_email.data)
form_words = get_word_bag(u' '.join((form_description, form_perks, form_how_to_apply)))
similar = False
for oldpost in JobPost.query.filter(db.or_(
db.and_(
JobPost.email_domain == form_email_domain,
JobPost.status.in_(POSTSTATUS.POSTPENDING)),
JobPost.status == POSTSTATUS.SPAM)).filter(
JobPost.datetime > datetime.utcnow() - agelimit).all():
if oldpost.id != post.id:
if oldpost.words:
s = SequenceMatcher(None, form_words, oldpost.words)
if s.ratio() > 0.6:
similar = True
break
if similar:
flash("This listing is very similar to an earlier listing. You may not relist the same job "
"in less than %d days." % agelimit.days, category='interactive')
else:
post.headline = form.job_headline.data
post.type_id = form.job_type.data
post.category_id = form.job_category.data
post.location = form.job_location.data
post.relocation_assist = form.job_relocation_assist.data
post.description = form_description
post.perks = form_perks
post.how_to_apply = form_how_to_apply
post.company_name = form.company_name.data
post.company_url = form.company_url.data
post.hr_contact = form.hr_contact.data
post.pay_type = form.job_pay_type.data
if post.pay_type != PAY_TYPE.NOCASH:
post.pay_currency = form.job_pay_currency.data
post.pay_cash_min = form.job_pay_cash_min.data # TODO: Sanitize
post.pay_cash_max = form.job_pay_cash_max.data # TODO: Sanitize
else:
post.pay_currency = None
post.pay_cash_min = None
post.pay_cash_max = None
if form.job_pay_equity.data:
post.pay_equity_min = form.job_pay_equity_min.data # TODO: Sanitize
post.pay_equity_max = form.job_pay_equity_max.data # TODO: Sanitize
else:
post.pay_equity_min = None
post.pay_equity_max = None
if form.collaborators.data:
post.admins = []
userdata = lastuser.getuser_by_userids(form.collaborators.data)
for userinfo in userdata:
if userinfo['type'] == 'user':
user = User.query.filter_by(userid=userinfo['buid']).first()
if not user:
# New user on Hasjob. Don't set username right now. It's not relevant
# until first login and we don't want to deal with conflicts
user = User(userid=userinfo['buid'], fullname=userinfo['title'])
db.session.add(user)
post.admins.append(user)
# Allow name and email to be set only on non-confirmed posts
if not no_email:
# post.fullname = form.poster_name.data # Deprecated 2013-11-20
post.email = form.poster_email.data
post.email_domain = form_email_domain
post.md5sum = md5sum(post.email)
# To protect from gaming, don't allow words to be removed in edited listings once the post
# has been confirmed. Just add the new words.
if post.status in POSTSTATUS.POSTPENDING:
prev_words = post.words or u''
else:
prev_words = u''
post.words = get_word_bag(u' '.join((prev_words, form_description, form_perks, form_how_to_apply)))
post.language = identify_language(post)
if post.status == POSTSTATUS.MODERATED:
post.status = POSTSTATUS.CONFIRMED
if request.files['company_logo']:
# The form's validator saved the processed logo in g.company_logo.
thumbnail = g.company_logo
logofilename = uploaded_logos.save(thumbnail, name='%s.' % post.hashid)
post.company_logo = logofilename
else:
if form.company_logo_remove.data:
post.company_logo = None
db.session.commit()
tag_locations.delay(post.id)
userkeys = session.get('userkeys', [])
userkeys.append(post.edit_key)
session['userkeys'] = userkeys
session.permanent = True
return redirect(url_for('jobdetail', hashid=post.hashid), code=303)
elif request.method == 'POST':
flash("Please correct the indicated errors", category='interactive')
elif request.method == 'GET':
# Populate form from model
form.job_headline.data = post.headline
form.job_type.data = post.type_id
form.job_category.data = post.category_id
form.job_location.data = post.location
form.job_relocation_assist.data = post.relocation_assist
form.job_description.data = post.description
form.job_perks.data = True if post.perks else False
form.job_perks_description.data = post.perks
form.job_how_to_apply.data = post.how_to_apply
form.company_name.data = post.company_name
form.company_url.data = post.company_url
# form.poster_name.data = post.fullname # Deprecated 2013-11-20
form.poster_email.data = post.email
form.hr_contact.data = int(post.hr_contact or False)
form.collaborators.data = [u.userid for u in post.admins]
form.job_pay_type.data = post.pay_type
form.job_pay_currency.data = post.pay_currency
form.job_pay_cash_min.data = post.pay_cash_min
form.job_pay_cash_max.data = post.pay_cash_max
form.job_pay_equity.data = bool(post.pay_equity_min and post.pay_equity_max)
form.job_pay_equity_min.data = post.pay_equity_min
form.job_pay_equity_max.data = post.pay_equity_max
return render_template('postjob.html', form=form, no_email=no_email,
getuser_autocomplete=lastuser.endpoint_url(lastuser.getuser_autocomplete_endpoint),
getuser_userids=lastuser.endpoint_url(lastuser.getuser_userids_endpoint))
def check_url(
self,
url: str,
allowed_schemes: AllowedList,
allowed_domains: AllowedList,
invalid_urls: InvalidUrlPatterns,
text: Union[str, None] = None,
):
"""
Inner method to actually check the URL.
This method accepts ``allowed_schemes``, ``allowed_domains`` and
``invalid_urls`` as direct parameters despite their availability via `self`
because they may be callables, and in :class:`AllUrlsValid` we call
:meth:`check_url` repeatedly. The callables should be called only once. This
optimization has no value in the base class :class:`ValidUrl`.
As the validator is instantiated once per form field, it cannot mutate itself
at runtime to cache the callables' results, and must instead pass them from one
method to the next.
"""
urlparts = urlparse(url)
if allowed_schemes:
if urlparts.scheme not in allowed_schemes:
return self.message_schemes.format(
url=url, schemes=_(', ').join(allowed_schemes))
if allowed_domains:
if urlparts.netloc.lower() not in allowed_domains:
return self.message_domains.format(
url=url, domains=_(', ').join(allowed_domains))
if urlparts.scheme not in ('http', 'https') or not self.visit_url:
# The rest of this function only validates HTTP urls.
return
cache_key = 'linkchecker/' + md5sum(url)
try:
cache_check = asset_cache.get(cache_key)
except ValueError: # Possible error from a broken pickle
cache_check = None
# Read from cache, but assume cache may be broken since Flask-Cache stores data
# as a pickle, which is version-specific
if cache_check and isinstance(cache_check, dict):
rurl = cache_check.get('url')
code = cache_check.get('code')
else:
rurl = None # rurl is the response URL after following redirects
code = None
# TODO: Also honour the robots.txt protocol and stay off URLs that aren't meant
# to be checked. https://docs.python.org/3/library/urllib.robotparser.html
if not rurl or not code:
try:
r = requests.get(
url,
timeout=30,
verify=False, # skipcq: BAN-B501
headers={'User-Agent': self.user_agent},
)
code = r.status_code
rurl = r.url
except (
# Still a relative URL? Must be broken
requests.exceptions.MissingSchema,
# Name resolution or connection failed
requests.exceptions.ConnectionError,
# Didn't respond in time
requests.exceptions.Timeout,
):
code = None
except Exception as e: # NOQA: B902
exception_catchall.send(e)
code = None
if (rurl is not None and code is not None and code in (
200,
201,
202,
203,
204,
205,
206,
207,
208,
226,
403, # For Cloudflare
999, # For LinkedIn
)):
# Cloudflare returns HTTP 403 for urls behind its bot protection.
# Hence we're accepting 403 as an acceptable code.
#
# 999 is a non-standard too-many-requests error. We can't look past it to
# check a URL, so we let it pass
# The URL works, so now we check if it's in a reject list. This part
# runs _after_ attempting to load the URL as we want to catch redirects.
for patterns, message in invalid_urls:
for pattern in patterns:
# For text patterns, do a substring search. For regex patterns
# (assumed so if not text), do a regex search. Test with the final
# URL from the response, after redirects, but report errors using
# the URL the user provided
if (pattern in rurl if isinstance(
pattern, str) else pattern.search(rurl) is
not None # type: ignore[attr-defined]
):
return message.format(url=url, text=text)
# All good. The URL works and isn't invalid, so save to cache and return
# without an error message
asset_cache.set(cache_key, {
'url': rurl,
'code': code
},
timeout=86400)
return
else:
if text is not None and url != text:
return self.message_urltext.format(url=url, text=text)
else:
return self.message.format(url=url)
def check_url(self,
url,
allowed_schemes,
allowed_domains,
invalid_urls,
text=None):
"""
Inner method to actually check the URL.
This method accepts ``allowed_schemes``, ``allowed_domains`` and
``invalid_urls`` as direct parameters despite their availability via `self`
because they may be callables, and in :class:`AllUrlsValid` we call
:meth:`check_url` repeatedly. The callables should be called only once. This
optimization has no value in the base class :class:`ValidUrl`.
As the validator is instantiated once per form field, it cannot mutate itself
at runtime to cache the callables' results, and must instead pass them from one
method to the next.
"""
urlparts = urlparse(url)
if allowed_schemes:
if urlparts.scheme not in allowed_schemes:
return self.message_schemes.format(
url=url, schemes=_(', ').join(allowed_schemes))
if allowed_domains:
if urlparts.netloc.lower() not in allowed_domains:
return self.message_domains.format(
url=url, domains=_(', ').join(allowed_domains))
if urlparts.scheme not in ('http', 'https') or not self.visit_url:
# The rest of this function only validates HTTP urls.
return
if six.PY2:
cache_key = b'linkchecker/' + md5sum(
url.encode('utf-8') if isinstance(url, six.text_type) else url)
else:
cache_key = 'linkchecker/' + md5sum(url)
try:
cache_check = asset_cache.get(cache_key)
except ValueError: # Possible error from a broken pickle
cache_check = None
# Read from cache, but assume cache may be broken
# since Flask-Cache stores data as a pickle,
# which is version-specific
if cache_check and isinstance(cache_check, dict):
rurl = cache_check.get('url')
code = cache_check.get('code')
else:
rurl = None # rurl is the response URL after following redirects
if not rurl or not code:
try:
r = requests.get(
url,
timeout=30,
allow_redirects=True,
verify=False,
headers={'User-Agent': self.user_agent},
)
code = r.status_code
rurl = r.url
except (
# Still a relative URL? Must be broken
requests.exceptions.MissingSchema,
# Name resolution or connection failed
requests.exceptions.ConnectionError,
# Didn't respond in time
requests.exceptions.Timeout,
):
code = None
except Exception as e:
exception_catchall.send(e)
code = None
if rurl is not None and code in (
200,
201,
202,
203,
204,
205,
206,
207,
208,
226,
999,
):
# 999 is a non-standard too-many-requests error. We can't look past it to
# check a URL, so we let it pass
# The URL works, so now we check if it's in a reject list. This part
# runs _after_ attempting to load the URL as we want to catch redirects.
for patterns, message in invalid_urls:
for pattern in patterns:
# For text patterns, do a substring search. For regex patterns
# (assumed so if not text), do a regex search. Test with the final
# URL from the response, after redirects, but report errors using
# the URL the user provided
if (pattern in rurl
if isinstance(pattern, six.string_types) else
pattern.search(rurl) is not None):
return message.format(url=url, text=text)
# All good. The URL works and isn't invalid, so save to cache and return
# without an error message
asset_cache.set(cache_key, {
'url': rurl,
'code': code
},
timeout=86400)
return
else:
if text is not None and url != text:
return self.message_urltext.format(url=url, text=text)
else:
return self.message.format(url=url)
def __init__(self, email, **kwargs):
super(UserEmailClaim, self).__init__(**kwargs)
self.verification_code = newsecret()
self._email = email.lower()
self.md5sum = md5sum(self._email)
self.domain = email.split('@')[-1]
def __init__(self, email, **kwargs):
super(UserEmail, self).__init__(**kwargs)
self._email = email.lower()
self.md5sum = md5sum(self._email)
self.domain = email.split('@')[-1]
def email(self, value):
self._email = value.lower()
self.md5sum = md5sum(value)
def editjob(hashid,
key,
domain=None,
form=None,
validated=False,
newpost=None):
if form is None:
form = forms.ListingForm(request.form)
form.job_type.choices = JobType.choices(g.board)
form.job_category.choices = JobCategory.choices(g.board)
if g.board and not g.board.require_pay:
form.job_pay_type.choices = [(-1, u'Confidential')
] + PAY_TYPE.items()
post = None
no_email = False
if not newpost:
post = JobPost.query.filter_by(hashid=hashid).first_or_404()
if not ((key is None and g.user is not None and post.admin_is(g.user))
or (key == post.edit_key)):
abort(403)
# Once this post is published, require editing at /domain/<hashid>/edit
if not key and post.status not in POSTSTATUS.UNPUBLISHED and post.email_domain != domain:
return redirect(post.url_for('edit'), code=301)
# Don't allow editing jobs that aren't on this board as that may be a loophole when
# the board allows no pay (except in the 'www' root board, where editing is always allowed)
with db.session.no_autoflush:
if g.board and g.board.not_root and post.link_to_board(
g.board) is None and request.method == 'GET':
blink = post.postboards.first()
if blink:
return redirect(
post.url_for('edit',
subdomain=blink.board.name,
_external=True))
else:
return redirect(
post.url_for('edit', subdomain=None, _external=True))
# Don't allow email address to be changed once it's confirmed
if post.status in POSTSTATUS.POSTPENDING:
no_email = True
if request.method == 'POST' and post and post.status in POSTSTATUS.POSTPENDING:
# del form.poster_name # Deprecated 2013-11-20
form.poster_email.data = post.email
if request.method == 'POST' and (validated or form.validate()):
form_description = bleach.linkify(
bleach.clean(form.job_description.data, tags=ALLOWED_TAGS))
form_perks = bleach.linkify(
bleach.clean(form.job_perks_description.data,
tags=ALLOWED_TAGS)) if form.job_perks.data else ''
form_how_to_apply = form.job_how_to_apply.data
form_email_domain = get_email_domain(form.poster_email.data)
form_words = get_word_bag(u' '.join(
(form_description, form_perks, form_how_to_apply)))
similar = False
with db.session.no_autoflush:
for oldpost in JobPost.query.filter(
db.or_(
db.and_(JobPost.email_domain == form_email_domain,
JobPost.status.in_(POSTSTATUS.POSTPENDING)),
JobPost.status == POSTSTATUS.SPAM)).filter(
JobPost.datetime > datetime.utcnow() -
agelimit).all():
if not post or (oldpost.id != post.id):
if oldpost.words:
s = SequenceMatcher(None, form_words, oldpost.words)
if s.ratio() > 0.6:
similar = True
break
if similar:
flash(
"This post is very similar to an earlier post. You may not repost the same job "
"in less than %d days." % agelimit.days,
category='interactive')
else:
if newpost:
post = JobPost(**newpost)
db.session.add(post)
if g.board:
post.add_to(g.board)
if g.board.not_root:
post.add_to('www')
post.headline = form.job_headline.data
post.headlineb = form.job_headlineb.data
post.type_id = form.job_type.data
post.category_id = form.job_category.data
post.location = form.job_location.data
post.relocation_assist = form.job_relocation_assist.data
post.description = form_description
post.perks = form_perks
post.how_to_apply = form_how_to_apply
post.company_name = form.company_name.data
post.company_url = form.company_url.data
post.hr_contact = form.hr_contact.data
post.twitter = form.twitter.data
post.pay_type = form.job_pay_type.data
if post.pay_type == -1:
post.pay_type = None
if post.pay_type is not None and post.pay_type != PAY_TYPE.NOCASH:
post.pay_currency = form.job_pay_currency.data
post.pay_cash_min = form.job_pay_cash_min.data
post.pay_cash_max = form.job_pay_cash_max.data
else:
post.pay_currency = None
post.pay_cash_min = None
post.pay_cash_max = None
if form.job_pay_equity.data:
post.pay_equity_min = form.job_pay_equity_min.data
post.pay_equity_max = form.job_pay_equity_max.data
else:
post.pay_equity_min = None
post.pay_equity_max = None
post.admins = form.collaborators.data
# Allow name and email to be set only on non-confirmed posts
if not no_email:
# post.fullname = form.poster_name.data # Deprecated 2013-11-20
post.email = form.poster_email.data
post.email_domain = form_email_domain
post.md5sum = md5sum(post.email)
with db.session.no_autoflush:
# This is dependent on the domain's DNS validity already being confirmed
# by the form's email validator
post.domain = Domain.get(post.email_domain, create=True)
# To protect from gaming, don't allow words to be removed in edited posts once the post
# has been confirmed. Just add the new words.
if post.status in POSTSTATUS.POSTPENDING:
prev_words = post.words or u''
else:
prev_words = u''
post.words = get_word_bag(u' '.join(
(prev_words, form_description, form_perks, form_how_to_apply)))
post.language, post.language_confidence = identify_language(post)
if post.status == POSTSTATUS.MODERATED:
post.status = POSTSTATUS.CONFIRMED
if request.files['company_logo']:
# The form's validator saved the processed logo in g.company_logo.
thumbnail = g.company_logo
logofilename = uploaded_logos.save(thumbnail,
name='%s.' % post.hashid)
post.company_logo = logofilename
else:
if form.company_logo_remove.data:
post.company_logo = None
db.session.commit()
tag_jobpost.delay(post.id) # Keywords
tag_locations.delay(post.id) # Locations
post.uncache_viewcounts('pay_label')
session.pop('userkeys', None) # Remove legacy userkeys dict
session.permanent = True
return redirect(post.url_for(), code=303)
elif request.method == 'POST':
flash("Please review the indicated issues", category='interactive')
elif request.method == 'GET':
# Populate form from model
form.job_headline.data = post.headline
form.job_headlineb.data = post.headlineb
form.job_type.data = post.type_id
form.job_category.data = post.category_id
form.job_location.data = post.location
form.job_relocation_assist.data = post.relocation_assist
form.job_description.data = post.description
form.job_perks.data = True if post.perks else False
form.job_perks_description.data = post.perks
form.job_how_to_apply.data = post.how_to_apply
form.company_name.data = post.company_name
form.company_url.data = post.company_url
# form.poster_name.data = post.fullname # Deprecated 2013-11-20
form.poster_email.data = post.email
form.twitter.data = post.twitter
form.hr_contact.data = int(post.hr_contact or False)
form.collaborators.data = post.admins
form.job_pay_type.data = post.pay_type
if post.pay_type is None:
# This kludge required because WTForms doesn't know how to handle None in forms
form.job_pay_type.data = -1
form.job_pay_currency.data = post.pay_currency
form.job_pay_cash_min.data = post.pay_cash_min
form.job_pay_cash_max.data = post.pay_cash_max
form.job_pay_equity.data = bool(post.pay_equity_min
and post.pay_equity_max)
form.job_pay_equity_min.data = post.pay_equity_min
form.job_pay_equity_max.data = post.pay_equity_max
return render_template('postjob.html', form=form, no_email=no_email)