def cmc(deck_id: int, attempts: int = 0) -> str:
if attempts > 3:
msg = 'Unable to generate cmc chart for {id} in 3 attempts.'.format(
id=deck_id)
logger.error(msg)
raise OperationalException(msg)
path = determine_path(str(deck_id) + '-cmc.png')
if acceptable_file(path):
return path
d = deck.load_deck(deck_id)
costs: Dict[str, int] = {}
for ci in d.maindeck:
c = ci.card
if c.is_land():
continue
if c.mana_cost is None:
cost = '0'
elif next((s for s in c.mana_cost if '{X}' in s), None) is not None:
cost = 'X'
else:
converted = int(float(c.cmc))
cost = '7+' if converted >= 7 else str(converted)
costs[cost] = ci.get('n') + costs.get(cost, 0)
path = image(path, costs)
if acceptable_file(path):
return path
return cmc(deck_id, attempts + 1)
def internal_server_error(self, e: Exception) -> Union[Tuple[str, int], Response]:
log_exception(request, e)
path = request.path
try:
repo.create_issue('500 error at {path}\n {e}'.format(path=path, e=e), session.get('mtgo_username', session.get('id', 'logged_out')), self.name, 'PennyDreadfulMTG/perf-reports', exception=e)
except GithubException:
logger.error('Github error', e)
if request.path.startswith('/api/'):
return return_json(generate_error('INTERNALERROR', 'Internal Error.', exception=e), status=404)
view = InternalServerError(e)
return view.page(), 500
def generate_TOP801(reportID, spam):
try:
header = shared.processed_mgs[reportID]['header']
except (KeyError, TypeError, ValueError, IndexError) as e:
logger.error("header cannot be found in processed messages")
logger.debug(str(type(e)) + str(e))
return None
msg = dict()
msg['header'] = Validator.generate_header(header,
topic_name=shared.TOP801)
msg['body'] = dict()
msg['body']['incidentID'] = reportID
msg['body']['spam'] = spam
return msg
def _preprocess_image(img_data: dict, local_file: str) -> Image:
try:
image = Image.new_from_file(local_file, access='sequential')
max_height = config.DEFAULT_MAX_HEIGHT
max_width = config.DEFAULT_MAX_WIDTH
if img_data["copyrightStatus"] and \
img_data["copyrightStatus"].lower() == "copyright":
max_height = config.COPYRIGHT_MAX_HEIGHT
max_width = config.COPYRIGHT_MAX_WIDTH
if image.height > max_height or image.width > max_width:
if image.height >= image.width:
shrink_by = image.height / max_height
else:
shrink_by = image.width / max_width
logger.debug(f"Resizing {img_data['id']} by {shrink_by}")
image = image.shrink(shrink_by, shrink_by)
except Error as pye:
image = None
Statistic.vips_err(img_data)
logger.error(f"VIPs error - {pye.message}")
return image
def listen(self, performed_action, topics=None):
# Topics should be a list of topic names e.g. ['topic1', 'topic2']
if topics is None:
topics = self.default_topics
self.listening = True
# Subscribe to topics
try:
self.consumer.subscribe(topics)
except Exception as e:
logger.error("Error @ BusConsumer.listen()")
logger.debug(str(type(e)) + str(e))
return False
logger.info("listener subscribed successfully to topics:" +
str(topics))
# Initiate a loop for continuous listening
while self.listening:
msg = self.consumer.poll(0)
# If a message is received and it is not an error message
if msg is not None and msg.error() is None:
# Add incoming message to requests database
try:
message_text = msg.value().decode('utf-8')
except:
message_text = msg.value()
performed_action(message_text)
# TODO: check if it works ok with the sleep .5
time.sleep(0.5)
# Unsubscribe and close consumer
self.consumer.unsubscribe()
self.consumer.close()
def scrape(limit: int = 1) -> None:
page = 1
while page <= limit:
time.sleep(0.1)
url = 'https://www.mtggoldfish.com/deck/custom/penny_dreadful?page={n}#online'.format(
n=page)
soup = BeautifulSoup(
fetch_tools.fetch(url, character_encoding='utf-8'), 'html.parser')
raw_decks = soup.find_all('div', {'class': 'deck-tile'})
if len(raw_decks) == 0:
logger.warning(
'No decks found in {url} so stopping.'.format(url=url))
break
for raw_deck in raw_decks:
d = Container({'source': 'MTG Goldfish'})
a = raw_deck.select_one('.title > span.deck-price-online > a')
d.identifier = re.findall(r'/deck/(\d+)#online', a.get('href'))[0]
d.url = 'https://www.mtggoldfish.com/deck/{identifier}#online'.format(
identifier=d.identifier)
d.name = a.contents[0].strip()
d.mtggoldfish_username = without_by(
raw_deck.select_one(
'div.deck-tile-author').contents[0].strip())
try:
d.created_date = scrape_created_date(d)
except InvalidDataException as e:
msg = f'Got {e} trying to find a created_date in {d}, {raw_deck}'
logger.error(msg)
raise InvalidDataException(msg) from e
time.sleep(5)
d.cards = scrape_decklist(d)
err = vivify_or_error(d)
if err:
logger.warning(err)
continue
deck.add_deck(d)
page += 1
def generate_header(header, topic_name, action_type=None):
""" generate a dict corresponding to header with correct values and return it
Read the immutable values from the parameter header
Generate the values that are specific
Return the header dict
If something goes wrong, then return None
"""
try:
header['topicName'] = topic_name
if action_type is not None:
header['actionType'] = action_type
header['sentUTC'] = Validator.time_UTC()
header['sender'] = 'VAL'
header['msgIdentifier'] = 'VAL_' + str(uuid.uuid4()) + "_" + str(
time.time())
except (KeyError, TypeError, ValueError, IndexError) as e:
logger.error(
"validator.Validator.generate_header: header does not have the correct fields"
)
logger.debug(str(type(e)) + str(e))
return None
return header
def on_delivery(self, err, msg):
if err:
logger.error(
str(err) + str(msg) +
str(" (message wasn't successfully delivered to bus)"))
def import_job(**kwargs):
"""Import job.
Keyword arguments:
employer_name -- Employer name
job_title -- Title of job
summary -- Job summary
year -- Year the job was advertised
term -- Term job was advertised [Fall, Winter, Spring]
location -- Location job was advertised
openings -- Number of job openings
remaining -- Number of job openings remaining
applicants -- Number of applicants job has (Optional)
levels -- Levels job is intended for [Junior, Intermediate, Senior]
programs -- Programs the job is specified for
url -- URL of job
date -- Date job was crawled (useful for knowing exactly # of applicants at what time)
index -- Boolean to indicate whether to index or not (default True)
"""
employer_name = kwargs['employer_name'].lower()
job_title = kwargs['job_title'].lower()
term = kwargs['term']
levels = []
for level in kwargs['levels']:
uw_level = Term.get_level(level)
if uw_level:
levels.append(uw_level)
else:
logger.error(COMPONENT, 'Error processing level: {}'.format(level))
programs = []
for program in kwargs['programs']:
uw_program = Program.get_program(program)
if uw_program:
programs.append(uw_program)
else:
logger.error(COMPONENT, 'Error processing program: {}'.format(program))
location = kwargs['location'].lower()
openings = int(kwargs['openings'])
remaining = int(kwargs['remaining']) if 'remaining' in kwargs else openings
summary = kwargs['summary']
filtered_summary = engine.filter_summary(summary)
summary_keywords = engine.get_keywords(filtered_summary, programs)
date = kwargs['date']
year = date.year
url = kwargs['url']
applicants = 0
try:
if kwargs['applicants']:
applicants = int(kwargs['applicants'])
except Exception:
pass
index = False
if index in kwargs:
index = kwargs['index']
logger.info(COMPONENT, 'Importing job: {} from {}'.format(job_title, employer_name))
# If employer does not exist, create it
if not Employer.employer_exists(employer_name):
logger.info(COMPONENT, 'Employer: {} does not exist, creating..'.format(employer_name))
employer = Employer(name=employer_name)
logger.info(COMPONENT, 'Creating job: {}'.format(job_title))
location = Location(name=location)
applicant = Applicant(applicants=applicants, date=date)
keywords = [Keyword(keyword=k['keyword'], types=k['types']) for k in summary_keywords]
# New job so number of remaining positions is same as openings
job = Job(title=job_title, summary=filtered_summary, year=year,
term=term, location=[location], openings=openings, remaining=remaining,
applicants=[applicant], levels=levels, programs=programs, url=url,
keywords=keywords)
job.save()
job.reload()
employer.jobs.append(job)
employer.save()
employer.reload()
if index:
elastic.index_employer_waterlooworks(employer)
elastic.index_job_waterlooworks(employer, job)
# Employer already exists
else:
employer = Employer.objects(name=employer_name).no_dereference().first()
logger.info(COMPONENT, 'Employer: {} already exists'.format(employer_name))
# If job does not exist, create it
if not employer.job_exists(job_title):
logger.info(COMPONENT, 'Creating job: {}'.format(job_title))
location = Location(name=location)
applicant = Applicant(applicants=applicants, date=date)
keywords = [Keyword(keyword=k['keyword'], types=k['types']) for k in summary_keywords]
# New job so number of remaining positions is same as openings
job = Job(title=job_title, summary=engine.filter_summary(summary), year=year,
term=term, location=[location], openings=openings, remaining=remaining,
applicants=[applicant], levels=levels, programs=programs, url=url,
keywords=keywords)
job.save()
job.reload()
employer.update(push__jobs=job)
if index:
elastic.update_employer_waterlooworks(employer)
elastic.index_job_waterlooworks(employer, job)
# Job already exists
else:
logger.info(COMPONENT, 'Job: {} already exists'.format(job_title))
job = Job.objects(id__in=[job.id for job in employer.jobs], title=job_title).first()
if not year >= job.year:
raise DataIntegrityError('Job: {} by {} cannot be advertised before {}'
.format(job_title, employer_name, job.year))
filtered_summary_compare = re.sub(r'\W+', '', filtered_summary.lower().strip()).strip()
job_summary_compare = re.sub(r'\W+', '', job.summary.lower().strip()).strip()
# Job summary is not the same. In this case the employer most likely changed the job
if not filtered_summary_compare == job_summary_compare:
if openings >= 1:
logger.info(COMPONENT, 'Job: {}: different summary detected, deprecating and creating new job..'
.format(job_title))
job.update(set__deprecated=True)
location = Location(name=location)
applicant = Applicant(applicants=applicants, date=date)
keywords = [Keyword(keyword=k['keyword'], types=k['types']) for k in summary_keywords]
# Assume new job so number of remaining positions is same as openings
new_job = Job(title=job_title, summary=filtered_summary, year=year, term=term,
location=[location], openings=openings, remaining=remaining, applicants=[applicant],
levels=levels, programs=programs, url=url, keywords=keywords)
new_job.save()
new_job.reload()
employer.update(push__jobs=new_job)
if index:
elastic.delete_employer_waterlooworks(employer)
elastic.delete_job_waterlooworks(employer, job)
elastic.index_employer_waterlooworks(employer)
elastic.index_job_waterlooworks(employer, new_job)
else:
logger.info(COMPONENT, 'Job: {}: different summary detected but invalid openings: {}, ignoring..'
.format(job_title, openings))
# Job is the same (same title and description)
else:
# If job is being advertised in new term
if year != job.year or term != job.term:
logger.info(COMPONENT, 'Job: {}: being advertised in new term, updating..'.format(job_title))
# Add hire ratio for previous term
hire_ratio = float(job.openings - job.remaining) / job.openings
job.hire_rate.add_rating(hire_ratio)
location = Location(name=location)
applicant = Applicant(applicants=applicants, date=date)
hire_rate = AggregateRating(rating=job.hire_rate.rating, count=job.hire_rate.count)
job.update(set__year=year, set__term=term, add_to_set__location=location, set__openings=openings,
set__remaining=remaining, push__applicants=applicant, set__hire_rate=hire_rate,
set__levels=levels, set__programs=programs, set__url=url, set__last_indexed=datetime.now())
if index:
elastic.update_job_waterlooworks(employer, job)
# Job is being updated. We need to update location, openings, levels, remaining, hire_rate, applicants
else:
logger.info(COMPONENT, 'Job: {}: updating for current term'.format(job_title))
remaining = job.remaining
# Job posting has decreased, some positions filled up
if openings < remaining:
remaining = openings
location = Location(name=location)
applicant = Applicant(applicants=applicants, date=date)
job.update(add_to_set__location=location, set__remaining=remaining,
set__levels=list(set(levels + job.levels)), push__applicants=applicant,
set__programs=list(set(programs + job.programs)), set__url=url,
set__last_indexed=datetime.now())
if index:
elastic.update_job_waterlooworks(employer, job)
def update_job(**kwargs):
"""Update job.
Keyword arguments:
id -- Job ID
summary -- Job summary
location -- Location job was advertised
programs -- Programs the job is specified for
levels -- Levels job is intended for [Junior, Intermediate, Senior]
openings -- Number of job openings
index -- Boolean to indicate whether to index or not (default True)
"""
summary = kwargs['summary']
location = kwargs['location'].lower()
levels = kwargs['levels']
programs = []
for program in kwargs['programs']:
uw_program = Program.get_program(program)
if uw_program:
programs.append(uw_program)
else:
logger.error(COMPONENT, 'Error processing program: {}'.format(program))
openings = 0
try:
if kwargs['openings']:
openings = int(kwargs['openings']) or 0
except Exception:
pass
index = False
if index in kwargs:
index = kwargs['index']
job = Job.objects(id=kwargs['id']).first()
remaining = job.openings
# Job posting has decreased, some positions filled up
if openings < job.openings:
remaining = openings
filtered_summary = engine.filter_summary(summary)
summary_keywords = engine.get_keywords(filtered_summary, programs)
filtered_summary_compare = re.sub(r'\W+', '', filtered_summary.lower().strip()).strip()
job_summary_compare = re.sub(r'\W+', '', job.summary.lower().strip()).strip()
employer = Employer.objects(jobs=kwargs['id']).first()
# Job summary is not the same. In this case the employer most likely changed the job
if not filtered_summary_compare == job_summary_compare:
if openings >= 1:
logger.info(COMPONENT, 'Job: {}: different summary detected, deprecating and creating new job..'
.format(kwargs['id']))
job.update(set__deprecated=True)
location = Location(name=location)
keywords = [Keyword(keyword=k['keyword'], types=k['types']) for k in summary_keywords]
# Assume new job so number of remaining positions is same as openings
new_job = Job(title=job.title, summary=filtered_summary, year=job.year, term=job.term,
location=[location], openings=openings, remaining=openings,
levels=levels, programs=programs, url=job.url, keywords=keywords)
new_job.save()
employer.update(push__jobs=new_job)
if index:
elastic.delete_employer_waterlooworks(employer)
elastic.delete_job_waterlooworks(employer, job)
elastic.index_employer_waterlooworks(employer)
elastic.index_job_waterlooworks(employer, new_job)
else:
logger.info(COMPONENT, 'Job: {}: different summary detected but invalid openings: {}, ignoring..'
.format(job.title, openings))
else:
logger.info(COMPONENT, 'Job: {}: updating for current term'.format(kwargs['id']))
location = Location(name=location)
job.update(add_to_set__location=location, set__remaining=remaining,
set__levels=list(set(levels + job.levels)),
set__programs=list(set(programs + job.programs)), set__last_indexed=datetime.now())
if index:
elastic.update_job_waterlooworks(employer, job)
def log_exception(r: Request, e: Exception) -> None:
logger.error(f'At request path: {r.path}', repo.format_exception(e))
def update_job(**kwargs):
"""Update job.
Keyword arguments:
id -- Job ID
summary -- Job summary
location -- Location job was advertised
programs -- Programs the job is specified for
levels -- Levels job is intended for [Junior, Intermediate, Senior]
openings -- Number of job openings
index -- Boolean to indicate whether to index or not (default True)
"""
summary = kwargs['summary']
location = kwargs['location'].lower()
levels = kwargs['levels']
programs = []
for program in kwargs['programs']:
uw_program = Program.get_program(program)
if uw_program:
programs.append(uw_program)
else:
logger.error(COMPONENT,
'Error processing program: {}'.format(program))
openings = 0
try:
if kwargs['openings']:
openings = int(kwargs['openings']) or 0
except Exception:
pass
index = False
if index in kwargs:
index = kwargs['index']
job = Job.objects(id=kwargs['id']).first()
remaining = job.openings
# Job posting has decreased, some positions filled up
if openings < job.openings:
remaining = openings
filtered_summary = engine.filter_summary(summary)
summary_keywords = engine.get_keywords(filtered_summary, programs)
filtered_summary_compare = re.sub(
r'\W+', '',
filtered_summary.lower().strip()).strip()
job_summary_compare = re.sub(r'\W+', '',
job.summary.lower().strip()).strip()
employer = Employer.objects(jobs=kwargs['id']).first()
# Job summary is not the same. In this case the employer most likely changed the job
if not filtered_summary_compare == job_summary_compare:
if openings >= 1:
logger.info(
COMPONENT,
'Job: {}: different summary detected, deprecating and creating new job..'
.format(kwargs['id']))
job.update(set__deprecated=True)
location = Location(name=location)
keywords = [
Keyword(keyword=k['keyword'], types=k['types'])
for k in summary_keywords
]
# Assume new job so number of remaining positions is same as openings
new_job = Job(title=job.title,
summary=filtered_summary,
year=job.year,
term=job.term,
location=[location],
openings=openings,
remaining=openings,
levels=levels,
programs=programs,
url=job.url,
keywords=keywords)
new_job.save()
employer.update(push__jobs=new_job)
if index:
elastic.delete_employer_waterlooworks(employer)
elastic.delete_job_waterlooworks(employer, job)
elastic.index_employer_waterlooworks(employer)
elastic.index_job_waterlooworks(employer, new_job)
else:
logger.info(
COMPONENT,
'Job: {}: different summary detected but invalid openings: {}, ignoring..'
.format(job.title, openings))
else:
logger.info(COMPONENT,
'Job: {}: updating for current term'.format(kwargs['id']))
location = Location(name=location)
job.update(add_to_set__location=location,
set__remaining=remaining,
set__levels=list(set(levels + job.levels)),
set__programs=list(set(programs + job.programs)),
set__last_indexed=datetime.now())
if index:
elastic.update_job_waterlooworks(employer, job)
def import_job(**kwargs):
"""Import job.
Keyword arguments:
employer_name -- Employer name
job_title -- Title of job
summary -- Job summary
year -- Year the job was advertised
term -- Term job was advertised [Fall, Winter, Spring]
location -- Location job was advertised
openings -- Number of job openings
remaining -- Number of job openings remaining
applicants -- Number of applicants job has (Optional)
levels -- Levels job is intended for [Junior, Intermediate, Senior]
programs -- Programs the job is specified for
url -- URL of job
date -- Date job was crawled (useful for knowing exactly # of applicants at what time)
index -- Boolean to indicate whether to index or not (default True)
"""
employer_name = kwargs['employer_name'].lower()
job_title = kwargs['job_title'].lower()
term = kwargs['term']
levels = []
for level in kwargs['levels']:
uw_level = Term.get_level(level)
if uw_level:
levels.append(uw_level)
else:
logger.error(COMPONENT, 'Error processing level: {}'.format(level))
programs = []
for program in kwargs['programs']:
uw_program = Program.get_program(program)
if uw_program:
programs.append(uw_program)
else:
logger.error(COMPONENT,
'Error processing program: {}'.format(program))
location = kwargs['location'].lower()
openings = int(kwargs['openings'])
remaining = int(kwargs['remaining']) if 'remaining' in kwargs else openings
summary = kwargs['summary']
filtered_summary = engine.filter_summary(summary)
summary_keywords = engine.get_keywords(filtered_summary, programs)
date = kwargs['date']
year = date.year
url = kwargs['url']
applicants = 0
try:
if kwargs['applicants']:
applicants = int(kwargs['applicants'])
except Exception:
pass
index = False
if index in kwargs:
index = kwargs['index']
logger.info(COMPONENT,
'Importing job: {} from {}'.format(job_title, employer_name))
# If employer does not exist, create it
if not Employer.employer_exists(employer_name):
logger.info(
COMPONENT,
'Employer: {} does not exist, creating..'.format(employer_name))
employer = Employer(name=employer_name)
logger.info(COMPONENT, 'Creating job: {}'.format(job_title))
location = Location(name=location)
applicant = Applicant(applicants=applicants, date=date)
keywords = [
Keyword(keyword=k['keyword'], types=k['types'])
for k in summary_keywords
]
# New job so number of remaining positions is same as openings
job = Job(title=job_title,
summary=filtered_summary,
year=year,
term=term,
location=[location],
openings=openings,
remaining=remaining,
applicants=[applicant],
levels=levels,
programs=programs,
url=url,
keywords=keywords)
job.save()
job.reload()
employer.jobs.append(job)
employer.save()
employer.reload()
if index:
elastic.index_employer_waterlooworks(employer)
elastic.index_job_waterlooworks(employer, job)
# Employer already exists
else:
employer = Employer.objects(
name=employer_name).no_dereference().first()
logger.info(COMPONENT,
'Employer: {} already exists'.format(employer_name))
# If job does not exist, create it
if not employer.job_exists(job_title):
logger.info(COMPONENT, 'Creating job: {}'.format(job_title))
location = Location(name=location)
applicant = Applicant(applicants=applicants, date=date)
keywords = [
Keyword(keyword=k['keyword'], types=k['types'])
for k in summary_keywords
]
# New job so number of remaining positions is same as openings
job = Job(title=job_title,
summary=engine.filter_summary(summary),
year=year,
term=term,
location=[location],
openings=openings,
remaining=remaining,
applicants=[applicant],
levels=levels,
programs=programs,
url=url,
keywords=keywords)
job.save()
job.reload()
employer.update(push__jobs=job)
if index:
elastic.update_employer_waterlooworks(employer)
elastic.index_job_waterlooworks(employer, job)
# Job already exists
else:
logger.info(COMPONENT, 'Job: {} already exists'.format(job_title))
job = Job.objects(id__in=[job.id for job in employer.jobs],
title=job_title).first()
if not year >= job.year:
raise DataIntegrityError(
'Job: {} by {} cannot be advertised before {}'.format(
job_title, employer_name, job.year))
filtered_summary_compare = re.sub(
r'\W+', '',
filtered_summary.lower().strip()).strip()
job_summary_compare = re.sub(r'\W+', '',
job.summary.lower().strip()).strip()
# Job summary is not the same. In this case the employer most likely changed the job
if not filtered_summary_compare == job_summary_compare:
if openings >= 1:
logger.info(
COMPONENT,
'Job: {}: different summary detected, deprecating and creating new job..'
.format(job_title))
job.update(set__deprecated=True)
location = Location(name=location)
applicant = Applicant(applicants=applicants, date=date)
keywords = [
Keyword(keyword=k['keyword'], types=k['types'])
for k in summary_keywords
]
# Assume new job so number of remaining positions is same as openings
new_job = Job(title=job_title,
summary=filtered_summary,
year=year,
term=term,
location=[location],
openings=openings,
remaining=remaining,
applicants=[applicant],
levels=levels,
programs=programs,
url=url,
keywords=keywords)
new_job.save()
new_job.reload()
employer.update(push__jobs=new_job)
if index:
elastic.delete_employer_waterlooworks(employer)
elastic.delete_job_waterlooworks(employer, job)
elastic.index_employer_waterlooworks(employer)
elastic.index_job_waterlooworks(employer, new_job)
else:
logger.info(
COMPONENT,
'Job: {}: different summary detected but invalid openings: {}, ignoring..'
.format(job_title, openings))
# Job is the same (same title and description)
else:
# If job is being advertised in new term
if year != job.year or term != job.term:
logger.info(
COMPONENT,
'Job: {}: being advertised in new term, updating..'.
format(job_title))
# Add hire ratio for previous term
hire_ratio = float(job.openings -
job.remaining) / job.openings
job.hire_rate.add_rating(hire_ratio)
location = Location(name=location)
applicant = Applicant(applicants=applicants, date=date)
hire_rate = AggregateRating(rating=job.hire_rate.rating,
count=job.hire_rate.count)
job.update(set__year=year,
set__term=term,
add_to_set__location=location,
set__openings=openings,
set__remaining=remaining,
push__applicants=applicant,
set__hire_rate=hire_rate,
set__levels=levels,
set__programs=programs,
set__url=url,
set__last_indexed=datetime.now())
if index:
elastic.update_job_waterlooworks(employer, job)
# Job is being updated. We need to update location, openings, levels, remaining, hire_rate, applicants
else:
logger.info(
COMPONENT,
'Job: {}: updating for current term'.format(job_title))
remaining = job.remaining
# Job posting has decreased, some positions filled up
if openings < remaining:
remaining = openings
location = Location(name=location)
applicant = Applicant(applicants=applicants, date=date)
job.update(add_to_set__location=location,
set__remaining=remaining,
set__levels=list(set(levels + job.levels)),
push__applicants=applicant,
set__programs=list(set(programs +
job.programs)),
set__url=url,
set__last_indexed=datetime.now())
if index:
elastic.update_job_waterlooworks(employer, job)