def operation():
JobDropped.create(type=exception.__class__.__name__,
reason=str(exception),
item=item,
source=spider.name,
**response_data)
self.stats.inc_value('monitoring/job_dropped_saved')
def item_dropped(self, item, response, exception, spider):
with db:
JobDropped.create(type=exception.__class__.__name__,
reason=str(exception),
response_url=response.url,
response_backup_path=get_response_backup_path(
response.url),
item=item)
self.stats.inc_value('monitoring/job_dropped_saved')
def test_aggregate_metrics_companies_count_past_jobs_unique(db_connection):
JobDropped.create(type='Expired',
item=dict(company_link='https://example.com/company1'),
reason='...',
response_url='...',
source='...')
JobDropped.create(type='Expired',
item=dict(company_link='https://example.com/company1'),
reason='...',
response_url='...',
source='...')
assert Job.aggregate_metrics()['companies_count'] == 1
def read_data_jobdropped():
jobsdropped = JobDropped.select(
JobDropped.id,
JobDropped.item).where(JobDropped.type == 'NotEntryLevel').execute()
df = map_jobs_to_df(jobsdropped)
return jobsdropped, df
def admin_jobs_dropped_unexpected():
with db:
jobs_dropped = JobDropped.admin_listing(
['MissingRequiredFields', 'ShortDescription'])
return render_template('admin_jobs_dropped.html',
title='Podezřele zahozené nabídky',
jobs_dropped=models_to_dicts(jobs_dropped))
async def manage_jobs_voting_channel(client): # experimenting with Mila and ML
channel = await client.fetch_channel(JOBS_VOTING_CHANNEL)
seen_links = set()
log.info('Processing voting for jobs')
jobs = list(Job.select().where(Job.magic_is_junior == False)
) # TODO PoC, move this to models or revamp models altogether?
async for message in channel.history(limit=None, after=None):
for job in jobs:
link = job.link
if link.rstrip('/') in message.content:
log.info(f'Job {link} exists')
seen_links.add(link)
if message.reactions:
job.upvotes_count += count_upvotes(message.reactions)
job.downvotes_count += count_downvotes(message.reactions)
with db:
job.save()
log.info(f'Saved {link} reactions')
log.info('Processing voting for dropped jobs')
jobs_dropped = list(JobDropped.select().where(
JobDropped.magic_is_junior ==
True)) # TODO PoC, move this to models or revamp models altogether?
async for message in channel.history(limit=None, after=None):
for job_dropped in jobs_dropped:
link = job_dropped.item['link']
if link.rstrip('/') in message.content:
log.info(f'Job {link} exists')
seen_links.add(link)
if message.reactions:
job_dropped.upvotes_count += count_upvotes(
message.reactions)
job_dropped.downvotes_count += count_downvotes(
message.reactions)
with db:
job_dropped.save()
log.info(f'Saved {link} reactions')
if DISCORD_MUTATIONS_ENABLED:
new_jobs = [job for job in jobs if job.link not in seen_links]
log.info(f'Posting {len(new_jobs)} new jobs')
for job in new_jobs:
await channel.send(
f'**{job.title}**\n{job.company_name} – {job.location}\n{job.link}'
)
new_jobs_dropped = [
job_dropped for job_dropped in jobs_dropped
if job_dropped.item['link'] not in seen_links
]
log.info(f'Posting {len(new_jobs_dropped)} new dropped jobs')
for job_dropped in new_jobs_dropped:
await channel.send(
f"**{job_dropped.item['title']}**\n{job_dropped.item['company_name']} – {', '.join(job_dropped.item['locations_raw'])}\n{job_dropped.item['link']}"
)
else:
log.warning(
"Skipping Discord mutations, DISCORD_MUTATIONS_ENABLED not set")
def test_aggregate_metrics_rejected_jobs_count(db_connection):
JobDropped.create(source='juniorguru',
type='...',
reason='...',
response_url='...',
item={})
JobDropped.create(source='abc',
type='...',
reason='...',
response_url='...',
item={})
JobDropped.create(source='xyz',
type='...',
reason='...',
response_url='...',
item={})
assert Job.aggregate_metrics()['rejected_jobs_count'] == 2
def admin_jobs_dropped_all():
with db:
jobs_dropped = JobDropped.admin_listing()
return render_template('admin_jobs_dropped.html',
title='Všechny zahozené nabídky',
jobs_dropped=models_to_dicts(jobs_dropped))
def admin_jobs_dropped_expected():
with db:
jobs_dropped = JobDropped.admin_listing(['Expired', 'NotApproved'])
return render_template('admin_jobs_dropped.html',
title='Vědomě zahozené nabídky',
jobs_dropped=models_to_dicts(jobs_dropped))
def admin_jobs_dropped():
with db:
jobs_dropped = JobDropped.admin_listing(['NotEntryLevel'])
return render_template('admin_jobs_dropped.html',
title='Nejuniorní zahozené nabídky',
jobs_dropped=models_to_dicts(jobs_dropped))
def admin_jobs_dropped():
with db:
jobs_dropped = models_to_dicts(JobDropped.admin_listing())
return render_template('admin_jobs_dropped.html', jobs_dropped=jobs_dropped)