def test_database_id_prefilled(db, pipeline, item, spider):
item['id'] = 'honza42'
pipeline.process_item(item, spider)
with db:
job = Job.select()[0]
assert job.id == 'honza42'
def test_database_company_logo_path(db, pipeline, item, spider):
item['company_logos'] = [
{
'checksum':
'6b874bd7b996e9323fd2e094be83ca4c',
'path':
'company-logos/d40730d4068db31a09687ebb42f7637e26864a30.png',
'status':
'uptodate',
'url':
'https://www.startupjobs.cz/uploads/d6e95f8c946b72f36783aa0a0238341b.png'
},
{
'checksum':
'f3e2f82d7d8b24367f0a2c24b3d1aea3',
'path':
'company-logos/d1eed8447fb59dc9587dd97148a109a3cca77ed8.png',
'status':
'uptodate',
'url':
'https://www.startupjobs.cz/uploads/GQ1A8RDZWYUJfavicon155377551420.png'
},
]
pipeline.process_item(item, spider)
with db:
job = Job.select()[0]
assert job.company_logo_path == 'images/company-logos/d40730d4068db31a09687ebb42f7637e26864a30.png'
def test_database(db, pipeline, item, spider):
pipeline.process_item(item, spider)
with db:
job = Job.select()[0]
assert len(job.id) == 56 # sha224 hex digest length
assert job.source == 'dummy' # spider name
async def manage_jobs_voting_channel(client): # experimenting with Mila and ML
channel = await client.fetch_channel(JOBS_VOTING_CHANNEL)
seen_links = set()
log.info('Processing voting for jobs')
jobs = list(Job.select().where(Job.magic_is_junior == False)
) # TODO PoC, move this to models or revamp models altogether?
async for message in channel.history(limit=None, after=None):
for job in jobs:
link = job.link
if link.rstrip('/') in message.content:
log.info(f'Job {link} exists')
seen_links.add(link)
if message.reactions:
job.upvotes_count += count_upvotes(message.reactions)
job.downvotes_count += count_downvotes(message.reactions)
with db:
job.save()
log.info(f'Saved {link} reactions')
log.info('Processing voting for dropped jobs')
jobs_dropped = list(JobDropped.select().where(
JobDropped.magic_is_junior ==
True)) # TODO PoC, move this to models or revamp models altogether?
async for message in channel.history(limit=None, after=None):
for job_dropped in jobs_dropped:
link = job_dropped.item['link']
if link.rstrip('/') in message.content:
log.info(f'Job {link} exists')
seen_links.add(link)
if message.reactions:
job_dropped.upvotes_count += count_upvotes(
message.reactions)
job_dropped.downvotes_count += count_downvotes(
message.reactions)
with db:
job_dropped.save()
log.info(f'Saved {link} reactions')
if DISCORD_MUTATIONS_ENABLED:
new_jobs = [job for job in jobs if job.link not in seen_links]
log.info(f'Posting {len(new_jobs)} new jobs')
for job in new_jobs:
await channel.send(
f'**{job.title}**\n{job.company_name} – {job.location}\n{job.link}'
)
new_jobs_dropped = [
job_dropped for job_dropped in jobs_dropped
if job_dropped.item['link'] not in seen_links
]
log.info(f'Posting {len(new_jobs_dropped)} new dropped jobs')
for job_dropped in new_jobs_dropped:
await channel.send(
f"**{job_dropped.item['title']}**\n{job_dropped.item['company_name']} – {', '.join(job_dropped.item['locations_raw'])}\n{job_dropped.item['link']}"
)
else:
log.warning(
"Skipping Discord mutations, DISCORD_MUTATIONS_ENABLED not set")
def test_database(item, spider, db):
Pipeline(db=db, model=Job).process_item(item, spider)
with db:
job = Job.select()[0]
assert len(job.id) == 56 # sha224 hex digest length
assert job.source == 'dummy' # spider name
assert job.is_approved is False
def test_database_same_link_items(item, spider, db):
for location in ['Ostrava', 'Brno', 'Pardubice']:
item['location'] = location
Pipeline(db=db, model=Job).process_item(item, spider)
with db:
jobs = Job.select()
assert len(jobs) == 3
assert len({job.id for job in jobs}) == 3
assert len({job.location for job in jobs}) == 3
def test_database_id(db, pipeline, item, spider):
pipeline.process_item(item, spider)
with db:
job = Job.select()[0]
assert len(job.id) == 56 # sha224 hex digest length
def read_data_job():
jobs = Job.select(Job.id, Job.item).where(Job.junior_rank > 0)
df = map_jobs_to_df(jobs)
return jobs, df