def add_job(self, job_item):
"""
Add job to PyJobsWeb database
:param job_item: Scrapy pyjobs_crawlers item object
:return:
"""
job_public_id = job_item['url']
if self.job_exist(job_public_id):
print 'Skip existing item'
return
job = Job()
attributes = ['title', 'description', 'company', 'address', 'company_url',
'publication_datetime', 'publication_datetime_is_fake']
# Populate job attributes if item contain it
for attribute in attributes:
if attribute in job_item:
setattr(job, attribute, job_item[attribute])
job.url = job_item['url']
job.source = job_item['source']
job.crawl_datetime = job_item['initial_crawl_datetime']
if 'tags' in job_item:
import json
tags = [{'tag': t.tag, 'weight': t.weight} for t in job_item['tags']]
job.tags = json.dumps(tags)
DBSession.add(job)
transaction.commit()
def save_item_as_job(item):
# def uid(item):
# return '{}--{}'.format(item['source'], item['source_local_uid'])
#
existing = DBSession.query(Job).filter(Job.url==item['url']).count()
if existing:
print 'Skip existing item'
return
job = Job()
attributes = ['title', 'description', 'company', 'address', 'company_url',
'publication_datetime']
# Populate job attributes if item contain it
for attribute in attributes:
if attribute in item:
setattr(job, attribute, item[attribute])
job.url = item['url']
job.crawl_datetime = item['initial_crawl_datetime']
if 'tags' in item:
import json
tags = [{'tag': t.tag, 'weight': t.weight} for t in item['tags']]
job.tags = json.dumps(tags)
DBSession.add(job)
transaction.commit()
def submit(self, *args, **kwargs):
company = self._build_company_obj(**kwargs)
transaction.begin()
DBSession.add(company)
transaction.commit()
self._redirect()
def submit(self, *args, **kwargs):
company = self._build_company_obj(**kwargs)
transaction.begin()
DBSession.add(company)
transaction.commit()
self._redirect()
def log(self, source, action, more=None):
if more is not None:
message = '%s (%s)' % (action, more)
else:
message = action
log = Log()
log.source = source
log.message = message
log.datetime = datetime.datetime.now()
DBSession.add(log)
transaction.commit()
def setUp(self):
"""Setup test fixture for each model test method."""
try:
new_attrs = {}
new_attrs.update(self.attrs)
new_attrs.update(self.do_get_dependencies())
self.obj = self.klass(**new_attrs)
DBSession.add(self.obj)
DBSession.flush()
return self.obj
except:
DBSession.rollback()
raise
def add_job(self, job_item):
"""
Add job to PyJobsWeb database
:param job_item: Scrapy pyjobs_crawlers item object
:return:
"""
job_public_id = job_item['url']
if self.job_exist(job_public_id):
print 'Skip existing item'
return
job = model.JobAlchemy()
# Populate attributes which do not require special treatments before
# population
attributes = ['title', 'description', 'company', 'address',
'company_url', 'publication_datetime',
'publication_datetime_is_fake']
# Populate job attributes if item contain it
for attribute in attributes:
if attribute in job_item:
setattr(job, attribute, job_item[attribute])
job.url = job_item['url']
job.source = job_item['source']
job.crawl_datetime = job_item['initial_crawl_datetime']
# Populate attributes which require special treatments before population
if 'tags' in job_item:
tags = [{'tag': t.tag, 'weight': t.weight}
for t in job_item['tags']]
job.tags = json.dumps(tags)
# Insert the job offer in the Postgresql database
DBSession.add(job)
transaction.commit()
