def add_job(self, job_item): """ Add job to PyJobsWeb database :param job_item: Scrapy pyjobs_crawlers item object :return: """ job_public_id = job_item['url'] if self.job_exist(job_public_id): print 'Skip existing item' return job = Job() attributes = ['title', 'description', 'company', 'address', 'company_url', 'publication_datetime', 'publication_datetime_is_fake'] # Populate job attributes if item contain it for attribute in attributes: if attribute in job_item: setattr(job, attribute, job_item[attribute]) job.url = job_item['url'] job.source = job_item['source'] job.crawl_datetime = job_item['initial_crawl_datetime'] if 'tags' in job_item: import json tags = [{'tag': t.tag, 'weight': t.weight} for t in job_item['tags']] job.tags = json.dumps(tags) DBSession.add(job) transaction.commit()
def save_item_as_job(item): # def uid(item): # return '{}--{}'.format(item['source'], item['source_local_uid']) # existing = DBSession.query(Job).filter(Job.url==item['url']).count() if existing: print 'Skip existing item' return job = Job() attributes = ['title', 'description', 'company', 'address', 'company_url', 'publication_datetime'] # Populate job attributes if item contain it for attribute in attributes: if attribute in item: setattr(job, attribute, item[attribute]) job.url = item['url'] job.crawl_datetime = item['initial_crawl_datetime'] if 'tags' in item: import json tags = [{'tag': t.tag, 'weight': t.weight} for t in item['tags']] job.tags = json.dumps(tags) DBSession.add(job) transaction.commit()
def submit(self, *args, **kwargs): company = self._build_company_obj(**kwargs) transaction.begin() DBSession.add(company) transaction.commit() self._redirect()
def log(self, source, action, more=None): if more is not None: message = '%s (%s)' % (action, more) else: message = action log = Log() log.source = source log.message = message log.datetime = datetime.datetime.now() DBSession.add(log) transaction.commit()
def setUp(self): """Setup test fixture for each model test method.""" try: new_attrs = {} new_attrs.update(self.attrs) new_attrs.update(self.do_get_dependencies()) self.obj = self.klass(**new_attrs) DBSession.add(self.obj) DBSession.flush() return self.obj except: DBSession.rollback() raise
def add_job(self, job_item): """ Add job to PyJobsWeb database :param job_item: Scrapy pyjobs_crawlers item object :return: """ job_public_id = job_item['url'] if self.job_exist(job_public_id): print 'Skip existing item' return job = model.JobAlchemy() # Populate attributes which do not require special treatments before # population attributes = ['title', 'description', 'company', 'address', 'company_url', 'publication_datetime', 'publication_datetime_is_fake'] # Populate job attributes if item contain it for attribute in attributes: if attribute in job_item: setattr(job, attribute, job_item[attribute]) job.url = job_item['url'] job.source = job_item['source'] job.crawl_datetime = job_item['initial_crawl_datetime'] # Populate attributes which require special treatments before population if 'tags' in job_item: tags = [{'tag': t.tag, 'weight': t.weight} for t in job_item['tags']] job.tags = json.dumps(tags) # Insert the job offer in the Postgresql database DBSession.add(job) transaction.commit()