def test_find_all(self):
self.job_item.save()
another_job_item = JobItem()
another_job_item.job_title = 'Another Test Job'
another_job_item.save()
records = JobItem.findall()
print 'Job Items', records
self.assertEqual(2, len(records))
def test_find_with_pagination(self):
for i in range(0, 20):
job_item = JobItem()
job_item.job_title='job_item_%d' % i
job_item.save()
records = JobItem.find_with_pagination(page_request={'page_no': 2, 'size': 10})
print 'Job Items', records
self.assertEqual(10, len(records))
def setUp(self):
self.clean_db()
self.job_item = JobItem()
self.job_item.job_title="Test Job"
self.job_item.employer_name="Test Job Employer"
# self.job_item.crawled_date = datetime.datetime.now()
# self.job_item.publish_date = datetime.datetime.strptime('2014-10-31', '%Y-%m-%d')
self.job_item.job_country = "Singapore"
self.job_item.job_desc = "This is a test job"
self.job_item.contact = "88888888"
self.source = 'unit_test'
def parse_item_requests_callback(self, response, item_xpath_selector=''):
requests = []
for job_item in response.xpath(item_xpath_selector):
job_crawler_item = JobItem()
self.populate_job_crawler_item(job_item, job_crawler_item)
if self.should_load_details(job_crawler_item):
requests.append(
Request(url=job_crawler_item.job_details_link,
callback=self.retrieve_job_details,
meta={'item': job_crawler_item},
dont_filter=True))
return requests
def parse_item(self, response):
requests = []
for job_item in response.xpath('//tr'):
job_crawler_item = JobItem()
for index, detail_item in enumerate(job_item.xpath('./td')):
self.populate_job_crawler_item(index, detail_item,
job_crawler_item)
if index == 4:
if self.should_load_details(job_crawler_item):
requests.append(
Request(url=job_crawler_item.job_details_link,
callback=self.retrieve_job_details,
meta={'item': job_crawler_item},
dont_filter=True))
return requests
def test_remove_records_matches_rejection_pattern(self):
for i in range(0, 20):
job_item = JobItem()
job_item.job_title=u'人员_%d' % i
job_item.save()
# mark the title as blocked
RejectionPattern(u'人员_\d+', 'For Testing').save()
# run the remove action
JobItem.remove_records_matches_rejection_pattern()
conn = self.connect_db()
try:
c = conn.cursor()
c.execute('SELECT COUNT(*) FROM ' + JobItem.table_name)
self.assertEqual(c.fetchone()[0], 0, 'Count of job items should be 0')
except:
pass
finally:
conn.close()
def test_remove_blocked_records(self):
for i in range(0, 20):
job_item = JobItem()
job_item.job_title=u'人员_%d' % i
job_item.contact = str(random.randint(90000000, 99999999))
job_item.save()
# mark the contact as blocked
BlockedContact(job_item.contact, u'人员').save()
# run the remove action
JobItem.remove_blocked_records()
conn = self.connect_db()
try:
c = conn.cursor()
c.execute('SELECT COUNT(*) FROM ' + JobItem.table_name)
self.assertEqual(c.fetchone()[0], 0, 'Count of job items should be 0')
except:
pass
finally:
conn.close()
