def _pages_update(context, data_dict):
if db.pages_table is None:
db.init_db(context['model'])
org_id = data_dict.get('org_id')
page = data_dict.get('page')
# we need the page in the context for name validation
context['page'] = page
context['group_id'] = org_id
data, errors = df.validate(data_dict, schema, context)
if errors:
raise p.toolkit.ValidationError(errors)
out = db.Page.get(group_id=org_id, name=page)
if not out:
out = db.Page()
out.group_id = org_id
out.name = page
items = ['title', 'content', 'name', 'private', 'order']
for item in items:
setattr(out, item, data.get(item))
out.modified = datetime.datetime.utcnow()
out.user_id = p.toolkit.c.userobj.id
out.save()
session = context['session']
session.add(out)
session.commit()
def read_sample(self, response):
'''detect sample page'''
now = datetime.datetime.utcnow()
client_number = response.meta['data-sclabs-client-number']
self.logger.debug('%d: Reading sample "%s".', client_number,
response.url)
page = self.session.query(
db.Page).filter(db.Page.source_url == response.url).first()
if page:
self.logger.debug('%d: Sample page already exists in DB.',
client_number)
page.fetched_at = now
else:
self.logger.debug('%d: Sample page does not exist in DB yet.',
client_number)
page = db.Page(source_url=response.url,
status=response.status,
fetched_at=now)
self.session.commit()
xpath_savepage_loaded_match = response.xpath(
self.xpath_savepage_loaded)
# if xpath_savepage_loaded_match:
# page.status = response.status
# self.save_this(response)
# else:
# self.log('PAGE {} WILL NOT BE SAVED!'.format(response.url))
# page.status = -404
page.status = response.status
self.session.commit()
self.save_this(response)
def _pages_update(context, data_dict):
if db.pages_table is None:
db.init_db(context['model'])
org_id = data_dict.get('org_id')
page = data_dict.get('page')
# we need the page in the context for name validation
context['page'] = page
context['group_id'] = org_id
data, errors = df.validate(data_dict, schema, context)
if errors:
raise p.toolkit.ValidationError(errors)
out = db.Page.get(group_id=org_id, name=page)
if not out:
out = db.Page()
out.group_id = org_id
out.name = page
items = [
'title',
'content',
'name',
'private',
'order',
'page_type',
'publish_date',
'image_url',
]
for item in items:
# backward compatible with older version where page_type does not exist
setattr(out, item,
data.get(item, 'page' if item == 'page_type' else None))
extras = {}
extra_keys = set(schema.keys()) - set(items + ['id', 'created'])
for key in extra_keys:
if key in data:
extras[key] = data.get(key)
out.extras = json.dumps(extras)
out.modified = datetime.datetime.utcnow()
out.user_id = p.toolkit.c.userobj.id
out.save()
session = context['session']
session.add(out)
session.commit()
def save_this(self, response):
'''Save the given page'''
client_number = response.meta['data-sclabs-client-number']
self.logger.debug('%d: Saving page from URL "%s".', client_number,
response.url)
counter = 1
parsedURL = urllib.parse.urlparse(response.url)
re_sampleURLPath_match = re_sampleURLPath.match(parsedURL.path)
if re_sampleURLPath_match:
sample_id = re_sampleURLPath_match.group('sample_id')
self.logger.debug('%d: Got sample ID "%s".', client_number,
sample_id)
else:
sample_id = 'NOID'
self.logger.debug('%d: No sample ID found.', client_number)
filename = '{}.html'.format(sample_id)
filepath = os.path.join(SAVE_FOLDER, filename)
while os.path.exists(filepath) or self.session.query(
db.Page).filter(db.Page.file == filename).first():
counter += 1
filename = '{}_{}.html'.format(sample_id, counter)
filepath = os.path.join(SAVE_FOLDER, filename)
with open(filepath, 'wb') as f:
f.write(response.body)
page = self.session.query(
db.Page).filter(db.Page.source_url == response.url).first()
if page:
self.logger.debug('%d: Sample page already exists in DB.',
client_number)
page.file = filename
else:
self.logger.debug('%d: Sample page not existing in DB yet.',
client_number)
page = db.Page(source_url=response.url, file=filename)
self.session.add(page)
self.session.commit()
self.logger.debug('%d: Saved page "%s" under file "%s".',
client_number, response.url, filepath)
def start_requests(self):
'''Generates the client links'''
self.logger.debug('Launching initial requests . . .')
for client_number in range(7600, -1, -1): #TODO
self.logger.debug('%d: Building request for client.',
client_number)
wanted_link = self.template_resultspage.format(
client_number=client_number, page=1)
request = scrapy.Request(url=wanted_link,
callback=self.read_client,
errback=self.errback)
request.meta['data-sclabs-client-number'] = client_number
request.meta['data-sclabs-client-lastknown-maxpage'] = 10000
request.meta['data-sclabs-client-currentpage'] = 1
self.logger.debug('%d: Setting currentpage counter to "%d".',
client_number,
request.meta['data-sclabs-client-currentpage'])
request.meta['data-sclabs-NTterpenes-counter'] = 0
self.logger.debug('%d: Setting NTterpenes counter to "%d".',
client_number,
request.meta['data-sclabs-NTterpenes-counter'])
request.meta['dont_cache'] = True
page = self.session.query(
db.Page).filter(db.Page.source_url == wanted_link).first()
if page:
self.logger.debug('%d: Page already in database.',
client_number)
page.online_database_id = self.lab_id
else:
self.logger.debug('%d: Page not in database yet.',
client_number)
page = db.Page(source_url=wanted_link,
online_database_id=self.lab_id)
self.session.add(page)
self.session.commit()
self.logger.debug('%d: Yielding request.', client_number)
yield request
def errback(self, failure):
'''Logs failed pages'''
now = datetime.datetime.utcnow()
source_url = failure.value.response.url
status_code = failure.value.response.status
self.logger.warning('Detected error while requesting page "%s".',
source_url)
if failure.check(scrapy.spidermiddlewares.httperror.HttpError):
page = self.session.query(
db.Page).filter(db.Page.source_url == source_url).first()
if page:
self.logger.debug('Page already exists in DB.')
page.status = status_code
page.fetched_at = now
page.online_database_id = self.lab_id
else:
self.logger.debug('Page not existing in DB yet.')
page = db.Page(source_url=source_url,
status=status_code,
fetched_at=now,
online_database_id=self.lab_id)
self.session.add(page)
self.session.commit()
def read_client(self, response):
'''flip through client pages'''
now = datetime.datetime.utcnow()
client_number = response.meta['data-sclabs-client-number']
no_terpenes_counter = response.meta['data-sclabs-NTterpenes-counter']
page = self.session.query(
db.Page).filter(db.Page.source_url == response.url).first()
page.status = self.real_statusCode(response)
page.fetched_at = now
page.online_database_id = self.lab_id
self.session.commit()
self.logger.debug('%d: reading client, NTterpenes is "%d".',
client_number, no_terpenes_counter)
for elem_sample in response.xpath(self.xpath_samples):
relative_sample_link = elem_sample.xpath(self.xpath_savepages)[0]
sample_link = response.urljoin(relative_sample_link.attrib['href'])
if no_terpenes_counter >= MAX_NO_CONSECUTIVE_TERPENES:
self.logger.debug(
'%d consecutive samples without terpene tests, aborting page.',
no_terpenes_counter)
break
raw_totalTerpenes = elem_sample.xpath(
self.xpath_samplepreview_totalterpenesNT)
should_crawl = True
if len(raw_totalTerpenes) > 0:
re_totalTerpenesNT_match = re_totalTerpenesNT.match(
raw_totalTerpenes[0].extract())
if re_totalTerpenesNT_match:
no_terpenes_counter += 1
should_crawl = False
self.logger.debug(
'Terpene test not present, will not crawl sample "%s", counter is now "%d".',
sample_link, no_terpenes_counter)
else:
no_terpenes_counter = 0
should_crawl = True
self.logger.debug(
'Terpene test present, will crawl sample "%s", counter has been reset.',
sample_link)
if should_crawl:
self.logger.debug('%d: Building request for sample "%s".',
client_number, sample_link)
request = scrapy_splash.SplashRequest(
url=sample_link,
callback=self.read_sample,
args=splash_args,
errback=self.errback)
request.meta['data-sclabs-client-number'] = client_number
page = self.session.query(
db.Page).filter(db.Page.source_url == sample_link).first()
if page:
self.logger.debug('%d: Sample page already in DB.',
client_number)
page.prev_url = response.url
page.online_database_id = self.lab_id
else:
self.logger.debug('%d: Sample page not in DB yet.',
client_number)
page = db.Page(source_url=sample_link,
prev_url=response.url,
online_database_id=self.lab_id)
self.session.add(page)
self.session.commit()
if page.file:
self.logger.debug(
'%d: A file is already associated with sample page of URL "%s", skipping.',
client_number, sample_link)
else:
self.logger.debug(
'%d: No file yet associated with sample page of URL "%s", yielding request.',
client_number, sample_link)
yield request
self.logger.debug(
'%d: Now I am determining if I should crawl the next client page.',
client_number)
if no_terpenes_counter < MAX_NO_CONSECUTIVE_TERPENES:
self.logger.debug(
'%d: NTterpenes counter is "%d" below limit "%d".',
client_number, no_terpenes_counter,
MAX_NO_CONSECUTIVE_TERPENES)
self.logger.debug('%d: Determining current page', client_number)
currentpage_match = response.xpath(
self.xpath_resultspage_currentpage)
if currentpage_match:
try:
currentpage_num = int(currentpage_match[0].extract())
except ValueError:
currentpage_num = None
if (currentpage_num == None) or (not currentpage_match):
currentpage_num = response.meta[
'data-sclabs-client-currentpage']
self.logger.debug('%d: Current page seems to be "%d".',
client_number, currentpage_num)
self.logger.debug('%d: Determining last (maximum) page of client.',
client_number)
maxpage_match = response.xpath(self.xpath_resultspage_maxpage)
if maxpage_match:
try:
maxpage_num = int(maxpage_match[0].extract())
except ValueError:
maxpage_num = None
if (maxpage_num == None) or (not maxpage_match):
maxpage_num = response.meta[
'data-sclabs-client-lastknown-maxpage']
self.logger.debug('%d: Last client page seems to be "%d".',
client_number, maxpage_num)
if currentpage_num < maxpage_num and currentpage_num < 10000:
self.logger.debug('%d: Current page is below limit :)',
client_number)
wanted_link = self.template_resultspage.format(
client_number=client_number, page=currentpage_num + 1)
request = scrapy.Request(url=wanted_link,
callback=self.read_client,
errback=self.errback)
request.meta[
'data-sclabs-client-lastknown-maxpage'] = maxpage_num
request.meta[
'data-sclabs-client-currentpage'] = currentpage_num + 1
request.meta['data-sclabs-client-number'] = client_number
request.meta[
'data-sclabs-NTterpenes-counter'] = no_terpenes_counter
# if currentpage_num+1 == maxpage_num:
# request.meta['dont_cache'] = True
next_page = self.session.query(
db.Page).filter(db.Page.source_url == wanted_link).first()
if next_page:
self.logger.debug('%d: Next client page already in DB.',
client_number)
next_page.prev_url = response.url
next_page.online_database_id = self.lab_id
else:
self.logger.debug('%d: Next client page not in DB yet.',
client_number)
next_page = db.Page(source_url=wanted_link,
prev_url=response.url,
online_database_id=self.lab_id)
self.session.add(next_page)
self.session.commit()
self.logger.debug('%d: Yielding request for next client page.',
client_number)
yield request