def data_fetch(self, url, task):
'''A fake fetcher for dataurl'''
self.on_fetch('data', task)
result = {}
result['orig_url'] = url
result['content'] = dataurl.decode(url)
result['headers'] = {}
result['status_code'] = 200
result['url'] = url
result['cookies'] = {}
result['time'] = 0
result['save'] = task.get('fetch', {}).get('save')
if len(result['content']) < 70:
logger.info("[200] %s:%s %s 0s", task.get('project'),
task.get('taskid'), url)
else:
logger.info("[200] %s:%s data:,%s...[content:%d] 0s",
task.get('project'), task.get('taskid'),
result['content'][:70], len(result['content']))
return result
def data_fetch(self, url, task):
'''A fake fetcher for dataurl'''
self.on_fetch('data', task)
result = {}
result['orig_url'] = url
result['content'] = dataurl.decode(url)
result['headers'] = {}
result['status_code'] = 200
result['url'] = url
result['cookies'] = {}
result['time'] = 0
result['save'] = task.get('fetch', {}).get('save')
if len(result['content']) < 70:
logger.info("[200] %s:%s %s 0s", task.get('project'), task.get('taskid'), url)
else:
logger.info(
"[200] %s:%s data:,%s...[content:%d] 0s",
task.get('project'), task.get('taskid'),
result['content'][:70],
len(result['content'])
)
return result