def work(self, task):
entry_name = task.split('-=-=-')[0]
url = task.split('-=-=-')[1]
entry_name, data = self._fetch_data(entry_name, url)
if not data:
return None
entry_name = cogtu_misc.escape_fwd_slash(entry_name)
filename = self.get_save_path() + '/' + entry_name
self.write_data_to_file(data=data, filename=filename)
return filename
def work(self, task):
if not task:
logging.info('keyword is invalid. Skip')
return None
keyword = task['keyword']
num_urls = int(task['num_urls'])
image_urls = self.fetch_image_urls(keyword, num_urls)
unique_filename = str(keyword) + '.txt'
unique_filename = cogtu_misc.escape_space(cogtu_misc.escape_fwd_slash(unique_filename))
self.write_data_to_file(image_urls, unique_filename)
return unique_filename
def work(self, keyword):
if not keyword:
logging.info('keyword is invalid. Skip')
return None
# s = random.random() * 10.0
# logging.info('SLEEPING FOR ' + str(s) + ' SECONDS')
# sleep(s)
# logging.info('SLEPT')
info = self.fetch_info(keyword)
unique_filename = str(keyword) + '.txt'
unique_filename = cogtu_misc.escape_space(cogtu_misc.escape_fwd_slash(unique_filename))
self.write_data_to_file(info, unique_filename)
return unique_filename
def work(self, url):
entry_name, data = self._fetch_data(url)
entry_name = cogtu_misc.escape_fwd_slash(entry_name)
filename = self.get_save_path() + '/' + entry_name
self.write_data_to_file(data=data, filename=filename)
return filename