def work(self, task):
     entry_name = task.split('-=-=-')[0]
     url = task.split('-=-=-')[1]
     entry_name, data = self._fetch_data(entry_name, url)
     if not data:
         return None
     entry_name = cogtu_misc.escape_fwd_slash(entry_name)
     filename = self.get_save_path() + '/' + entry_name
     self.write_data_to_file(data=data, filename=filename)
     return filename
 def work(self, task):
     if not task:
         logging.info('keyword is invalid. Skip')
         return None
     keyword = task['keyword']
     num_urls = int(task['num_urls'])
     image_urls = self.fetch_image_urls(keyword, num_urls)
     unique_filename = str(keyword) + '.txt'
     unique_filename = cogtu_misc.escape_space(cogtu_misc.escape_fwd_slash(unique_filename))
     self.write_data_to_file(image_urls, unique_filename)
     return unique_filename
 def work(self, keyword):
     if not keyword:
         logging.info('keyword is invalid. Skip')
         return None
     # s = random.random() * 10.0
     # logging.info('SLEEPING FOR ' + str(s) + ' SECONDS')
     # sleep(s)
     # logging.info('SLEPT')
     info = self.fetch_info(keyword)
     unique_filename = str(keyword) + '.txt'
     unique_filename = cogtu_misc.escape_space(cogtu_misc.escape_fwd_slash(unique_filename))
     self.write_data_to_file(info, unique_filename)
     return unique_filename
Esempio n. 4
0
 def work(self, url):
     entry_name, data = self._fetch_data(url)
     entry_name = cogtu_misc.escape_fwd_slash(entry_name)
     filename = self.get_save_path() + '/' + entry_name
     self.write_data_to_file(data=data, filename=filename)
     return filename