def move_warcs(self): while True: for dir_ in [ dir_ for dir_ in os.listdir('.') if os.path.isdir(dir_) ]: if dir_ == settings.dir_ready: continue while not settings.upload_running: time.sleep(1) files = [ file for file in os.listdir(dir_) if os.path.isfile(os.path.join(dir_, file)) and file.endswith('.warc.gz') ] grab_finished = len( filter(lambda file: file.endswith('-meta.warc.gz'), files)) != 0 if grab_finished: for file in files: os.rename(os.path.join(dir_, file), os.path.join(settings.dir_ready, file)) else: for file in files: warc_num = int(file[-13:-8]) warc_num_second = str(warc_num + 1).zfill(5) if file[:-13] + warc_num_second + '.warc.gz' in files: os.rename(os.path.join(dir_, file), os.path.join(settings.dir_ready, file)) if grab_finished: shutil.rmtree(dir_) self.upload() time.sleep(10)
def upload(self): for file in [file for file in os.listdir(settings.dir_ready) if file.endswith('.warc.gz') and not os.path.isfile(os.path.join(settings.dir_ready, file+'.upload'))]: time.sleep(1) if self.concurrent_uploads > settings.max_concurrent_uploads: time.sleep(1) self.uploads[file] = threading.Thread(target=self.upload_single, args=(file,)) self.uploads[file].daemon = True self.uploads[file].start()
def upload(self): while True: for file in [ file for file in os.listdir(settings.dir_ready) if file.endswith('.warc.gz') and not os.path.isfile( os.path.join(settings.dir_ready, file + '.upload')) ]: while not settings.upload_running: time.sleep(1) time.sleep(1) while self.concurrent_uploads > settings.max_concurrent_uploads or not self.upload_allowed( ): time.sleep(10) self.concurrent_uploads += 1 open(os.path.join(settings.dir_ready, file + '.upload'), 'a').close() date = re.search(r'([0-9]{4}-[0-9]{2}-[0-9]{2})', file).group(1) if not file in self.uploads: self.uploads[file] = {} self.uploads[file]['date'] = date.replace('-', '') self.uploads[file]['size'] = os.path.getsize(file) if not self.uploads[file]['date'] in self.items: self.items[self.uploads[file]['date']] = {} self.items[self.uploads[file]['date']]['item_num'] = 0 self.items[self.uploads[file]['date']]['item_size'] = 0 elif self.items[self.uploads[file]['date']][ 'item_size'] > self.max_item_size: self.items[self.uploads[file]['date']]['item_num'] += 1 self.items[self.uploads[file]['date']]['item_size'] = 0 self.items[self.uploads[file]['date']][ 'item_size'] += self.uploads[file]['size'] self.uploads[file]['item_num'] = self.items[ self.uploads[file]['date']]['item_num'] self.uploads[file]['item_size'] = self.items[ self.uploads[file]['date']]['item_size'] name = self.uploads[file]['date'] + str( self.uploads[file]['item_num']).zfill(4) ia_args = { 'title': 'Archive Team Newsgrab: {name}'.format(name=name), 'mediatype': 'web', 'description': 'A collection of news articles grabbed from a wide variety of sources around the world automatically by Archive Team scripts.', 'collection': 'archiveteam_newssites', 'date': date } threading.thread(target=self.upload_single, args=(name, file, ia_args)).start() self.concurrent_uploads -= 1 os.remove(os.path.join(settings.dir_ready, file + '.upload')) if os.path.isfile(os.path.join(settings.dir_ready, file)): settings.irc_bot.send( 'PRIVMSG', '{name} uploaded unsuccessful.'.format(name=file), settings.irc_channel_bot)
def start_services(self): for key, value in settings.services.iteritems(): settings.services[key].running = False for file in [file for file in os.listdir('services') if file.startswith( 'web__') and file.endswith('.py')]: service_name = file.replace('.py', '') if not service_name in self.assigned_services: continue self.services.append(service_name) settings.services[service_name] = Service(service_name) settings.services[service_name].daemon = True settings.services[service_name].start()
def start_services(self): for key, value in settings.services.iteritems(): settings.services[key].running = False for file in [file for file in os.listdir('services') if file.startswith( 'web__') and file.endswith('.py')]: service_name = file.replace('.py', '') if not service_name in self.assigned_services: continue self.services.append(service_name) settings.services[service_name] = Service(service_name) settings.services[service_name].daemon = True settings.services[service_name].start()
def upload(self): for file in [ file for file in os.listdir(settings.dir_ready) if file.endswith('.warc.gz') and not os.path.isfile( os.path.join(settings.dir_ready, file + '.upload')) ]: while not settings.upload_running: time.sleep(1) time.sleep(1) while self.concurrent_uploads > self.max_concurrent_uploads: time.sleep(1) self.uploads[file] = threading.Thread(target=self.upload_single, args=(file, )) self.uploads[file].daemon = True self.uploads[file].start()
def start_services(self): for file in [file for file in os.listdir('services') if file.startswith( 'web__') and file.endswith('.py')]: service_name = file.replace('.py', '') if not service_name in self.services: self.new_services += 1 self.services.append(service_name) settings.services[service_name] = Service(service_name) settings.services[service_name].daemon = True settings.services[service_name].start() settings.services[service_name].read_urls() settings.services[service_name].get_data() settings.irc_bot.send('PRIVMSG', 'Found {new_services} new services'.format( new_services=self.new_services), settings.irc_channel_bot) self.new_services = 0 if not settings.get_urls: settings.get_urls = Urls() settings.get_urls.daemon = True settings.get_urls.start() self.distribute_services()
def start_services(self): for file in [file for file in os.listdir('services') if file.startswith( 'web__') and file.endswith('.py')]: service_name = file.replace('.py', '') if not service_name in self.services: self.new_services += 1 self.services.append(service_name) settings.services[service_name] = Service(service_name) settings.services[service_name].daemon = True settings.services[service_name].start() settings.services[service_name].read_urls() settings.services[service_name].get_data() settings.irc_bot.send('PRIVMSG', 'Found {new_services} new services'.format( new_services=self.new_services), settings.irc_channel_bot) self.new_services = 0 if not settings.get_urls: settings.get_urls = Urls() settings.get_urls.daemon = True settings.get_urls.start() self.distribute_services()
def move_warcs(self): while True: for dir_ in [dir_ for dir_ in os.listdir('.') if os.path.isdir(dir_)]: if dir_ == settings.dir_ready: continue if not settings.upload_running: time.sleep(1) files = [file for file in os.listdir(dir_) if os.path.isfile(os.path.join(dir_, file)) and file.endswith('.warc.gz')] grab_finished = len(filter(lambda file: file.endswith('-meta.warc.gz'), files)) != 0 if grab_finished: for file in files: os.rename(os.path.join(dir_, file), os.path.join(settings.dir_ready, file)) else: for file in files: warc_num = int(file[-13:-8]) warc_num_second = str(warc_num + 1).zfill(5) if file[:-13] + warc_num_second + '.warc.gz' in file: os.rename(os.path.join(dir_, file), os.path.join(settings.dir_ready, file)) if grab_finished: shutil.rmtree(dir_) self.upload()
def upload(self): while True: for file in [ file for file in os.listdir(settings.dir_ready) if file.endswith('.warc.gz') and not os.path.isfile(os.path.join(settings.dir_ready, file+'.upload'))]: while not settings.upload_running: time.sleep(1) time.sleep(1) if concurrent_uploads > settings.max_concurrent_uploads or not self.upload_allowed(): time.sleep(10) concurrent_uploads += 1 open(os.path.join(settings.dir_ready, file+'.upload'), 'a').close() date = re.search(r'([0-9]{4}-[0-9]{2}-[0-9]{2})', file).group(1) if not file in self.uploads: self.uploads[file] = {} self.uploads[file]['date'] = date.replace('-', '') self.uploads[file]['size'] = os.path.getsize(file) if not self.uploads[file]['date'] in self.items: self.items[self.uploads[file]['date']] = {} self.items[self.uploads[file]['date']]['item_num'] = 0 self.items[self.uploads[file]['date']]['item_size'] = 0 elif self.items[self.uploads[file]['date']]['item_size'] > self.max_item_size: self.items[self.uploads[file]['date']]['item_num'] += 1 self.items[self.uploads[file]['date']]['item_size'] = 0 self.items[self.uploads[file]['date']]['item_size'] += self.uploads[file]['size'] self.uploads[file]['item_num'] = self.items[self.uploads[file]['date']]['item_num'] self.uploads[file]['item_size'] = self.items[self.uploads[file]['date']]['item_size'] name = self.uploads[file]['date']+str(self.uploads[file]['item_num']).zfill(4) ia_args = {'title': 'Archive Team Newsgrab: {name}'.format(name=name), 'mediatype': 'web', 'description': 'A collection of news articles grabbed from a wide variety of sources around the world automatically by Archive Team scripts.', 'collection': 'archiveteam_newssites', 'date': date} threading.thread(target=self.upload_single, args=(name, file, ia_args)).start() concurrent_uploads -= 1 os.remove(os.path.join(settings.dir_ready, file+'.upload')) if os.path.isfile(os.path.join(settings.dir_ready, file)): settings.irc_bot.send('PRIVMSG', '{name} uploaded unsuccessful.'.format( name=file), settings.irc_channel_bot)