PrepareStatsForTracker( defaults={ "downloader": downloader, "version": VERSION }, file_groups={ "data": [ItemInterpolation("%(item_dir)s/%(warc_file_base)s.txt.gz")] }, id_function=stats_id_function, ), MoveFiles(), LimitConcurrent( NumberConfigValue( min=1, max=4, default="1", name="shared:rsync_threads", title="Rsync threads", description="The maximum number of concurrent uploads."), UploadWithTracker( "http://%s/%s" % (TRACKER_HOST, TRACKER_ID), downloader=downloader, version=VERSION, files=[ ItemInterpolation("%(data_dir)s/%(warc_file_base)s.txt.gz") ], rsync_target_source_path=ItemInterpolation("%(data_dir)s/"), rsync_extra_args=[ "--recursive", "--partial", "--partial-dir", ".rsync-tmp" ]), ),
PrepareStatsForTracker( defaults={ 'downloader': downloader, 'version': VERSION }, file_groups={ 'data': [ItemInterpolation('%(item_dir)s/%(warc_file_base)s.warc.gz')] }, id_function=stats_id_function, ), MoveFiles(), LimitConcurrent( NumberConfigValue( min=1, max=20, default='20', name='shared:rsync_threads', title='Rsync threads', description='The maximum number of concurrent uploads.'), UploadWithTracker( 'http://%s/%s' % (TRACKER_HOST, TRACKER_ID), downloader=downloader, version=VERSION, files=[ ItemInterpolation('%(data_dir)s/%(warc_file_base)s_data.txt'), ItemInterpolation('%(data_dir)s/%(warc_file_base)s.warc.gz') ], rsync_target_source_path=ItemInterpolation('%(data_dir)s/'), rsync_extra_args=[ '--recursive', '--partial',
ExternalProcess( 'Begin', [sys.executable, 'helper.py', 'begin'], env={ 'user_agent': user_agent, 'bind_address': globals().get('bind_address', ''), 'disco_tracker': DISCO_TRACKER_URL, "item_dir": ItemValue("item_dir"), }, accept_on_exit_code=[0], ), LimitConcurrent( NumberConfigValue( min=1, max=6, default=globals().get("num_procs", "1"), name="shared:fagrab:num_procs", title="Number of Processes", description="The maximum number of concurrent download processes." ), WgetDownload(WgetArgs(), max_tries=1, accept_on_exit_code=[0, 4, 7, 8], env={ "item_dir": ItemValue("item_dir"), "downloader": downloader, "item_name": ItemValue("item_name"), }), ), ExternalProcess( 'End', [sys.executable, 'helper.py', 'end'],
def __init__(self, projects_dir, data_dir, warrior_hq_url, real_shutdown=False, keep_data=False): if not os.access(projects_dir, os.W_OK): raise Exception( "Couldn't write to projects directory: %s" % projects_dir) if not os.access(data_dir, os.W_OK): raise Exception("Couldn't write to data directory: %s" % data_dir) self.projects_dir = projects_dir self.data_dir = data_dir self.warrior_hq_url = warrior_hq_url self.real_shutdown = real_shutdown self.keep_data = keep_data # disable the password prompts self.gitenv = dict( list(os.environ.items()) + list({ 'GIT_ASKPASS': '******', 'SSH_ASKPASS': '******' }.items()) ) self.warrior_id = StringConfigValue( name="warrior_id", title="Warrior ID", description="The unique number of your warrior instance.", editable=False ) self.selected_project_config_value = StringConfigValue( name="selected_project", title="Selected project", description="The project (to be continued when the warrior " "restarts).", default="none", editable=False ) self.downloader = StringConfigValue( name="downloader", title="Your nickname", description="We use your nickname to show your results on our " "tracker. Letters and numbers only.", regex="^[-_a-zA-Z0-9]{3,30}$", advanced=False ) self.concurrent_items = NumberConfigValue( name="concurrent_items", title="Concurrent items", description="How many items should the warrior download at a " "time? (Max: 6)", min=1, max=6, default=2 ) self.http_username = StringConfigValue( name="http_username", title="HTTP username", description="Enter a username to protect the web interface, " "or leave empty.", default="" ) self.http_password = StringConfigValue( name="http_password", title="HTTP password", description="Enter a password to protect the web interface, " "or leave empty.", default="" ) self.config_manager = ConfigManager(os.path.join(projects_dir, "config.json")) self.config_manager.add(self.warrior_id) self.config_manager.add(self.selected_project_config_value) self.config_manager.add(self.downloader) self.config_manager.add(self.concurrent_items) self.config_manager.add(self.http_username) self.config_manager.add(self.http_password) self.bandwidth_monitor = BandwidthMonitor("eth0") self.bandwidth_monitor.update() self.runner = Runner(concurrent_items=self.concurrent_items, keep_data=self.keep_data) self.runner.on_finish += self.handle_runner_finish self.current_project_name = None self.current_project = None self.selected_project = None self.projects = {} self.installed_projects = set() self.failed_projects = set() self.on_projects_loaded = Event() self.on_project_installing = Event() self.on_project_installed = Event() self.on_project_installation_failed = Event() self.on_project_refresh = Event() self.on_project_selected = Event() self.on_status = Event() self.on_broadcast_message_received = Event() self.http_client = AsyncHTTPClient() self.installing = False self.shut_down_flag = False self.reboot_flag = False io_loop = ioloop.IOLoop.instance() def update_warror_callback(): io_loop.add_future( self.update_warrior_hq(), lambda fut: fut.result() ) def update_project_callback(): io_loop.add_future(self.update_project(), lambda fut: fut.result()) self.hq_updater = ioloop.PeriodicCallback(update_warror_callback, 10 * 60 * 1000) self.project_updater = ioloop.PeriodicCallback(update_project_callback, 30 * 60 * 1000) self.forced_reboot_timeout = None self.lat_lng = None self.find_lat_lng() self.install_output = None self.broadcast_message = None self.contacting_hq_failed = False self.internal_log_handler = InternalTempLogHandler() self.internal_log_handler.setFormatter( logging.Formatter(seesaw.script.run_warrior.LOG_FORMAT)) self.internal_log_handler.addFilter( seesaw.script.run_warrior.LogFilter()) logging.getLogger().addHandler(self.internal_log_handler)
"--waitretry", "5" ], max_tries=5, accept_on_exit_code=[ 0 ], ),""" pipeline = Pipeline( GetItemFromTracker("http://%s/%s" % (TRACKER_HOST, TRACKER_ID), downloader, VERSION), PrepareDirectories(file_prefix="isohunt"), LimitConcurrent( NumberConfigValue( min=1, max=10, default="10", name="isohunt:download_threads", title="Isohunt downloading threads", description= "How many threads downloading Isohunt torrents and pages can run at once, to avoid throttling." ), WgetDownloadTorrentRange( [ WGET_LUA, "-U", USER_AGENT, "--no-check-certificate", "-e", "robots=off", "--rotate-dns", "--timeout", "60",