output = subprocess.check_output([WPULL_EXE, '--version'],
stderr=subprocess.STDOUT)
return output.decode('utf-8').strip()
class AcceptAny:
def __contains__(self, item):
return True
DEFAULT_USER_AGENT = \
'ArchiveTeam ArchiveBot/%s (wpull %s) and not Mozilla/5.0 ' \
'(Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) ' \
'Chrome/42.0.2311.90 Safari/537.36' % (VERSION, wpull_version())
_, _, _, pipeline_id = monitoring.pipeline_id()
wpull_args = WpullArgs(default_user_agent=DEFAULT_USER_AGENT,
wpull_exe=WPULL_EXE,
youtube_dl_exe=YOUTUBE_DL,
phantomjs_exe=PHANTOMJS,
finished_warcs_dir=os.environ["FINISHED_WARCS_DIR"],
warc_max_size=WARC_MAX_SIZE)
pipeline = Pipeline(
CheckIP(),
GetItemFromQueue(control,
pipeline_id,
downloader,
ao_only=env.get('AO_ONLY'),
large=env.get('LARGE')), StartHeartbeat(control),
def wpull_version():
output = subprocess.check_output([WPULL_EXE, '--version'],
stderr=subprocess.STDOUT)
return output.decode('utf-8').strip()
class AcceptAny:
def __contains__(self, item):
return True
DEFAULT_USER_AGENT = \
'ArchiveTeam ArchiveBot/%s (wpull %s) and not Mozilla/5.0 ' \
'(Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) ' \
'Chrome/42.0.2311.90 Safari/537.36' % (VERSION, wpull_version())
_, _, _, pipeline_id = monitoring.pipeline_id()
wpull_args = WpullArgs(
default_user_agent=DEFAULT_USER_AGENT,
wpull_exe=WPULL_EXE,
youtube_dl_exe=YOUTUBE_DL,
phantomjs_exe=PHANTOMJS,
finished_warcs_dir=os.environ["FINISHED_WARCS_DIR"]
)
pipeline = Pipeline(
CheckIP(),
GetItemFromQueue(control, pipeline_id, downloader, ao_only=env.get('AO_ONLY')),
StartHeartbeat(control),
SetFetchDepth(),
PreparePaths(),
