def __init__(self, args):
print(args)
ExternalProcess.__init__(
self,
"UploadToIA",
args=args,
)
def __init__(self, args, max_tries=1, accept_on_exit_code=None,
retry_on_exit_code=None, env=None, stdin_data_function=None):
ExternalProcess.__init__(
self, "YgaDownload",
args=args, max_tries=max_tries,
accept_on_exit_code=(accept_on_exit_code
if accept_on_exit_code is not None else [0]),
retry_on_exit_code=retry_on_exit_code,
env=env)
self.stdin_data_function = stdin_data_function
def __init__(self):
args = [
sys.executable,
os.path.join(PIPELINE_DIR, "warc2warc_greader.py"),
"--gzip",
"--decode_http",
"--output", ItemInterpolation("%(data_dir)s/%(warc_file_base)s.cooked.warc.gz"),
ItemInterpolation("%(data_dir)s/%(warc_file_base)s.warc.gz")
]
ExternalProcess.__init__(self, "CookWARC", args)
def __init__(self):
env = {'PYTHONPATH': 'terroroftinytown'}
ExternalProcess.__init__(
self,
'RunScraper', [
sys.executable, 'scraper.py', TRACKER_HOST, VERSION,
globals()['downloader'],
globals().get('bind_address', ''), USER_AGENT
],
env=env)
def __init__(self):
args = [
sys.executable,
os.path.join(PIPELINE_DIR, "warc2warc_greader.py"), "--gzip",
"--decode_http", "--strip-404s", "--json-hrefs-file",
ItemInterpolation("%(data_dir)s/%(warc_file_base)s.hrefs.bz2"),
"--output",
ItemInterpolation(
"%(data_dir)s/%(warc_file_base)s.cooked.warc.gz"),
ItemInterpolation("%(data_dir)s/%(warc_file_base)s.warc.gz")
]
ExternalProcess.__init__(self, "CookWARC", args)
def __init__(self, args):
ExternalProcess.__init__(
self, "DeduplicateWarcExtProc", args=args, accept_on_exit_code=[0],
retry_on_exit_code=[2])
def __init__(self):
ExternalProcess.__init__(self,
'UpdateSubmodule',
self.NEW_ARGS,
max_tries=5,
retry_delay=2)
def __init__(self, *args, **kwargs):
ExternalProcess.__init__(self, *args, **kwargs)
self.output_buffer = StringIO()
self.return_code = None
self.exit_count = 0
self.retry_delay = 0.1
def __init__(self, args):
ExternalProcess.__init__(
self,
"WgetDownload",
args=args,
)
def __init__(self, args):
ExternalProcess.__init__(
self,
"DeduplicateWarcExtProc",
args=args,
)
def __init__(self, *args, **kwargs):
ExternalProcess.__init__(self, *args, **kwargs)
self.output_buffer = StringIO()
self.return_code = None
self.exit_count = 0
self.retry_delay = 0.1
