PreparePaths(),
WriteInfo(),
DownloadUrlFile(control),
WgetDownload(
wpull_args,
accept_on_exit_code=AcceptAny(),
env={
'ITEM_IDENT': ItemInterpolation('%(ident)s'),
'LOG_KEY': ItemInterpolation('%(log_key)s'),
'REDIS_URL': REDIS_URL,
'PATH': os.environ['PATH']
}
),
RelabelIfAborted(control),
WriteInfo(),
MoveFiles(),
LimitConcurrent(2,
RsyncUpload(
target = RSYNC_URL,
target_source_path = ItemInterpolation("%(data_dir)s"),
files=ItemValue("all_target_files"),
extra_args = [
'--partial',
'--partial-dir', '.rsync-tmp'
]
)
),
StopHeartbeat(),
MarkItemAsDone(control, EXPIRE_TIME)
)
pipeline = Pipeline(
CheckIP(),
GetItemFromQueue(control,
pipeline_id,
downloader,
ao_only=env.get('AO_ONLY'),
large=env.get('LARGE')), StartHeartbeat(control),
SetFetchDepth(), PreparePaths(), WriteInfo(), DownloadUrlFile(control),
WgetDownload(wpull_args,
accept_on_exit_code=AcceptAny(),
env={
'ITEM_IDENT': ItemInterpolation('%(ident)s'),
'LOG_KEY': ItemInterpolation('%(log_key)s'),
'REDIS_URL': REDIS_URL,
'PATH': os.environ['PATH']
}), RelabelIfAborted(control), WriteInfo(), MoveFiles(),
LimitConcurrent(
2,
RsyncUpload(target=RSYNC_URL,
target_source_path=ItemInterpolation("%(data_dir)s"),
files=ItemValue("all_target_files"),
extra_args=['--partial', '--partial-dir', '.rsync-tmp'])),
StopHeartbeat(), MarkItemAsDone(control, EXPIRE_TIME))
def stop_control():
#control.flag_logging_thread_for_termination()
control.unregister_pipeline(pipeline_id)
pipeline.on_cleanup += stop_control
pipeline_id,
downloader,
ao_only=env.get('AO_ONLY'),
large=env.get('LARGE'),
version_check=(VERSION, pipeline_version)),
StartHeartbeat(control), SetFetchDepth(), PreparePaths(), WriteInfo(),
DownloadUrlFile(control),
Wpull(wpull_args,
accept_on_exit_code=AcceptAny(),
env={
'ITEM_IDENT': ItemInterpolation('%(ident)s'),
'LOG_KEY': ItemInterpolation('%(log_key)s'),
'REDIS_URL': REDIS_URL,
'PATH': os.environ['PATH']
}), RelabelIfAborted(control), CompressLogIfFailed(), WriteInfo(),
MoveFiles(target_directory=os.environ["FINISHED_WARCS_DIR"]),
StopHeartbeat(), MarkItemAsDone(control, EXPIRE_TIME))
def stop_control():
#control.flag_logging_thread_for_termination()
control.unregister_pipeline(pipeline_id)
pipeline.on_cleanup += stop_control
pipeline.running_status = "Running"
def status_running():
pipeline.running_status = "Running"