def fetch_input_files_from_s3(input_files, input_dir_s3, result_dir_local):
for f in input_files:
s3_file = os.path.join(input_dir_s3, f)
local_file = os.path.join(result_dir_local, f)
local_dir = os.path.dirname(local_file)
command.make_dirs(local_dir)
# copy the file over
output_file = idseq_dag.util.s3.fetch_from_s3(s3_file, local_dir, allow_s3mi=True)
if output_file:
# write the done_file
done_file = PipelineStep.done_file(local_file)
fmt_now = datetime.datetime.now(tz=pytz.UTC).strftime("%a %b %e %H:%M:%S %Z %Y")
command.write_text_to_file(fmt_now, done_file)
else:
raise RuntimeError(f"{s3_file} likely doesn't exist")
def fetch_input_files_from_s3(input_files, input_dir_s3, result_dir_local):
for f in input_files:
s3_file = os.path.join(input_dir_s3, f)
local_file = os.path.join(result_dir_local, f)
local_dir = os.path.dirname(local_file)
command.execute("mkdir -p %s" % local_dir)
# copy the file over
output_file = idseq_dag.util.s3.fetch_from_s3(s3_file,
local_dir,
allow_s3mi=True)
if output_file:
# write the done_file
done_file = PipelineStep.done_file(local_file)
command.execute("date > %s" % done_file)
else:
raise RuntimeError(f"{s3_file} likely doesn't exist")