def dmrpp_generate(self, input_file, local=False, dmrpp_meta=None):
"""
Generate DMRPP from S3 file
"""
# Force dmrpp_meta to be an object
dmrpp_meta = dmrpp_meta if isinstance(dmrpp_meta, dict) else {}
# If not running locally use Cumulus logger
logger = logging if local else self.logger
cmd_output = ""
try:
file_name = input_file if local else s3.download(input_file,
path=self.path)
cmd = self.get_dmrpp_command(dmrpp_meta, self.path, file_name,
local)
cmd_output = self.run_command(cmd)
logger.error(
f"{self.dmrpp_version}: command {cmd} returned {cmd_output.stderr}"
) if cmd_output.stderr else ""
out_files = [f"{file_name}.dmrpp"] + self.add_missing_files(
dmrpp_meta, f'{file_name}.dmrpp.missing')
return out_files
except Exception as ex:
logger.error(
f"{self.dmrpp_version}: error {ex}: {cmd_output.stdout} {cmd_output.stderr}"
)
return []
def test_download(self):
""" Download file from S3 """
# first upload something
uri = self.s3path + '/file.txt'
s3.upload(self.payload, uri)
fout = s3.download(uri, path=self.path)
self.assertEqual(fout, os.path.join(self.path, 'file.txt'))
s3.delete(uri)
os.remove(fout)
def dmrpp_generate(self, input_file):
"""
"""
try:
file_name = s3.download(input_file, path=self.path)
cmd = f"get_dmrpp -b {self.path} -o {file_name}.dmrpp {os.path.basename(file_name)}"
self.run_command(cmd)
return f"{file_name}.dmrpp"
except Exception as ex:
self.logger.error(f"DMRPP error {ex}")
return None
def dmrpp_generate(self, input_file):
"""
"""
if not match(f"{self.processing_regex}$", input_file):
return {}
try:
file_name = s3.download(input_file, path=self.path)
cmd = f"get_dmrpp -b {self.path} -o {file_name}.dmrpp {os.path.basename(file_name)}"
self.run_command(cmd)
return {'file_local_path': f"{file_name}.dmrpp", 's3_path': self.upload_file(f"{file_name}.dmrpp")}
except Exception as ex:
self.logger.error(f"DMRPP error {ex}")
return {}
def fetch(self, key, remote=False):
""" Get local (default) or remote input filename """
regex = self.input_keys.get(key, None)
if regex is None:
raise Exception('No files matching %s' % regex)
outfiles = []
for f in self.input:
m = re.match(regex, os.path.basename(f))
if m is not None:
# if remote desired, or input is already local
if remote or os.path.exists(f):
outfiles.append(f)
else:
fname = download(f, path=self.path)
outfiles.append(fname)
self.downloads.append(fname)
return outfiles
def dmrpp_generate(self, input_file, local=False, dmrpp_meta=None):
"""
Generate DMRPP from S3 file
"""
# Force dmrpp_meta to be an object
dmrpp_meta = dmrpp_meta if isinstance(dmrpp_meta, dict) else {}
try:
file_name = input_file if local else s3.download(input_file,
path=self.path)
cmd = self.get_dmrpp_command(dmrpp_meta, self.path, file_name)
self.run_command(cmd)
out_files = [f"{file_name}.dmrpp"]
if "-M" in dmrpp_meta.values():
out_files += [f"{file_name}.missing"]
return out_files
except Exception as ex:
self.logger.error(f"DMRPP error {ex}")
return []