def getShellOutput(cmd, verbose=False):
"""Function to run a shell command and return returncode, stdout and stderr
Parameters
----------
cdm: list
command to run
verbose: bool
to print messages
:return: (returncode, stdout and stderr)
:rtype: tuple
"""
#not logging these commands
log_message = " ".join(cmd)
if verbose:
pu.print_blue("$ " + log_message)
try:
result = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
stdout, stderr = result.communicate()
return (result.returncode, stdout, stderr)
except:
return (-1, "", "Command failed to execute")
def generate_multiqc_from_log(logFile,
filterList,
tempDir,
outDir="",
coverage='a',
verbose=False,
cleanup=False):
#dump stdout from logs to temp directory
stdout = getStdoutFromLog(logFile, filterList, coverage)
#create tmpdir
pu.mkdir(tempDir)
flist = []
for o in stdout:
thisName = o + ".txt"
tempFile = os.path.join(tempDir, thisName)
# print("opening:"+tempFile)
f = open(tempFile, "w")
f.write(stdout[o])
#rint(stdout[o])
f.close()
flist.append(tempFile)
#run multiqc
#tempDir stores .txt files for MQC to read
mc.run(analysis_dir=tempDir, outdir=outDir)
#cleanup
if cleanup:
for f in flist:
pu.print_blue("Removing {}".format(f))
os.remove(f)
def get_shell_output(cmd, verbose=None):
"""Function to run a shell command and return returncode, stdout and stderr
Currently (pyrpipe v 0.0.4) this function is called in
get_return_status(), get_program_version()
pyrpipe v0.0.5 onwards the get_shell_output function allows shell=True
Parameters
----------
cdm: list or string
command to run
verbose: bool
to print messages
:return: (returncode, stdout and stderr)
:rtype: tuple: (int,str,str)
"""
if verbose == None: verbose = _verbose
#not logging these commands
cmd = parse_cmd(cmd)
log_message = cmd
command_name = cmd.split()[0]
if _safe and command_name in ['rm']:
pu.print_warning('SAFE MODE: Skipping command {}'.format(cmd))
return True
starttime_str = time.strftime("%y-%m-%d %H:%M:%S",
time.localtime(time.time()))
if verbose:
pu.print_notification("Start:" + starttime_str)
pu.print_blue("$ " + log_message)
try:
result = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
shell=True)
stdout, stderr = result.communicate()
if stdout:
stdout = stdout.decode("utf-8")
else:
stdout = ''
if stderr:
stderr = stderr.decode("utf-8")
else:
stderr = ''
return (result.returncode, stdout, stderr)
except:
return (-1, "Command failed to execute", "Command failed to execute")
def __init__(self,log_file,env_log,out_dir=""):
if not pu.check_files_exist(log_file,env_log):
raise Exception("Please check input for benchmark report. {} {}".format(log_file,env_log))
if not out_dir:
out_dir=os.getcwd()
self.log_file=log_file
self.env_log=env_log
self.runtimes_by_prog={}
self.runtimes_by_object={}
#init
pu.print_blue("parsing log...")
self.parse_logs()
pu.print_blue("done.")
#out_dir
self.benchmark_dir=os.path.join(out_dir,'benchmark_reports')
if not pu.check_paths_exist(self.benchmark_dir):
if not pu.mkdir(self.benchmark_dir):
raise Exception("Error running benchmarks. Can not create output directory {}".format(self.benchmark_dir))
def dried(cmd, *args, **kwargs):
#print, log and exit
starttime_str = time.strftime("%y-%m-%d %H:%M:%S",
time.localtime(time.time()))
#convert cmd to string
cmd = parse_cmd(cmd)
command_name = cmd.split(' ')[0]
log_message = cmd
pu.print_blue("$ " + log_message)
#log
#create a dict and dump as json
logDict = {
'cmd': log_message,
'exitcode': "0",
'runtime': "0",
'starttime': str(starttime_str),
'stdout': "dryrun",
'stderr': "",
'objectid': '',
'commandname': command_name
}
if _logging:
pyrpipe_logger.cmd_logger.debug(json.dumps(logDict))
return True
def execute_command(cmd,
verbose=False,
quiet=False,
logs=True,
dryrun=False,
objectid="NA",
command_name=""):
"""Function to execute commands using popen.
All commands executed by this function can be logged and saved to pyrpipe logs.
Parameters
----------
cmd: list
command to execute via popen in a list
verbose: bool
Whether to print stdout and stderr. Default: False. All stdout and stderr will be saved to logs regardless of this flag.
quiet: bool
Absolutely no output on screen
logs: bool
Log the execution
dryrun: bool
If True, perform a dry run i.e. print commands to screen and log and exit
objectid: string
An id to be attached with the command. This is useful fo storing logs for SRA objects where object id is the SRR id.
command_name: string
Name of command to be save in log. If empty it is determined as the first element of the cmd list.
:return: Return status.True is returncode is 0
:rtype: bool
"""
if not command_name:
command_name = cmd[0]
log_message = " ".join(cmd)
#dryrun: print and exit
if dryrun:
pu.print_blue("$ " + log_message)
#log
#create a dict and dump as json
logDict = {
'cmd': log_message,
'exitcode': "0",
'runtime': "0",
'starttime': "0",
'stdout': "dryrun",
'stderr': "",
'objectid': objectid,
'commandname': command_name
}
pyrpipeLoggerObject.cmd_logger.debug(json.dumps(logDict))
return True
if not quiet:
pu.print_blue("$ " + log_message)
time_start = time.time()
starttime_str = time.strftime("%y-%m-%d %H:%M:%S",
time.localtime(time.time()))
try:
result = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
stdout, stderr = result.communicate()
#convert to string
if stdout:
stdout = stdout.decode("utf-8")
else:
stdout = ""
if stderr:
stderr = stderr.decode("utf-8")
else:
stderr = ""
timeDiff = round(time.time() -
time_start) #round to remove microsecond term
if verbose:
if stdout:
pu.print_blue("STDOUT:\n" + stdout)
if stderr:
pu.print_boldred("STDERR:\n" + stderr)
if not quiet:
pu.print_green("Time taken:" + str(timedelta(seconds=timeDiff)))
exitCode = result.returncode
##Add to logs
if logs:
##get the program used and log its path
if command_name not in pyrpipeLoggerObject.logged_programs:
##get which thisProgram
#if subcommands are present use parent command
parent_command = cmd[0]
progDesc = {
'name': command_name,
'version': getProgramVersion(parent_command).strip(),
'path': getProgramPath(parent_command).strip()
}
pyrpipeLoggerObject.env_logger.debug(json.dumps(progDesc))
pyrpipeLoggerObject.logged_programs.append(command_name)
#create a dict and dump as json
logDict = {
'cmd': log_message,
'exitcode': exitCode,
'runtime': str(timedelta(seconds=timeDiff)),
'starttime': str(starttime_str),
'stdout': stdout,
'stderr': stderr,
'objectid': objectid,
'commandname': command_name
}
pyrpipeLoggerObject.cmd_logger.debug(json.dumps(logDict))
if exitCode == 0:
return True
else:
#print the output
print(
"Following error occured executing above command (return code={}):"
.format(str(exitCode)))
print("STDOUT:\n" + stdout)
print("STDERR:\n" + stderr)
return False
#handle exceptions
except OSError as e:
pu.print_boldred("OSError exception occured.\n" + str(e))
#log error
timeDiff = round(time.time() - time_start)
logDict = {
'cmd': log_message,
'exitcode': '-1',
'runtime': str(timedelta(seconds=timeDiff)),
'starttime': str(starttime_str),
'stdout': "",
'stderr': "OSError exception occured.\n" + str(e),
'objectid': objectid,
'commandname': command_name
}
pyrpipeLoggerObject.cmd_logger.debug(json.dumps(logDict))
return False
except subprocess.CalledProcessError as e:
pu.print_boldred("CalledProcessError exception occured.\n" + str(e))
#log error
timeDiff = round(time.time() - time_start)
logDict = {
'cmd': log_message,
'exitcode': '-1',
'runtime': str(timedelta(seconds=timeDiff)),
'starttime': str(starttime_str),
'stdout': "",
'stderr': "CalledProcessError exception occured.\n" + str(e),
'objectid': objectid,
'commandname': command_name
}
pyrpipeLoggerObject.cmd_logger.debug(json.dumps(logDict))
return False
except:
pu.print_boldred("Fatal error occured during execution.\n" +
str(sys.exc_info()[0]))
#log error
timeDiff = round(time.time() - time_start)
logDict = {
'cmd':
log_message,
'exitcode':
'-1',
'runtime':
str(timedelta(seconds=timeDiff)),
'starttime':
str(starttime_str),
'stdout':
"",
'stderr':
str("Fatal error occured during execution.\n" +
str(sys.exc_info()[0])),
'objectid':
objectid,
'commandname':
command_name
}
pyrpipeLoggerObject.cmd_logger.debug(json.dumps(logDict))
return False