def run_as(cmd, user=None, pythonpath=None, shell=False, rc_ok=0):
"""
Run the given command `cmd` as the given UNIX username `user`. This assumes
that the current UNIX user is allowed to do so (i.e. there is an appropriate
entry in /etc/sudoers or equivalent).
`cmd` is a list of the form [executable, arg1, arg2, ...]
`user` is a string. If None, run as the current user.
return (exit code, job id list)
"""
# Make sure that we convert all Unicode str in cmd to simple strings.
cmd = [str(ch) for ch in cmd]
if user is None or user == getpass.getuser():
sudoed_cmd = cmd
else:
# environment variable to passed with the sudo command
sudoed_cmd = ["sudo", "-E", "-su", str(user)] + DB_ENV + cmd
logger.get_log().info("Executing %s" % sudoed_cmd)
proc = subprocess.Popen(sudoed_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell)
(out, err) = proc.communicate()
ec = proc.returncode
proc.stdout.close()
proc.stderr.close()
return (ec, err, out)
def run_as(cmd, user=None, pythonpath=None, shell=False, rc_ok=0):
"""
Run the given command `cmd` as the given UNIX username `user`. This assumes
that the current UNIX user is allowed to do so (i.e. there is an appropriate
entry in /etc/sudoers or equivalent).
`cmd` is a list of the form [executable, arg1, arg2, ...]
`user` is a string. If None, run as the current user.
return (exit code, job id list)
"""
# Make sure that we convert all Unicode str in cmd to simple strings.
cmd = [str(ch) for ch in cmd]
if user is None or user == getpass.getuser():
sudoed_cmd = cmd
else:
#environment variable to passed with the sudo command
sudoed_cmd = ['sudo', '-E', '-su', str(user), ] + DB_ENV + cmd
logger.get_log().info("Executing %s" % sudoed_cmd)
proc = subprocess.Popen(sudoed_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=shell)
(out, err) = proc.communicate()
ec = proc.returncode
proc.stdout.close()
proc.stderr.close()
return (ec, err, out)
def __init__(self):
"""
Just initialize the logger
"""
self.log = logger.get_log()
def __init__(self, name, spec, steps, user, run_id=None, output_dir=None):
self.log = logger.get_log()
def __init__(self, name, spec, steps, user, run_id=None, output_dir=None):
self.log = logger.get_log()
class LocalScheduler(Scheduler):
""" Methods for job submission and status reporting for a 'Local' scheduler
In the context of pipelines this is mainly for testing purposes
"""
log = logger.get_log()
job_ids = {}
def submit(self, cmd, cmd_args, work_dir, reqs=None):
""" submits a command 'locally' in the background
Returns the pid as job id
"""
if os.path.exists(os.path.join(work_dir, FAILED_FILE)):
os.remove(os.path.join(work_dir, FAILED_FILE))
newpid = os.fork()
if newpid == 0:
self.__submit(cmd, cmd_args, work_dir)
return
else:
self.job_ids[newpid] = {'cwd': work_dir}
return newpid
def __submit(self, cmd, cmd_args, work_dir):
self.log.info('cmd_args: %s, cmd: %s, work_dir: %s' %
(cmd_args, cmd, work_dir))
cmd_args.insert(0, cmd)
err = open(os.path.join(work_dir, JOB_ERR), 'w')
out = open(os.path.join(work_dir, JOB_OUT), 'w')
err = subprocess.call(cmd_args, stderr=err, stdout=out)
if err != 0:
with open(os.path.join(work_dir, FAILED_FILE), 'w') as fh:
fh.write('[%s] Failed with error code %d\n' %
(datetime.datetime.now(), err))
os._exit(err)
def stop(self, job_ids):
"""
Stop all jobs
"""
job_ids = set(job_ids)
for job_id in job_ids:
p = psutil.Process(job_id)
p.terminate()
def status(self, job_ids):
""" Returns the status of a pid from a local background 'job'
"""
job_ids = set(job_ids)
status_map = dict.fromkeys(job_ids, 'Unknown')
for job_id in job_ids:
job_failed_file = os.path.join(
self.job_ids.get(job_id, {}).get('cwd'), FAILED_FILE)
if not psutil.pid_exists(int(job_id)):
if os.path.exists(job_failed_file):
status_map[job_id] = JOB_STATUS.FAILED
else:
status_map[job_id] = JOB_STATUS.SUCCEEDED
elif psutil.Process(job_id).status() == psutil.STATUS_ZOMBIE:
if os.path.exists(job_failed_file):
status_map[job_id] = JOB_STATUS.FAILED
else:
status_map[job_id] = JOB_STATUS.SUCCEEDED
else:
status_map[job_id] = JOB_STATUS.RUNNING
return status_map
def __init__(self):
"""
Just initialize the logger
"""
self.log = logger.get_log()