def extract(archive, dest='', force=False):
if archive.endswith('.zip') or archive.endswith('.jar'):
a = zipfile.ZipFile(archive, 'r')
sane, internal_dest = check_sanity(a.namelist())
if sane:
exists = os.path.exists(os.path.join(dest, internal_dest))
else:
dest = os.path.join(dest,
os.path.splitext(os.path.basename(archive))[0])
exists = os.path.exists(dest)
if not exists:
os.makedirs(dest)
print 'extracting %s to %s' % (archive, dest)
command.execute(['unzip', '-q', archive, '-d', dest])
else:
print 'extracting %s (already extracted)' % (archive, )
elif archive.endswith('.tgz') or archive.endswith('.tar.gz'):
a = tarfile.open(archive, 'r:*')
sane, internal_dest = check_sanity(a.getnames())
if sane:
exists = os.path.exists(os.path.join(dest, internal_dest))
else:
dest = os.path.join(dest,
os.path.splitext(os.path.basename(archive))[0])
exists = os.path.exists(dest)
if not exists:
if not os.path.exists(dest): os.makedirs(dest)
print 'extracting %s to %s' % (archive, dest)
command.execute(['tar', '-xzf', archive, '-C', dest])
else:
print 'extracting %s (already extracted)' % (archive, )
else:
raise Exception('Archive format (%s) not recognised' % (archive, ))
def extract(archive, dest='', force=False):
if archive.endswith('.zip') or archive.endswith('.jar'):
a = zipfile.ZipFile(archive, 'r')
sane, internal_dest = check_sanity(a.namelist())
if sane:
exists = os.path.exists(os.path.join(dest, internal_dest))
else:
dest = os.path.join(dest,
os.path.splitext(os.path.basename(archive))[0])
exists = os.path.exists(dest)
if not exists:
os.makedirs(dest)
print 'extracting %s to %s' % (archive, dest)
command.execute(['unzip', '-q', archive, '-d', dest])
else:
print 'extracting %s (already extracted)' % (archive, )
elif archive.endswith('.tgz') or archive.endswith('.tar.gz'):
a = tarfile.open(archive, 'r:*')
sane, internal_dest = check_sanity(a.getnames())
if sane:
exists = os.path.exists(os.path.join(dest, internal_dest))
else:
dest = os.path.join(dest,
os.path.splitext(os.path.basename(archive))[0])
exists = os.path.exists(dest)
if not exists:
if not os.path.exists(dest): os.makedirs(dest)
print 'extracting %s to %s' % (archive, dest)
command.execute(['tar', '-xzf', archive, '-C', dest])
else:
print 'extracting %s (already extracted)' % (archive, )
else:
raise Exception('Archive format (%s) not recognised' % (archive, ))
def bot_command_action(bot_command_mo):
command.execute(
caller_nick=bot_command_mo.group(1),
channel=bot_command_mo.group(2),
command=bot_command_mo.group(4),
arguments=bot_command_mo.group(5)
)
def command_loop(self, stdscr): self.stdscr = stdscr # need except KeyboardInterrupt self.reader = Reader(**CONF.credential) self.ui = MainScreen(stdscr) self.mode = MODE.UNREAD command.execute(self, 'r') # switch unread mode while 1: command.execute(self, stdscr.getkey())
def modify_config(self):
conf_list = self.get_conf_list()
# print('conf_list is :'+conf_list)
for conf in conf_list:
dest = self.get_dets_conf_through_war(conf)
if (not len(dest) == 0):
print('\cp ' + os.path.join(self.conf_dir, conf) + ' ' +
os.path.abspath(dest))
command.execute('\cp ' + os.path.join(self.conf_dir, conf) +
' ' + dest)
print('dest is :' + dest)
def save(self, path):
import os
if not os.path.isdir(path):
raise RecordingException("Input path does not exist or is not a folder: " + path)
self.path = os.path.join(path, self.name)
for command in self._processing_list:
command.execute()
self._processing_list = []
def save(self, path):
import os
if not os.path.isdir(path):
raise RecordingException(
"Input path does not exist or is not a folder: " + path)
self.path = os.path.join(path, self.name)
for command in self._processing_list:
command.execute()
self._processing_list = []
def doc_to_vec(app, job_desc):
inputs = job_desc['inputs']
cmd = "python /home/ubuntu/ncses/doc2vec/turing_updated_pipeline.py"
walltime = int(job_desc.get("walltime", 24 * 60 * 60))
job_id = job_desc["job_id"]
retcode = 9999
try:
for i in inputs:
if i["type"] == "doc":
cmd = cmd + " -d {0}".format(i["dest"])
elif i["type"] == "params":
cmd = cmd + " -p {0}".format(i["dest"])
elif i["type"] == "model":
cmd = cmd + " -m {0}".format(i["dest"])
else:
return UNKNOWN_ARGS
logging.debug("doc_to_vec, executing {0}".format(cmd))
retcode = command.execute(app, cmd, walltime, job_id)
except Exception as e:
logging.error("Caught exception : {0}".format(e))
raise
return retcode
def script_executor(app, job_desc):
inputs = job_desc['inputs']
walltime = int(job_desc.get("walltime", 24 * 60 * 60))
job_id = job_desc["job_id"]
script_file = job_desc.get('i_script_name')
script = job_desc.get('i_script').replace('\r\n', '\n')
cmd = job_desc["executable"]
env = {
"wosuser": job_desc.get('i_wosuser', 'None'),
"wospasswd": job_desc.get('i_wospasswd', 'None')
}
with open(script_file, 'w') as ofile:
ofile.write(script)
os.chmod(script_file, 0o744)
retcode = 9999
try:
logging.debug("script_executor, executing {0}".format(cmd))
retcode = command.execute(app, cmd, walltime, job_id, env)
except Exception as e:
logging.error("Caught exception : {0}".format(e))
raise
return retcode
def execute(master_only, slave_only, addr, commands):
host, port = _parse_host_port(addr)
for r in command.execute(host, port, master_only, slave_only, commands):
if r['result'] is None:
print('%s -%s' % (r['node'].addr(), r['exception']))
else:
print('%s +%s' % (r['node'].addr(), r['result']))
def doc_to_vec (app, job_desc):
inputs = job_desc['inputs']
cmd = "python /home/ubuntu/ncses/doc2vec/turing_updated_pipeline.py"
walltime = int(job_desc.get("walltime", 24*60*60))
job_id = job_desc["job_id"]
retcode = 9999
try:
for i in inputs:
if i["type"] == "doc":
cmd = cmd + " -d {0}".format(i["dest"])
elif i["type"] == "params":
cmd = cmd + " -p {0}".format(i["dest"])
elif i["type"] == "model":
cmd = cmd + " -m {0}".format(i["dest"])
else:
return UNKNOWN_ARGS
logging.debug("doc_to_vec, executing {0}".format(cmd))
retcode = command.execute(app, cmd, walltime, job_id)
except Exception as e:
logging.error("Caught exception : {0}".format(e))
raise
return retcode
def script_executor (app, job_desc):
inputs = job_desc['inputs']
walltime = int(job_desc.get("walltime", 24*60*60))
job_id = job_desc["job_id"]
script_file = job_desc.get('i_script_name')
script = job_desc.get('i_script').replace('\r\n', '\n')
cmd = job_desc["executable"]
env = {"wosuser" : job_desc.get('i_wosuser', 'None'),
"wospasswd" : job_desc.get('i_wospasswd', 'None')}
with open(script_file, 'w') as ofile:
ofile.write(script)
os.chmod(script_file, 0o744)
retcode = 9999
try:
logging.debug("script_executor, executing {0}".format(cmd))
retcode = command.execute(app, cmd, walltime, job_id, env)
except Exception as e:
logging.error("Caught exception : {0}".format(e))
raise
return retcode
def _docker_compose(cwd, *cmd):
retcode, out, err = execute(cwd, 'docker-compose', *cmd)
if retcode == 0:
return out
logger.error('docker-compose failed: %s', cwd)
raise DockerComposeError('docker-compose exited with a non-zero exit code',
err)
def mvn_build(self):
buildResult = False
for project in self.project_list:
command.execute(COMMAMD_MAVEN_CLEAN)
if project == self.project_list[-1]:
cmd = COMMAMD_MAVEN_PACKAGE
else:
cmd = COMMAMD_MAVEN_BUILD
print('maven build ' + project + ' start')
os.chdir(os.path.join(self.workspace, project))
result = command.execute(cmd)
#for r in result:
#print(r)
print('maven build ' + project + ' end')
if not util.execute_result(MAVEN_BUILD_SUCCESS, result):
print('进入 maven失败流程')
return False
else:
buildResult = True
return buildResult
def compress_war(self):
dest_war = os.path.join(self.workspace,
self.project_name + FORMATE_WAR)
print('dest_war is :' + dest_war)
project_folder = os.path.join(FOLDER_TARGET, self.project_name)
os.chdir(os.path.abspath(project_folder))
relativeFileList = os.listdir(os.getcwd())
file_list_str = util.list_to_str(relativeFileList)
print(COMMAMD_WAR_COMPRESS + dest_war + ' ' + file_list_str)
result = command.execute(COMMAMD_WAR_COMPRESS + dest_war + ' ' +
file_list_str)
return util.execute_result(COMPRESS_SUCCESS, result)
def checkout_code(self):
result = False
print('checkout svn code start')
for project in self.project_list:
print(project)
check_result = command.execute(
COMMAMD_SVN_EXPORT +
util.url_join(self.svn_code_repository, project) +
self.svn_auth)
if not util.execute_result(EXPORT_CODE_SUCCESS, check_result):
return False
else:
result = True
return result
def fetch_and_execute():
instance = session.get(api_base + "/judge_command/unfetched/5").json()
if instance["status"] != "Success":
print("Failed")
print("Reason: %s" % instance["reason"])
return
print(instance)
result = instance["result"]
for obj in result:
command_id = get_command_id(obj)
command = get_judge_command(obj)
fetch_response = session.post(api_base + "/judge_command/%d/fetched" %
command_id).json()
if fetch_response["status"] != "Success":
print("Failed to fetch judge command #%d" % command_id)
print("Reason: %s" % fetch_response["reason"])
else:
print("Successfully fetched judge command #%d: %s" %
(command_id, command))
execute(command)
def execute(self):
for command in self.commands:
command.execute()
def _git(cwd, *cmd):
retcode, out, err = execute(cwd, '/usr/bin/git', *cmd)
if retcode == 0:
return out
raise GitError('git exited with a non-zero exit code', err)
def _git(cwd, *cmd):
retcode, out, err = execute(cwd, '/usr/bin/git', *cmd)
if retcode == 0:
return out
raise GitError('git exited with a non-zero exit code', err)
# Top-level api invocation
def osh(*ops):
"""Invoke osh interpreter. Each argument is a function invocation identifying a command.
The command sequence corresponds to the sequence of arguments.
"""
# Construct the pipeline
pipeline = None
try:
pipeline = apiparser._sequence_op(ops)
except Exception, e:
print >>sys.stderr, e
sys.exit(1)
# Run
command = core.Command(pipeline)
command.execute()
last_op = ops[-1]
if type(last_op) is _ReturnList:
return last_op.list
else:
return None
class _ReturnList(core.Op):
_unwrap_singleton = None
_list = None
def __init__(self, unwrap_singleton):
core.Op.__init__(self, '', (0, 0))
self._unwrap_singleton = unwrap_singleton
self._list = []
