def attach(self):
if self.attached:
raise Exception("Wrong usage. Ramdisk should not be attached")
if not check_is_same_device_as_root_fs(self.folder):
msg = ('Folder must be on / '
'(ROOT) and must not be a different device (possibly '
'already a RAMDISK)! Maybe try "umount {0}"?').format(self.folder)
raise argparse.ArgumentTypeError(msg)
create_ramdisk_stdout = subprocess.check_output(
['hdid','-nomount', 'ram://{0}'.format(self.size)])
ram_disk_device = create_ramdisk_stdout.strip().strip('\n')
check_is_not_normal_harddrive(ram_disk_device)
logger.info('Created RAM disk {0}'.format(ram_disk_device))
logger.info('Formatting RAM disk...')
format_stdout = subprocess.check_output(['newfs_hfs', ram_disk_device])
#Initialized /dev/rdisk13 as a 512 MB HFS Plus volume
assert format_stdout, format_stdout
old_ionode_nbr = os.stat(self.folder).st_ino
logger.info('Mounting RAM disk {0} as {1}'.format(ram_disk_device, self.folder))
subprocess.check_call(['mount','-t','hfs', ram_disk_device, self.folder])
assert old_ionode_nbr != os.stat(self.folder).st_ino
# TODO: probably remove this
assert not check_is_same_device_as_root_fs(self.folder)
self.ram_disk_device = ram_disk_device
self.attached = True
def test(suffix):
"""Test with subprocess."""
path = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', 'example{0}.py'.format(suffix)))
env = dict(PYTHONIOENCODING='utf-8')
if 'SystemRoot' in os.environ:
env['SystemRoot'] = os.environ['SystemRoot']
subprocess.check_output([sys.executable, path], env=env, stderr=subprocess.STDOUT)
def run(self):
self.prepareInputFiles()
bin_path = sample_confs.Path('')
cmds = ['perl', bin_path.ehalf_box_size_bin(), self.lig_sdf]
stdout = subprocess32.check_output(cmds)
box_size, x, y, z = stdout.split()
cmd = '''%s --receptor %s --ligand %s \
--center_x %s\
--center_y %s\
--center_z %s\
--size_x %s\
--size_y %s\
--size_z %s\
--randomize_only\
--cpu 1\
--out %s
''' % (bin_path.vina_bin(), self.prtPdbqt, self.lig_pdbqt, x, y, z,
box_size, box_size, box_size, self.output().path)
print cmd
vina_out = subprocess32.check_output(shlex.split(cmd))
ofn = self.output().path + ".rnd.txt"
with open(ofn, 'w') as ofs:
ofs.write(vina_out)
def run(self):
self.removeSdfs()
cmds = ['bash', self.trace_bin(),
self.sdf_id,
self.requires().output().path]
_ = subprocess32.check_output(cmds)
path_task = Path(self.sdf_id)
pdb_path = path_task.pdb_path()
sdfs = self.getSdfs()
with open(self.output().path, 'w') as ofs:
if len(sdfs) > 0:
count = 0
while count < len(sdfs):
sdf1 = random.choice(sdfs)
sdf2 = random.choice(sdfs)
cmds = ['cms', '-frc',
'--lig1', sdf1,
'--lig2', sdf2,
'--prt1', pdb_path,
'--prt2', pdb_path]
if sdf1 != sdf2:
try:
stdout = subprocess32.check_output(cmds)
ofs.write(stdout)
count += 1
except:
pass
else:
pass
def _delete_docker_interface(self):
"""Delete the existing veth connecting to the docker bridge."""
logger.debug('Deleting docker interface eth0')
# Get the PID of the container.
pid = str(self._get_container_pid(self.docker_id))
logger.debug('Container %s running with PID %s', self.docker_id, pid)
# Set up a link to the container's netns.
logger.debug("Linking to container's netns")
logger.debug(check_output(['mkdir', '-p', '/var/run/netns']))
netns_file = '/var/run/netns/' + pid
if not os.path.isfile(netns_file):
logger.debug(check_output(['ln', '-s', '/proc/' + pid + '/ns/net',
netns_file]))
# Log our container's interfaces before making any changes.
_log_interfaces(pid)
# Reach into the netns and delete the docker-allocated interface.
logger.debug(check_output(['ip', 'netns', 'exec', pid,
'ip', 'link', 'del', 'eth0']))
# Log our container's interfaces after making our changes.
_log_interfaces(pid)
# Clean up after ourselves (don't want to leak netns files)
logger.debug(check_output(['rm', netns_file]))
def get_from_requirements(self,
r, # type: Optional[Dict[Text, Text]]
req, # type: bool
pull_image, # type: bool
force_pull=False, # type: bool
tmp_outdir_prefix=None # type: Text
):
# type: (...) -> Optional[Text]
"""
Returns the filename of the Singularity image (e.g.
hello-world-latest.img).
"""
if r:
errmsg = None
try:
check_output(["singularity", "--version"])
except CalledProcessError as err:
errmsg = "Cannot execute 'singularity --version' {}".format(err)
except OSError as err:
errmsg = "'singularity' executable not found: {}".format(err)
if errmsg:
if req:
raise WorkflowException(errmsg)
else:
return None
if self.get_image(r, pull_image, force_pull):
return os.path.abspath(r["dockerImageId"])
if req:
raise WorkflowException(u"Container image {} not "
"found".format(r["dockerImageId"]))
return None
def run_two_test(test):
binname = ['./{0}'.format(test), 'input.txt', 'in.txt']
try:
output = subprocess.check_output(binname, stderr=FNULL, timeout=30.0)
except subprocess.CalledProcessError as e:
output = "Program Error: {0}\n score is 0.0\n".format(e)
if int(format(e).split(' ')[-1])<0:
print output
return output, get_score(output)
except subprocess.TimeoutExpired:
output = "Execution Timed out: score is 0.0\n"
return output, get_score(output)
except OSError:
output = "Compilation failed: score is 0.0\n"
return output, get_score(output)
binname = ['python', '{0}.py'.format(test), '-i', 'out_{0}.txt'.format(test), '-o', 'in.txt'.format(test)]
try:
output = subprocess.check_output(binname, timeout=30.0)
except subprocess.CalledProcessError:
output = "Program Error (Output file not in expected format): score is 0.0\n"
except subprocess.TimeoutExpired:
output = "Execution Timed out: score is 0.0\n"
except OSError:
output = "Compilation failed: score is 0.0\n"
print output
return output, get_score(output)
def __init__(self,platform_directory_string=None,root_directory_string=None,ssh_alias="",remote=False,hostname=None,shell_vars={},shell_setup_cmds=[],shell_exit_cmds=[]):
"""Constructor
Parameters
platform_directory_string - (string) location of platform specific code
root_directory_string - (string) location of F^3 on this system
ssh_alias - (string) SSH alias for this system, as stored in .ssh/sshconfig
remote - (bool) Is this system remote?
hostname - (string) - override the system hostname for generated code
shell_vars - (dict) - set any environmental variables
shell_setup_cmds (list of strings) - any commands that need to be run upon login
shell_exit_cmds (list of strings) - any commands that need to be run upon logout
Upon construction, if this is a remote system, all of the SSH set up is done. It might take several seconds to return.
"""
self.platform_directory_string = platform_directory_string
self.root_directory_string = root_directory_string
self.ssh_alias = ssh_alias
self.remote = remote
self.hostname = hostname
self.shell_vars = {}
for k in shell_vars: self.shell_vars[k] = shell_vars[k]
self.shell_setup_cmds = shell_setup_cmds
self.shell_exit_cmds = shell_exit_cmds
if(self.remote and not(self.ssh_alias)): raise ValueError("ssh alias needs to be set")
#in case the environmental variable isn't set
if not(self.root_directory_string) and not(self.remote):
try: self.root_directory_string = os.environ["F3_ROOT"]
except KeyError:
print("F3_ROOT environmental variable not set")
sys.exit(1)
#if the environmental variable isn't set, and its on a remote server
elif not(self.root_directory_string and self.remote):
try:
ssh_cmd = ["ssh","%s"%self.ssh_alias,"source",".profile;","printenv","|","grep","^F3_ROOT"]
output = subprocess.check_output(ssh_cmd)
if not(output): raise KeyError("F3_ROOT environmental variable not set on %s"%self.ssh_alias)
output= output.split("=")[1].strip("\r\n")
self.root_directory_string = output
except KeyError:
print("F3_ROOT environmental variable not set on %s"%self.ssh_alias)
sys.exit(1)
if(self.hostname==None and not(self.remote)):
self.hostname = os.uname()[1].replace(".","_") #System hostname
elif(self.hostname==None):
ssh_cmd = ["ssh","%s"%self.ssh_alias,"hostname"]
output = subprocess.check_output(ssh_cmd)
#if not(output): raise KeyError("F3_ROOT environmental variable not set on %s"%self.ssh_alias)
output = output.replace("\n","") #output.split("=")[1].strip("\r\n")
self.hostname = output.replace(".","_")
def generate(self,name_extension=".c",override=True,verbose=False,debug=False):
#os.chdir("..")
#os.chdir(self.platform.platform_directory())
if(override or not os.path.exists("%s/%s%s"%(os.path.join(self.platform.root_directory(),self.platform.platform_directory()),self.output_file_name,name_extension))):
#os.chdir(self.platform.root_directory())
#os.chdir("bin")
code_string = []
code_string.extend(self.generate_identifier())
code_string.extend(self.generate_libraries())
code_string.extend(self.generate_variable_declaration())
code_string.extend(self.generate_activity_thread())
code_string.extend(self.generate_main_thread())
#Actually writing to the file
self.generate_source(code_string,name_extension,verbose,debug)
#If remote connection
if(self.platform.remote):
copy_command = ["scp"]
copy_command += ["%s/%s%s"%(os.path.join(self.platform.root_directory(),self.platform.platform_directory()),self.output_file_name,name_extension)]
copy_command += ["%s:/home/gordon/workspace/ForwardFinancialFramework/%s"%(self.platform.ssh_alias,self.platform.platform_directory())]
subprocess.check_output(copy_command)
if(debug): print("Copied %s to %s"%(copy_command[1],copy_command[-1]))
def _update_slurm_node(self, nodename, updates):
cmd = ['scontrol', 'update', 'NodeName=' + nodename] + updates
try:
subprocess.check_output(cmd)
except:
self._logger.error(
"SLURM update %r failed", cmd, exc_info=True)
def generate(self,name_extension=".c",override=True,verbose=False,debug=False):
"""Code generation method
Parameters
name_extension - (string) file name extension to use for generated code
override - (bool) option to force code generation
verbose - (bool) option for setting verbosity level of code generation
debug - (bool) option passed to source code generation method
"""
if(override or not os.path.exists("%s/%s%s"%(os.path.join(self.platform.root_directory(),self.platform.platform_directory()),self.output_file_name,name_extension))):
code_string = []
code_string.extend(self.generate_identifier())
code_string.extend(self.generate_libraries())
code_string.extend(self.generate_variable_declaration())
code_string.extend(self.generate_activity_thread(debug))
code_string.extend(self.generate_main_thread())
#Actually writing to the file
self.generate_source(code_string,name_extension,verbose,debug)
#If remote connection
if(self.platform.remote):
copy_command = ["scp"]
copy_command += ["%s/%s%s"%(os.path.join(self.platform.root_directory(),self.platform.platform_directory()),self.output_file_name,name_extension)]
copy_command += ["%s:/home/gordon/workspace/ForwardFinancialFramework/%s"%(self.platform.ssh_alias,self.platform.platform_directory())]
subprocess.check_output(copy_command)
if(debug): print("Copied %s to %s"%(copy_command[1],copy_command[-1]))
def hashes(repo_dir):
hashes_ = dict(master='', feature='', tag_annotated='', tag_light='')
hashes_['master'] = subprocess32.check_output(['git', 'rev-parse', 'HEAD'], cwd=repo_dir).strip().decode('ascii')
subprocess32.check_call(['git', 'checkout', '-b', 'feature'], cwd=repo_dir)
with open(os.path.join(repo_dir, 'test.txt'), 'a') as f:
f.write('test')
subprocess32.check_call(['git', 'add', 'test.txt'], cwd=repo_dir)
subprocess32.check_call(['git', 'commit', '-m', 'Wrote to file.'], cwd=repo_dir)
hashes_['feature'] = subprocess32.check_output(['git', 'rev-parse', 'HEAD'], cwd=repo_dir).strip().decode('ascii')
subprocess32.check_call(['git', 'checkout', 'master'], cwd=repo_dir)
with open(os.path.join(repo_dir, 'test.txt'), 'a') as f:
f.write('test2')
subprocess32.check_call(['git', 'add', 'test.txt'], cwd=repo_dir)
subprocess32.check_call(['git', 'commit', '-m', 'Wrote to file2.'], cwd=repo_dir)
subprocess32.check_call(['git', 'tag', '-a', 'v1.0', '-m', 'First Version'], cwd=repo_dir)
hashes_['tag_annotated'] = subprocess32.check_output(['git', 'rev-parse', 'HEAD'],
cwd=repo_dir).strip().decode('ascii')
with open(os.path.join(repo_dir, 'test.txt'), 'a') as f:
f.write('test3')
subprocess32.check_call(['git', 'add', 'test.txt'], cwd=repo_dir)
subprocess32.check_call(['git', 'commit', '-m', 'Wrote to file3.'], cwd=repo_dir)
subprocess32.check_call(['git', 'tag', 'v1.0l'], cwd=repo_dir)
hashes_['tag_light'] = subprocess32.check_output(['git', 'rev-parse', 'HEAD'], cwd=repo_dir).strip().decode('ascii')
assert all(hashes_.values())
assert 4 == len(set(hashes_.values()))
return hashes_
def _init_instruments(self, bundle_id):
self._bootstrap = os.path.join(__dir__, 'bootstrap.sh')
self._bundle_id = bundle_id
self._env = {'UDID': self.udid, 'BUNDLE_ID': self._bundle_id}
# 1. remove pipe
subprocess.check_output([self._bootstrap, 'reset'], env=self._env)
# 2. start instruments
self._proc = subprocess.Popen([self._bootstrap, 'instruments'], env=self._env, stdout=subprocess.PIPE)
def set_veth_mac(veth_name_host, mac):
"""
Set the veth MAC address.
:param veth_name_host: The name of the veth.
:param mac: The MAC address.
:return: None. Raises CalledProcessError on error.
"""
# TODO MAC should be an EUI object.
check_output(["ip", "link", "set", "dev", veth_name_host, "address", mac], timeout=IP_CMD_TIMEOUT)
def test_app_force_stop(appiumSetup, moduleSetup, testSetup):
print 'in test app force stop'
subprocess32.call(['adb', '-s', config.udid, 'shell', 'am', 'force-stop', config.app_package])
app_process = subprocess32.check_output(['adb', '-s', config.udid, 'shell', 'ps', '|', 'grep', config.app_package])
assert not app_process, 'app is not forced stopped, process = %s' % app_process
subprocess32.call(['adb', '-s', config.udid, 'shell', 'monkey','-p', config.app_package, '-c', 'android.intent.category.LAUNCHER 1'])
config.appium_driver.background_app(5)
focused_app = subprocess32.check_output(['adb', '-s', config.udid, 'shell', 'dumpsys window windows', '|', 'grep', '-E', 'mCurrentFocus'])
assert config.app_package in focused_app, 'app package %s, not found in %s' % (config.app_package, focused_app)
def app_version(config):
import hashlib
import os
import subprocess
version = subprocess.check_output(["git", "-C", os.path.dirname(__file__), "describe"]).decode("utf-8").strip()
diff = subprocess.check_output(["git", "-C", os.path.dirname(__file__), "diff", "--no-ext-diff"])
if diff:
version += "-patch" + hashlib.sha1(diff).hexdigest()[:7]
config.registry.settings["snovault.app_version"] = version
def testProcessName( self ) : process = subprocess.Popen( [ "gaffer", "env", "sleep", "100" ] ) time.sleep( 1 ) command = subprocess.check_output( [ "ps", "-p", str( process.pid ), "-o", "command=" ] ).strip() name = subprocess.check_output( [ "ps", "-p", str( process.pid ), "-o", "comm=" ] ).strip() process.kill() self.assertEqual( command, "gaffer env sleep 100" ) self.assertEqual( name, "gaffer" )
def app_version(config):
import hashlib
import os
import subprocess
version = subprocess.check_output(
['git', '-C', os.path.dirname(__file__), 'describe']).decode('utf-8').strip()
diff = subprocess.check_output(
['git', '-C', os.path.dirname(__file__), 'diff', '--no-ext-diff'])
if diff:
version += '-patch' + hashlib.sha1(diff).hexdigest()[:7]
config.registry.settings['snovault.app_version'] = version
def remove_veth(veth_name_host):
"""
Remove the veth (pair).
:param veth_name_host: The name of the veth interface.
:return: True if veth was removed. False if veth does not exist.
Raises CalledProcessError on error.
"""
# The veth removal is best effort. If it fails then just log.
if not veth_exists(veth_name_host):
return False
check_output(["ip", "link", "del", veth_name_host], timeout=IP_CMD_TIMEOUT)
return True
def detect_device_build_file(device, typ, branch, build_dir):
"""Detect the current device build file"""
# Get the current build version
output = subprocess32.check_output('sudo adb shell getprop | grep "ro.build.version.incremental"', shell=True)
build_version = output.split(": [")[-1].strip().replace("]", "")
pattern = "{}.{}.{}.AutomationVersion.*B-{}*.tgz".format(device, typ, branch, build_version)
# Locate the build file in build_dir using the composed pattern
out = subprocess32.check_output('ls {}/{}'.format(build_dir, pattern), shell=True)
print "Device build name: {}".format(out.split(".tgz")[0] if out != "" else pattern)
return out
def test_other_apps_in_background(appiumSetup, moduleSetup, testSetup):
tests_app_android.get_all_packages()
for i in config.all_packages:
print i
subprocess32.call(['adb', '-s', config.udid, 'shell', 'monkey','-p', i, '-c', 'android.intent.category.LAUNCHER 1'])
time.sleep(5)
app_process = subprocess32.check_output(['adb', '-s', config.udid, 'shell', 'ps', '|', 'grep', config.app_package])
assert app_process, 'app is not running in background, process = %s' % app_process
tests_app_android.bringAppToForeground()
time.sleep(2)
focused_app = subprocess32.check_output(['adb', '-s', config.udid, 'shell', 'dumpsys window windows', '|', 'grep', '-E', 'mCurrentFocus'])
assert config.app_package in focused_app, 'app package %s, not found in %s after %s' % (config.app_package, focused_app, i)
def _check_tool(tool_name):
"""
Checks if the invoked tool produces a known command.
Returns the tool's name if the check is successful,
or a tuple with the tool's name and the error if it's not.
"""
try:
subprocess.check_output([tool_name, '-h'], timeout=TIMEOUT)
return tool_name
except (subprocess.CalledProcessError, OSError) as err:
logger.error(err)
return tool_name, err
def test_bluetooth_on_off(appiumSetup, moduleSetup, testSetup):
print 'in test bluetooth on off'
tests_app_android.bluetooth_toggle()
tests_app_android.bringAppToForeground()
time.sleep(2)
focused_app = subprocess32.check_output(['adb', '-s', config.udid, 'shell', 'dumpsys window windows', '|', 'grep', '-E', 'mCurrentFocus'])
assert config.app_package in focused_app, 'app package %s, not found in %s' % (config.app_package, focused_app)
tests_app_android.bluetooth_toggle()
tests_app_android.bringAppToForeground()
time.sleep(2)
focused_app = subprocess32.check_output(['adb', '-s', config.udid, 'shell', 'dumpsys window windows', '|', 'grep', '-E', 'mCurrentFocus'])
assert config.app_package in focused_app, 'app package %s, not found in %s' % (config.app_package, focused_app)
def set_veth_mac(veth_name_host, mac):
"""
Set the veth MAC address.
:param veth_name_host: The name of the veth.
:param mac: The MAC address.
:return: None. Raises CalledProcessError on error.
"""
#TODO MAC should be an EUI object.
check_output(['ip', 'link', 'set',
'dev', veth_name_host,
'address', mac],
timeout=IP_CMD_TIMEOUT)
def test_app_not_running_in_background(appiumSetup, moduleSetup, testSetup):
print 'in test app background'
# setup.setup_appium_driver(app=config.app)
# time.sleep(5)
# tests_app_android.get_package_name()
subprocess32.call(['adb', '-s', config.udid, 'shell', 'input', 'keyevent', 'KEYCODE_HOME'])
subprocess32.call(['adb', '-s', config.udid, 'shell', 'am', 'kill', config.app_package])
app_process = subprocess32.check_output(['adb', '-s', config.udid, 'shell', 'ps', '|', 'grep', config.app_package])
assert not app_process, 'app is running in background, process = %s' % app_process
subprocess32.call(['adb', '-s', config.udid, 'shell', 'monkey','-p', config.app_package, '-c', 'android.intent.category.LAUNCHER 1'])
time.sleep(2)
focused_app = subprocess32.check_output(['adb', '-s', config.udid, 'shell', 'dumpsys window windows', '|', 'grep', '-E', 'mCurrentFocus'])
assert config.app_package in focused_app, 'app package %s, not found in %s' % (config.app_package, focused_app)
def spawn():
tarball_file = flask.request.files["tarball"]
command = flask.request.form["command"]
title = flask.request.form.get("title", "Unnamed test")
emailees = flask.request.form.getlist("emailee")
for emailee in emailees:
assert "@" in emailee
with database_lock:
test_id = database.get("next_test_id", 0)
database["next_test_id"] = test_id + 1
database["%d.metadata" % test_id] = {
"title": title,
"start_time": time.time(),
"command": command,
"emailees": emailees
}
database.sync()
try:
running_tasks[test_id] = { }
try:
with database_lock:
database["%d.result" % test_id] = { "status": "starting" }
database.sync()
test_dir = os.path.join(root_dir, "test-%d" % test_id)
os.mkdir(test_dir)
box_dir = os.path.join(test_dir, "box")
os.mkdir(box_dir)
try:
tar_cmd = ["tar", "--extract", "-z", "-C", box_dir]
if hasattr(tarball_file, "fileno"):
subprocess32.check_output(tar_cmd + ["--file=-"], stdin = tarball_file, stderr = subprocess32.STDOUT)
else:
tarball_path = os.path.join(test_dir, "tarball-temp.tar")
tarball_file.save(tarball_path)
subprocess32.check_output(tar_cmd + ["--file", tarball_path], stderr = subprocess32.STDOUT)
os.remove(tarball_path)
except subprocess32.CalledProcessError, e:
raise ValueError("Bad tarball: " + e.output)
if not os.access(os.path.join(box_dir, "renderer"), os.X_OK):
raise ValueError("renderer is missing or not executable")
for emailee in emailees:
send_gmail.send_gmail(
subject = title,
body = flask.url_for("view", test_id = test_id, _external = True),
sender = (emailer, emailer_password),
receiver = emailee)
except Exception, e:
del running_tasks[test_id]
raise
def DHCP_test(trigger_line):
print color.INFO("<Test.py>"),"Got IP"
ip_string = vm.readline()
print color.INFO("<Test.py>"), "Assigned address: ", ip_string
print color.INFO("<Test.py>"), "Trying to ping"
time.sleep(1)
try:
command = ["ping", ip_string.rstrip(), "-c", str(ping_count), "-i", "0.2"]
print color.DATA(" ".join(command))
print subprocess32.check_output(command, timeout=thread_timeout)
vm.exit(0,"<Test.py> Ping test passed. Process returned 0 exit status")
except Exception as e:
print color.FAIL("<Test.py> Ping FAILED Process threw exception:")
print e
return False
def _check_modules():
"""
Check system kernel modules
:return: True if all the modules in REQUIRED_MODULES are available,
False if one is unloaded or other failure.
"""
all_available = True
try:
# Grab Kernel version with `uname`
kernel_version = check_output(["uname", "-r"]).rstrip()
modules_loadable_path = "/lib/modules/%s/modules.dep" % kernel_version
modules_builtin_path = "/lib/modules/%s/modules.builtin" % kernel_version
# For the modules we're expecting to look for, the mainline case is that
# they will be loadable modules. Therefore, loadable modules are checked
# first and builtins are checked only if needed.
available_lines = open(modules_loadable_path).readlines()
builtin_lines = None
for module in REQUIRED_MODULES:
module_available = check_module_lines(available_lines, module)
if not module_available:
# Open and check builtin modules
if not builtin_lines:
builtin_lines = open(modules_builtin_path).readlines()
module_builtin = check_module_lines(builtin_lines, module)
# If module is not available or builtin, issue warning
if not module_builtin:
print >> sys.stderr, "WARNING: Unable to detect the %s " \
"module as available or builtin." % module
all_available = False
# If something goes wrong with uname or file access, try lsmod.
except BaseException:
try:
modules = check_output(["lsmod"])
for module in REQUIRED_MODULES:
if module not in modules:
print >> sys.stderr, "WARNING: Unable to detect the %s " \
"module with lsmod." % module
all_available = False
except BaseException as e:
print >> sys.stderr, "ERROR: Could not check for loaded modules \n%s" % e
return False
return all_available
def RunMetrics(self, options):
Log.Info("Perform KPCA.", self.verbose)
# Split the command using shell-like syntax.
cmd = shlex.split(self.path + "mlpack_kernel_pca -i " + self.dataset +
" -v -o output.csv " + self.OptionsToStr(options))
# Run command with the nessecary arguments and return its output as a byte
# string. We have untrusted input so we disable all shell based features.
try:
s = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=False,
timeout=self.timeout)
except subprocess.TimeoutExpired as e:
Log.Warn(str(e))
return -2
except Exception as e:
Log.Fatal("Could not execute command: " + str(cmd))
return -1
# Datastructure to store the results.
metrics = {}
# Parse data: runtime.
timer = self.parseTimer(s)
if timer != -1:
metrics['Runtime'] = timer.total_time - timer.loading_time - timer.saving_time
Log.Info(("total time: %fs" % (metrics['Runtime'])), self.verbose)
return metrics
def __init__(self, dataset, timeout=0, path=os.environ["BINPATH"],
verbose=True, debug=os.environ["DEBUGBINPATH"]):
self.verbose = verbose
self.dataset = dataset
self.path = path
self.timeout = timeout
self.debug = debug
# Get description from executable.
cmd = shlex.split(self.path + "mlpack_kernel_pca -h")
try:
s = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=False)
except Exception as e:
Log.Fatal("Could not execute command: " + str(cmd))
else:
# Use regular expression pattern to get the description.
pattern = re.compile(br"""(.*?)Optional.*?options:""",
re.VERBOSE|re.MULTILINE|re.DOTALL)
match = pattern.match(s)
if not match:
Log.Warn("Can't parse description", self.verbose)
description = ""
else:
description = match.group(1)
self.description = description
def checkState(self):
"""calls the script and publishes the result"""
self.logger.info('Executing script with the following arguments: {0}'.format(self.cmdArgs))
try:
self.results = subprocess.check_output(self.cmdArgs, shell=False, universal_newlines=True)
self.logger.info('Command results to be published to {0}\n{1}'.format(self.dest, self.results))
except subprocess.CalledProcessError as e:
self.logger.warn('Command returned an error code: {0}\n{1}'.format(e.returncode, e.output))
self.results = 'Error'
self.publishState()
def sys_boot_wait(self):
logger.debug('Sys boot wait')
cmd = [
"adb", "-s", "emulator-{}".format(self.port), "shell", "getprop",
"sys.boot_completed"
]
pattern = re.compile(r"1")
while True:
output = subprocess.check_output(cmd, universal_newlines=True)
if pattern.match(output):
break
time.sleep(0.1)
def callExt(cmd):
retcode = 0
cmdOutput = ''
try:
cmdOutput = check_output(cmd, stderr=STDOUT, shell=True)
except OSError as e:
raise ShellError(e)
except CalledProcessError as e:
# Exception raised when a process returns a non-zero exit status
retcode = e.returncode
cmdOutput = e.output
return retcode, cmdOutput
def main():
options, args = cmdparameter(sys.argv)
#-----------------------------------
file = options.filein
lineno = options.line
number = options.number
op = options.op
if not op:
op = file
op = op.rstrip('.') + '.'
verbose = options.verbose
global debug
debug = options.debug
#-----------------------------------
if number:
count = ['grep', '-c', '">"', file]
#print >>sys.stderr, count
totalsequence = int(check_output(' '.join(count), shell=True))
#print >>sys.stderr, totalsequence
lineno = int(totalsequence / number) + 1
#print >>sys.stderr, lineno
if file == '-':
fh = sys.stdin
else:
fh = open(file)
#--------------------------------
count_file = 0
count_line = 1
fh_split = ''
for line in fh:
if line[0] == '>':
#print >>sys.stderr, "count_line", count_line
#print >>sys.stderr, "lineno", lineno
#print >>sys.stderr, "count_line % lineno", count_line % lineno
if count_line % lineno == 1:
#print >>sys.stderr, "count_line", count_line
count_file += 1
#print >>sys.stderr, 'here'
if fh_split:
fh_split.close()
fh_split = open(op + str(count_file), 'w')
#print >>sys.stderr, fh_split
count_line += 1
#print >>sys.stderr, fh_split
print >> fh_split, line.strip()
#-------------------------------
else:
print >> fh_split, line.strip()
fh_split.close()
#-------------END reading file----------
#----close file handle for files-----
if file != '-':
fh.close()
def _check_tool(tool_name):
"""
Checks if the invoked tool produces a known command.
Returns the tool's name if the check is successful,
or a tuple with the tool's name and the error if it's not.
"""
if platform.system() == 'Windows':
tool_name += '.exe'
# XXX start hack to bypass these issues
# https://github.com/adobe-type-tools/afdko/issues/347
# https://github.com/adobe-type-tools/afdko/issues/348
if tool_name.split('.')[0] in ('sfntdiff', 'sfntedit', 'makeotfexe',
'type1', 'detype1'):
return tool_name
# XXX end hack
try:
subprocess.check_output([tool_name, '-h'], timeout=60)
return tool_name
except (subprocess.CalledProcessError, OSError) as err:
logger.error(err)
return tool_name, err
def kubectl_create(jobfile, EXEC=True):
if EXEC:
try:
output = subprocess32.check_output([
"bash", "-c", config["kubelet-path"] + " create -f " + jobfile
])
except Exception as e:
print e
output = ""
else:
output = "Job " + jobfile + " is not submitted to kubernetes cluster"
return output
def kubectl_exec(params, timeout=None):
"""As defalut, never timeout."""
try:
#print ("bash -c %s %s" % (config["kubelet-path"], params))
# TODO set the timeout
output = subprocess32.check_output(
["bash", "-c", config["kubelet-path"] + " " + params],
timeout=timeout)
except Exception as e:
print "EXCEPTION: " + str(e)
output = ""
return output
def split_path_msys(path):
if path.startswith(('/', '\\')) and not path.startswith(('//', '\\\\')):
# msys paths can be in the form /usr/bin
global msysroot
if not msysroot:
# msys has python 2.7 or 3, so we can use this
msysroot = subprocess.check_output(['cygpath', '-w', '/'
]).decode(sys.stdout.encoding
or 'latin-1')
msysroot = msysroot.strip()
path = os.path.normpath(msysroot + os.sep + path)
return split_path_win32(path)
def get_num_processors():
"""
Return number of online processor cores.
"""
# try different strategies and use first one that succeeeds
try:
return os.cpu_count() # Py3 only
except AttributeError:
pass
try:
import multiprocessing
return multiprocessing.cpu_count()
except ImportError: # no multiprocessing?
pass
except NotImplementedError:
# multiprocessing cannot determine CPU count
pass
try:
from subprocess32 import check_output
ncpus = check_output('nproc')
return int(ncpus)
except CalledProcessError: # no `/usr/bin/nproc`
pass
except (ValueError, TypeError):
# unexpected output from `nproc`
pass
except ImportError: # no subprocess32?
pass
try:
from subprocess import check_output
ncpus = check_output('nproc')
return int(ncpus)
except CalledProcessError: # no `/usr/bin/nproc`
pass
except (ValueError, TypeError):
# unexpected output from `nproc`
pass
except ImportError: # no subprocess.check_call (Py 2.6)
pass
raise RuntimeError("Cannot determine number of processors")
def fashion_mnist_download(train=True, test=True, labels=True):
"""Download Fashion MNIST.
The original resource of Fashion MNIST is here:
https://github.com/zalandoresearch/fashion-mnist
The function uses wget to download
# Parameters
train : bool
if download training dataset
test : bool
if download testing dataset
labels : bool
if download labels
# Returns
downloaded datasets in ~/.pnslibres
"""
# define datasets URL
base_url = "http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/"
train_x_name = "train-images-idx3-ubyte.gz"
train_y_name = "train-labels-idx1-ubyte.gz"
test_x_name = "t10k-images-idx3-ubyte.gz"
test_y_name = "t10k-labels-idx1-ubyte.gz"
# construct save path
save_path = os.path.join(pnslib.PNSLIB_DATA, "fashion-mnist")
if not os.path.isdir(save_path):
os.makedirs(save_path)
if train is True:
train_x_path = os.path.join(save_path, train_x_name)
sp.check_output(["wget", base_url + train_x_name, "-O", train_x_path],
stderr=sp.PIPE).decode("UTF-8")
if labels is True:
train_y_path = os.path.join(save_path, train_y_name)
sp.check_output(
["wget", base_url + train_y_name, "-O", train_y_path],
stderr=sp.PIPE).decode("UTF-8")
if test is True:
test_x_path = os.path.join(save_path, test_x_name)
sp.check_output(["wget", base_url + test_x_name, "-O", test_x_path],
stderr=sp.PIPE).decode("UTF-8")
if labels is True:
test_y_path = os.path.join(save_path, test_y_name)
sp.check_output(
["wget", base_url + test_y_name, "-O", test_y_path],
stderr=sp.PIPE).decode("UTF-8")
def exec_query(root, docker_image, env, timeout=None, commit_name=None):
""" Create a container from the $docker_image,
append the $env list (of `-e` option) to the docker command.
Waits for the container to exit.
- timeout: enforce maximum running time.
- commit_name: if specified and the container returns successfully (0),
the container will be committed as $commit_name
Globals:
DEBUG, PRINT_ONLY, FORCE_LOAD, CMD_EXT, SUBPROCESS_LOGF, logger
Throws:
subprocess.TimeoutExpired: the timeout fired
subprocess.CalledProcessError: there was an error in the container
"""
# Check if what we are going to commit (if any) already exists
if commit_name:
command = ["docker", "images", "-q", commit_name]
if len(subprocess.check_output(command).strip('\n ')):
if not FORCE_LOAD:
logger.info("Loading (or committing): use existing {}"
.format(commit_name))
return
logger.info("Loading (or committing): overriding {}"
.format(commit_name))
# Build command
command = ["docker", "run", "-v", root + ':' + RUNTIME_DIR] + CMD_EXT + env
container_name = 'CNT_' + docker_image.replace('/', '-') + hashlib.sha256(
''.join(command + [docker_image]).encode()).hexdigest()
command += ['--name', container_name, docker_image]
try:
# Execute the query
logger.debug('Command: ' + ' '.join(command))
if PRINT_ONLY:
print(' '.join(command))
return
subprocess.check_call(command, stdout=SUBPROCESS_LOGF,
stderr=SUBPROCESS_LOGF, timeout=timeout)
# Commit if we should
if commit_name:
logger.info('Committing {} as {}'
.format(container_name, commit_name))
command = ["docker", "commit", container_name, commit_name]
subprocess.check_call(command, stdout=SUBPROCESS_LOGF,
stderr=SUBPROCESS_LOGF)
finally:
rmf(container_name)
def tag_version(self):
"""Get the current version number from ``git describe``, fall back to setup.json."""
try:
describe_byte_string = subprocess.check_output(
['git', 'describe', '--tags', '--match', 'v*.*.*'])
version_string = re.findall(self.version_pat,
describe_byte_string)[0]
except subprocess.CalledProcessError:
with open(self.setup_json, 'r') as setup_fo:
setup = json.load(setup_fo)
version_string = setup['version']
return version.parse(version_string)
def runCommand(command):
"""Execute shell command, return output, catch exceptions."""
try:
result = subprocess.check_output(shlex.split(command),
stderr=subprocess.STDOUT,
universal_newlines=True)
if 'NOTICE:' in result:
LOG.warn('NOTICE in output for: %s', command)
return ''
return result
except (OSError, subprocess.CalledProcessError) as e:
LOG.error('Error when runnning command %s: %r', command, e.output)
return ''
def __init__( self ):
super( Pane, self ).__init__( )
# One tmux channel for success, one for failures. See tmux(1).
self.channel_ids = tuple( uuid.uuid4( ) for _ in range( 2 ) )
# A queue between the daemon threads that service the channels and the client code. The
# queue items are the channel index, 0 for failure, 1 or success.
self.queue = Queue( maxsize=1 )
# The pane index.
self.index = len( self.panes )
window = '%s:0' % self.session
if self.index == 0:
self.log( "Run 'tmux attach -t %s' to monitor output" % self.session )
check_call(
[ 'tmux', 'new-session', '-d', '-s', self.session, '-x', '100', '-y', '80' ] )
self.tmux_id = check_output(
[ 'tmux', 'list-panes', '-t', window, '-F', '#{pane_id}' ] ).strip( )
else:
self.tmux_id = check_output(
[ 'tmux', 'split-window', '-v', '-t', window, '-PF', '#{pane_id}' ] ).strip( )
check_call( [ 'tmux', 'select-layout', '-t', window, 'even-vertical' ] )
self.panes.append( self )
self.threads = tuple( self._start_thread( i ) for i in range( 2 ) )
def _is_zombie(pid):
"""Check if pid is in zombie stat."""
if PS_PATH is None:
return False
try:
proc = subprocess.check_output([PS_PATH, 'axo', 'pid=,stat='])
except subprocess.CalledProcessError:
return False
for line in proc.strip().splitlines():
_pid, stat = line.strip().decode().split()
if int(_pid) == pid:
return 'Z' in stat
raise OSError(3, 'No such process')
def send_task_cli(self, path, params, queue=None, **kwargs):
if not self.started and queue:
self.start()
cli = ["python", "mrq/bin/mrq_run.py", "--quiet"]
if queue:
cli += ["--queue", queue]
cli += [path, json.dumps(params)]
out = subprocess.check_output(cli).strip()
if not queue:
return json.loads(out)
return out
def get_packed_cwl(workflow_url):
"""
Create 'packed' version of CWL workflow descriptor.
Args:
workflow_url (str): URL for main workflow descriptor file
Returns:
str: string with main and all secondary workflow descriptors
combined CWL workflow
"""
logger.debug("Packing descriptors for '{}'".format(workflow_url))
return subprocess32.check_output(['cwltool', '--pack', workflow_url])
def get_from_requirements(self,
r, # type: Optional[Dict[Text, Text]]
req, # type: bool
pull_image, # type: bool
force_pull=False, # type: bool
tmp_outdir_prefix=None # type: Text
):
# type: (...) -> Optional[Text]
"""
Returns the filename of the Singularity image (e.g.
hello-world-latest.img).
"""
if r:
errmsg = None
try:
commands = ["singularity", "--version"]
frameinfo = getframeinfo(currentframe())
viki_logger(frameinfo.filename, frameinfo.lineno, commands)
check_output(["singularity", "--version"])
except CalledProcessError as err:
errmsg = "Cannot execute 'singularity --version' {}".format(err)
except OSError as err:
errmsg = "'singularity' executable not found: {}".format(err)
if errmsg:
if req:
raise WorkflowException(errmsg)
else:
return None
if self.get_image(r, pull_image, force_pull):
return os.path.abspath(r["dockerImageId"])
else:
if req:
raise WorkflowException(u"Container image {} not "
"found".format(r["dockerImageId"]))
return None
def subproc_call(cmd, timeout=None):
try:
output = subprocess.check_output(cmd,
stderr=subprocess.STDOUT,
shell=True,
timeout=timeout)
return output
except subprocess.TimeoutExpired as e:
logger.warn("Command timeout!")
logger.warn(e.output)
except subprocess.CalledProcessError as e:
logger.warn("Commnad failed: {}".format(e.returncode))
logger.warn(e.output)
def _autodetect_zip_command():
try:
zip_output = subprocess.check_output(["zip", "-v"])
zip_cmd = "zip -q -r $ARCHIVE $DIR"
except subprocess.CalledProcessError as e:
raise ArchiveError("Archive handling requires 'zip' in path: %s" % e)
if zip_output.find("ZIP64_SUPPORT") < 0:
log.warn(
"installed 'zip' doesn't have Zip64 support so will fail for large archives"
)
log.debug("zip command: %s", zip_cmd)
return zip_cmd
def devices(self):
'''get a dict of attached devices. key is the device serial, value is device name.'''
out = subprocess.check_output([self.adb_path(),
'devices']).decode("utf-8")
match = "List of devices attached"
index = out.find(match)
if index < 0:
raise EnvironmentError("adb is not working.")
return dict([
s.split("\t")
for s in out[index + len(match):].strip().splitlines()
if s.strip() and not s.strip().startswith('*')
])
def move_veth_into_ns(namespace, veth_name_ns_temp, veth_name_ns):
"""
Move the veth into the namespace.
:param namespace: The Namespace to move the veth into.
:type namespace Namespace
:param veth_name_ns_temp: The temporary interface name of the veth that will be
moved into the namespace.
:param veth_name_ns: The name of the interface in the namespace.
:return: None. Raises CalledProcessError on error.
"""
with NamedNamespace(namespace) as ns:
_log.debug("Moving temp interface %s into ns %s.", veth_name_ns_temp,
ns.name)
# Create the veth pair and move one end into container:
check_output(
["ip", "link", "set", veth_name_ns_temp, "netns", ns.name],
timeout=IP_CMD_TIMEOUT)
ns.check_output([
"ip", "link", "set", "dev", veth_name_ns_temp, "name", veth_name_ns
])
ns.check_output(["ip", "link", "set", veth_name_ns, "up"])
def _fetch_child(ppid):
"""Get child processes of a leading process."""
chld_pid = [ppid]
if PGREP_PATH is None:
return chld_pid
try:
proc = subprocess.check_output([PGREP_PATH, '-P', str(ppid)])
except subprocess.CalledProcessError:
return chld_pid
for line in proc.strip().splitlines():
with contextlib.suppress(ValueError):
chld_pid.extend(_fetch_child(int(line.strip())))
return chld_pid
def getHexFp(smi_fn):
"""get the finger print in hex
Keyword Arguments:
smi_fn -- input file for the molecule in smi format
"""
cmd = "babel -ismi {} -ofpt - -xN1024".format(smi_fn)
cmds = shlex.split(cmd)
stdout = subprocess32.check_output(cmds)
hex_strs = []
for line in stdout.splitlines():
if not line.startswith('>'):
hex_strs.extend(line.split())
return hex_strs
def tag_git_version(packed):
if tool.tool["id"].startswith("file://"):
path = os.path.dirname(tool.tool["id"][7:])
try:
githash = subprocess.check_output([
'git', 'log', '--first-parent', '--max-count=1', '--format=%H'
],
stderr=subprocess.STDOUT,
cwd=path).strip()
except (OSError, subprocess.CalledProcessError):
pass
else:
packed["http://schema.org/version"] = githash
def do_run(cmd):
try:
cwd = os.getcwd() if inherit_cwd else None
if not async:
if stdin:
return subprocess.check_output(cmd, shell=True,
stderr=subprocess.STDOUT, stdin=subprocess.PIPE, env=env_dict, cwd=cwd)
return subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, env=env_dict, cwd=cwd)
# subprocess.Popen is not thread-safe, hence use a mutex here..
try:
mutex_popen.acquire()
stdin_arg = subprocess.PIPE if stdin else None
stdout_arg = open(outfile, 'wb') if isinstance(outfile, basestring) else outfile
process = subprocess.Popen(cmd, shell=True, stdin=stdin_arg,
stderr=subprocess.STDOUT, stdout=stdout_arg, env=env_dict, cwd=cwd)
return process
finally:
mutex_popen.release()
except subprocess.CalledProcessError, e:
if print_error:
print("ERROR: '%s': %s" % (cmd, e.output))
raise e
def ffprobe(fn, fmt=None):
cmd = [
"ffprobe", "-v", "quiet", "-print_format", "json", "-show_format",
"-show_streams"
]
cmd += [fn]
try:
ffprobe_output = subprocess.check_output(cmd)
except subprocess.CalledProcessError as e:
raise DataUnreadableError(fn)
return json.loads(ffprobe_output)
def run_decoder2_test(test):
binname = ['./{0}_segfault'.format(test)]
try:
output = subprocess.check_output(binname, timeout=30.0)
output = "Segfault check on decodeChar(NULL) passed: score is 1.0\n"
except subprocess.CalledProcessError:
output = "Segfault on decodeChar(NULL): score is 0.0\n"
except subprocess.TimeoutExpired:
output = "Execution Timed out: score is 0.0\n"
except OSError:
output = "Compilation failed: score is 0.0\n"
print output
return output, get_score(output)
def pic_to_video(self, arg, pic_file, output_file):
pic_to_video_arg = arg % (pic_file, 3, output_file)
pic_to_video_cmd = "%s %s" % (ffmpeg_path, pic_to_video_arg)
logger.info('pic_to_video_cmd:%s' % pic_to_video_cmd)
#status,output = commands.getstatusoutput(pic_to_video_cmd)
cmd_args = shlex.split(pic_to_video_cmd)
try:
output = check_output(cmd_args, stderr=STDOUT, timeout=TIMEOUT)
except:
logger.error(traceback.format_exc())
return False
return True
def physicalMemory():
"""
>>> n = physicalMemory()
>>> n > 0
True
>>> n == physicalMemory()
True
"""
try:
return os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
except ValueError:
return int(
subprocess.check_output(['sysctl', '-n', 'hw.memsize']).strip())
