def destroy(self):
try:
os.unlink(self.filename)
except OSError as error:
if error.errno != errno.ENOENT: # That's what we wanted anyway.
raise
self.close()
def test_check_links_fail(self):
with cd(self.target_dir):
os.unlink(os.path.join("output", "archive.html"))
try:
__main__.main(['check', '-l'])
except SystemExit as e:
self.assertNotEqual(e.code, 0)
def slicethread(fname, oname, wname, cfg, jobid):
retcode = "fail"
try:
con = sqlite3.connect('db.sqlite')
con.row_factory = sqlite3.Row
cfg = "config.ini" if cfg is None else cfg
proc = subprocess.Popen(["slic3r",
"--load", cfg,
fname, "-o", wname+'.gcode'])
con.execute('insert into journal(cmd, pid, action, status, timestamp) values(?,?,?,?,DateTime(\'now\'))',
('slice {} -c {}'.format(os.path.basename(fname),
os.path.basename(cfg)), proc.pid, 'start',
0 if proc.returncode == None else 1 ))
con.commit()
retcode = proc.wait()
con.execute('insert into journal(cmd, pid, action, status, timestamp) values(?,?,?,?,DateTime(\'now\'))',
('slice {} -c {}'.format(os.path.basename(fname),
os.path.basename(cfg)), proc.pid, 'stop',
proc.returncode))
con.commit()
try:
os.unlink(oname+'.gcode')
except OSError as e:
pass
finally:
try:
os.rename(wname+'.gcode', oname+'.gcode')
except Exception:
logging.info( wname+'.gcode')
logging.info( oname+'.gcode')
pass
finally:
_work_done(jobid, val=retcode)
def testNoDanglingFileDescriptorAfterCloseVariant2(self):
''' Test that when closing the provider all file handles are released '''
datasource = os.path.join(self.basetestpath, 'testNoDanglingFileDescriptorAfterCloseVariant2.csv')
with open(datasource, 'wt') as f:
f.write('id,WKT\n')
f.write('1,\n')
f.write('2,POINT(2 49)\n')
vl = QgsVectorLayer('{}|layerid=0'.format(datasource), 'test', 'ogr')
self.assertTrue(vl.isValid())
# Consume all features.
myiter = vl.getFeatures()
for feature in myiter:
pass
# The iterator is closed, but the corresponding connection still not closed
if sys.platform.startswith('linux'):
self.assertEqual(count_opened_filedescriptors(datasource), 2)
# Should release one file descriptor
del vl
# Non portable, but Windows testing is done with trying to unlink
if sys.platform.startswith('linux'):
self.assertEqual(count_opened_filedescriptors(datasource), 0)
# Check that deletion works well (can only fail on Windows)
os.unlink(datasource)
self.assertFalse(os.path.exists(datasource))
def _decr_iscsiSR_refcount(targetIQN, uuid):
filename = os.path.join(ISCSI_REFDIR, targetIQN)
if not os.path.exists(filename):
return 0
try:
f = open(filename, 'a+')
except:
raise xs_errors.XenError('LVMRefCount', \
opterr='file %s' % filename)
output = []
refcount = 0
for line in filter(match_uuid, f.readlines()):
if line.find(uuid) == -1:
output.append(line[:-1])
refcount += 1
if not refcount:
os.unlink(filename)
return refcount
# Re-open file and truncate
f.close()
f = open(filename, 'w')
for i in range(0,refcount):
f.write("%s\n" % output[i])
f.close()
return refcount
def test_create_volume_from_image_exception(self):
"""Verify that create volume from image, the volume status is
'downloading'."""
dst_fd, dst_path = tempfile.mkstemp()
os.close(dst_fd)
self.stubs.Set(self.volume.driver, 'local_path', lambda x: dst_path)
image_id = 'aaaaaaaa-0000-0000-0000-000000000000'
# creating volume testdata
volume_id = 1
db.volume_create(self.context,
{'id': volume_id,
'updated_at': datetime.datetime(1, 1, 1, 1, 1, 1),
'display_description': 'Test Desc',
'size': 20,
'status': 'creating',
'host': 'dummy'})
self.assertRaises(exception.ImageNotFound,
self.volume.create_volume,
self.context,
volume_id, None, None, None,
None,
image_id)
volume = db.volume_get(self.context, volume_id)
self.assertEqual(volume['status'], "error")
# cleanup
db.volume_destroy(self.context, volume_id)
os.unlink(dst_path)
def get(url, path, verbose=False):
sha_url = url + ".sha256"
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
temp_path = temp_file.name
with tempfile.NamedTemporaryFile(suffix=".sha256", delete=False) as sha_file:
sha_path = sha_file.name
try:
download(sha_path, sha_url, False, verbose)
if os.path.exists(path):
if verify(path, sha_path, False):
if verbose:
print("using already-download file " + path)
return
else:
if verbose:
print("ignoring already-download file " + path + " due to failed verification")
os.unlink(path)
download(temp_path, url, True, verbose)
if not verify(temp_path, sha_path, verbose):
raise RuntimeError("failed verification")
if verbose:
print("moving {} to {}".format(temp_path, path))
shutil.move(temp_path, path)
finally:
delete_if_present(sha_path, verbose)
delete_if_present(temp_path, verbose)
def POST(self):
web.header("Content-Type", "text/html")
i = web.input(myfile = {})
(parent, child) = (safepath(i["path"]), i["myfile"].filename)
if not auth.can_write(parent):
return json.dumps({"success": False, "msg": "Unauthorized"})
srcp = common.Mapper.d2s(parent)
if not srcp.startswith(config["juno-home"] + "/"):
return json.dumps({"success": False, "msg": "Unauthorized"})
if not os.path.isdir(srcp) or not child or child[0] == "." \
or "/" in child:
return json.dumps({"success": False, "msg": "Unauthorized"})
srcp = os.path.join(srcp, child)
if os.path.exists(srcp):
return json.dumps({"success": False, "msg": "Unauthorized"})
f = open(srcp, "w")
f.write(i["myfile"].value)
f.close()
if not odptools.odf.Odp.is_odp(srcp):
os.unlink(srcp)
return json.dumps({"success": False, "msg": "Unauthorized"})
if subprocess.call(("./index.py", "-p", srcp)):
os.unlink(srcp)
return json.dumps({"success": False, "msg": "Unauthorized"})
return json.dumps({"success": True})
def splitFasta (seqFile, segments):
#Split sequence file into
# as many fragments as appropriate
# depending on the size of original_fasta
#
# Clean up any segment files from previous runs before creating new one
#
for i in segments:
os.unlink(i)
current_file_index = 0
#current_size = 0
for line in open(original_fasta):
#
# start a new file for each accession line
#
if line[0] == '>':
#current_size +=1
current_file_index += 1
#if (current_size >= pack_size)
file_name = "%d.segment" % current_file_index
file_path = os.path.join(temp_directory, file_name)
current_file = open(file_path, "w")
#current_size = 0
if current_file_index:
current_file.write(line)
end_file = open(os.path.join(temp_directory, "end_tmp"), 'w')
end_file.write(str(current_file_index))
end_file.close()
os.rename(os.path.join(temp_directory, "end_tmp"), os.path.join(temp_directory, "end"))
def _testStreamRoundtrip(self):
inD = open(self.fName).read()
supp = Chem.SDMolSupplier(self.fName)
outName = tempfile.mktemp('.sdf')
writer = Chem.SDWriter(outName)
m1 = next(supp)
for m in supp:
writer.write(m)
writer.flush()
writer = None
outD = open(outName,'r').read()
try:
os.unlink(outName)
except Exception:
import time
time.sleep(1)
try:
os.unlink(outName)
except Exception:
pass
assert inD.count('$$$$')==outD.count('$$$$'),'bad nMols in output'
io = StringIO(outD)
supp = Chem.SDMolSupplier(stream=io)
outD2 = supp.Dump()
assert outD2.count('$$$$')==len(supp),'bad nMols in output'
assert outD2.count('$$$$')==outD.count('$$$$'),'bad nMols in output'
assert outD2==outD,'bad outd'
def print_xcf(self, filename_or_obj, *args, **kwargs):
"Writes the figure to a GIMP XCF image file"
# If filename_or_obj is a file-like object we need a temporary file for
# GIMP's output too...
if is_string(filename_or_obj):
out_temp_handle, out_temp_name = None, filename_or_obj
else:
out_temp_handle, out_temp_name = tempfile.mkstemp(suffix='.xcf')
try:
# Create a temporary file and write the "layer" to it as a PNG
in_temp_handle, in_temp_name = tempfile.mkstemp(suffix='.png')
try:
FigureCanvasAgg.print_png(self, in_temp_name, *args, **kwargs)
run_gimp_script(
SINGLE_LAYER_SCRIPT.format(
input=quote_string(in_temp_name),
output=quote_string(out_temp_name)))
finally:
os.close(in_temp_handle)
os.unlink(in_temp_name)
finally:
if out_temp_handle:
os.close(out_temp_handle)
# If we wrote the XCF to a temporary file, write its content to
# the file-like object we were given (the copy is chunked as
# XCF files can get pretty big)
with open(out_temp_name, 'rb') as source:
for chunk in iter(lambda: source.read(131072), ''):
filename_or_obj.write(chunk)
os.unlink(out_temp_name)
def __init__(self, flavor_id=""):
import sys
self.initialized = False
basename = os.path.splitext(os.path.abspath(sys.argv[0]))[0].replace("/", "-").replace(":", "").replace("\\", "-") + '-%s' % flavor_id + '.lock'
# os.path.splitext(os.path.abspath(sys.modules['__main__'].__file__))[0].replace("/", "-").replace(":", "").replace("\\", "-") + '-%s' % flavor_id + '.lock'
self.lockfile = os.path.normpath(tempfile.gettempdir() + '/' + basename)
logger.debug("SingleInstance lockfile: " + self.lockfile)
if sys.platform == 'win32':
try:
# file already exists, we try to remove (in case previous execution was interrupted)
if os.path.exists(self.lockfile):
os.unlink(self.lockfile)
self.fd = os.open(self.lockfile, os.O_CREAT | os.O_EXCL | os.O_RDWR)
except OSError:
type, e, tb = sys.exc_info()
if e.errno == 13:
logger.error("Another instance is already running, quitting.")
sys.exit(-1)
print(e.errno)
raise
else: # non Windows
import fcntl
self.fp = open(self.lockfile, 'w')
try:
fcntl.lockf(self.fp, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError:
logger.warning("Another instance is already running, quitting.")
sys.exit(-1)
self.initialized = True
def test_Writer(self):
" tests writes using a file name "
with open(self.fName,'r') as inf:
inD = inf.read()
supp = Chem.SDMolSupplier(self.fName)
outName = tempfile.mktemp('.sdf')
writer = Chem.SDWriter(outName)
m1 = next(supp)
writer.SetProps(m1.GetPropNames())
for m in supp:
writer.write(m)
writer.flush()
# The writer does not have an explicit "close()" so need to
# let the garbage collector kick in to close the file.
writer = None
with open(outName,'r') as inf:
outD = inf.read()
# The file should be closed, but if it isn't, and this
# is Windows, then the unlink() can fail. Wait and try again.
try:
os.unlink(outName)
except Exception:
import time
time.sleep(1)
try:
os.unlink(outName)
except Exception:
pass
self.assertEqual(inD.count('$$$$'),outD.count('$$$$'),'bad nMols in output')
def remove_test_databases():
db = dirs.tmp('test.db')
if os.path.exists(db):
os.unlink(db)
pristine = dirs.tmp('test.db.pristine')
if os.path.exists(pristine):
os.unlink(pristine)
def remove(path):
if os.path.exists(path):
print ' removing: ', path
if os.path.isdir(path):
shutil.rmtree(path)
else:
os.unlink(path)
def run_debugger(testname, pythonfile, pydb_opts='', args='',
outfile=None):
global srcdir, builddir, pydir
rightfile = os.path.join(srcdir, 'data', "%s.right" % testname)
os.environ['PYTHONPATH']=os.pathsep.join(sys.path)
cmdfile = os.path.join(srcdir, "%s.cmd" % testname)
outfile = "%s.out" % testname
outfile_opt = '--output=%s ' % outfile
# print "builddir: %s, cmdfile: %s, outfile: %s, rightfile: %s" % \
# (builddir, cmdfile, outfile, rightfile)
if os.path.exists(outfile): os.unlink(outfile)
cmd = "%s --command %s %s %s %s %s" % \
(pydb_path, cmdfile, outfile_opt, pydb_opts, pythonfile, args)
os.system(cmd)
fromfile = rightfile
fromdate = time.ctime(os.stat(fromfile).st_mtime)
fromlines = open(fromfile, 'U').readlines()
tofile = outfile
todate = time.ctime(os.stat(tofile).st_mtime)
tolines = open(tofile, 'U').readlines()
diff = list(difflib.unified_diff(fromlines, tolines, fromfile,
tofile, fromdate, todate))
if len(diff) == 0:
os.unlink(outfile)
for line in diff:
print line,
return len(diff) == 0
def _reset_database(self, conn_string):
conn_pieces = urlparse.urlparse(conn_string)
if conn_string.startswith('sqlite'):
# We can just delete the SQLite database, which is
# the easiest and cleanest solution
db_path = conn_pieces.path.strip('/')
if db_path and os.path.exists(db_path):
os.unlink(db_path)
# No need to recreate the SQLite DB. SQLite will
# create it for us if it's not there...
elif conn_string.startswith('mysql'):
# We can execute the MySQL client to destroy and re-create
# the MYSQL database, which is easier and less error-prone
# than using SQLAlchemy to do this via MetaData...trust me.
database = conn_pieces.path.strip('/')
loc_pieces = conn_pieces.netloc.split('@')
host = loc_pieces[1]
auth_pieces = loc_pieces[0].split(':')
user = auth_pieces[0]
password = ""
if len(auth_pieces) > 1:
if auth_pieces[1].strip():
password = "******" % auth_pieces[1]
sql = ("drop database if exists %(database)s; "
"create database %(database)s;") % locals()
cmd = ("mysql -u%(user)s %(password)s -h%(host)s "
"-e\"%(sql)s\"") % locals()
exitcode, out, err = execute(cmd)
self.assertEqual(0, exitcode)
def __restore_file(self,path):
try:
os.unlink(path)
except:
pass
if os.path.exists(path + '.rpmnew'):
os.rename(path + '.rpmnew', path)
def _stage_final_image(self):
try:
fs_related.makedirs(self.__ensure_isodir() + "/LiveOS")
minimal_size = self._resparse()
if not self.skip_minimize:
fs_related.create_image_minimizer(self.__isodir + \
"/LiveOS/osmin.img",
self._image,
minimal_size)
if self.skip_compression:
shutil.move(self._image, self.__isodir + "/LiveOS/ext3fs.img")
else:
fs_related.makedirs(os.path.join(
os.path.dirname(self._image),
"LiveOS"))
shutil.move(self._image,
os.path.join(os.path.dirname(self._image),
"LiveOS", "ext3fs.img"))
fs_related.mksquashfs(os.path.dirname(self._image),
self.__isodir + "/LiveOS/squashfs.img")
self.__create_iso(self.__isodir)
if self.pack_to:
isoimg = os.path.join(self._outdir, self.name + ".iso")
packimg = os.path.join(self._outdir, self.pack_to)
misc.packing(packimg, isoimg)
os.unlink(isoimg)
finally:
shutil.rmtree(self.__isodir, ignore_errors = True)
self.__isodir = None
def invoke_tool(self, test_name, reference_file, current_file, diff_file, no_save=None):
if no_save:
response = ReconcileTool._read_console(
"\n!!! MERGING NOT ALLOWED for {}: {}. Want to start kdiff3? ([y] | *): ".format(test_name, no_save)
)
if response != "y":
return None
cmd = (
'kdiff3 "{reference_file}" --L1 "{test_name}_REFERENCE" ' '"{current_file}" --L2 "{test_name}_CURRENT" '
).format(reference_file=reference_file, current_file=current_file, test_name=test_name)
else:
merged_file = self._aux_file_name(current_file, "merged")
cmd = (
'kdiff3 -m "{reference_file}" --L1 "{test_name}_REFERENCE" '
'"{current_file}" --L2 "{test_name}_CURRENT" '
' -o "{merged_file}"'
).format(
reference_file=reference_file, current_file=current_file, merged_file=merged_file, test_name=test_name
)
print(cmd)
code = ReconcileTool._invoke_command(cmd)
if no_save:
return None
else:
if code == 0:
# Merged ok
return merged_file
else:
if os.path.isfile(merged_file):
os.unlink(merged_file)
return None
def move_symlink(src, dst):
"""Create or overwrite a symlink"""
try:
os.unlink(dst)
except OSError:
pass
os.symlink(src, dst)
def polyop (query, prefix='impo: ', unlink=True) :
inpath = None
outpath = None
try :
# save the query into temp file
fd, inpath = tempfile.mkstemp (suffix='.pop.ml')
f = os.fdopen (fd, 'w')
f.write (query)
f.close ()
# run polyop
cmd = ['polyop', inpath]
exitcode, out = runit (cmd, prefix=prefix)
if exitcode != 0 :
raise Exception, 'polyop: exit code %d, output: "%s"' % (exitcode, out)
#print prefix, 'exit code 0'
#print prefix, 'stdout: "%s"' % out
# load result
outpath = inpath + '.res'
with open (outpath) as f : res = f.read ()
finally :
# remove temporary files, unless disabled by user
if unlink :
if inpath != None : os.unlink (inpath)
if outpath != None : os.unlink (outpath)
# strip and return
return res.strip (' \t\n')
def test_successful_download(self):
"""Test successful_download."""
page = pywikibot.FilePage(self.site, 'File:Albert Einstein.jpg')
filename = join_images_path('Albert Einstein.jpg')
status_code = page.download(filename)
self.assertTrue(status_code)
os.unlink(filename)
def build_gfortran(ctx, target):
binpath = os.path.join(ctx.out_dir, "bin")
manpath = os.path.join(ctx.out_dir, "share", "man", "man1")
includepath = os.path.join(ctx.out_dir, "include")
binfiles = os.listdir(binpath)
manfiles = os.listdir(manpath)
srcpath = ctx.env.SRCPATH
ctx.venv_exec("""
base="gcc-42-5666.3-darwin11"
pushd %(srcpath)s/3rdparty
rm -fr "$base"
mkdir -p "$base"
pushd "$base"
xar -xf "../$base.pkg"
mv *.pkg/Payload Payload.gz
pax --insecure -rz -f Payload.gz -s ",./usr,$VIRTUAL_ENV,"
ln -sf "$VIRTUAL_ENV/bin/gfortran-4.2" "$VIRTUAL_ENV/bin/gfortran"
popd
rm -fr "$base"
popd
""" % locals())
# Delete other files installed
shutil.rmtree(os.path.join(includepath, "gcc"))
for f in os.listdir(binpath):
if f not in binfiles and not "gfortran" in f:
os.unlink(os.path.join(binpath, f))
for f in os.listdir(manpath):
if f not in manfiles and not "gfortran" in f:
os.unlink(os.path.join(manpath, f))
def linkBestCurrentSolution(dpath):
alldsz = glob.glob(dpath + '/*dsz')
currentLink = None
bestDsz = None
bestN = None
stu = None
for x in alldsz:
if x.endswith('current.dsz'):
assert os.path.islink(x)
currentLink = os.readlink(x)
continue
fname = os.path.basename(x)
m = CURSOL_RE_.match(fname)
assert m is not None, ('failed to parse %s' % fname)
nstu = m.group(1).upper()
if stu is None:
stu = nstu
assert nstu == stu
nth = int(m.group(2))
if (bestDsz is None) or (nth > bestN):
bestN = nth
bestDsz = fname
if bestDsz is None:
return None
currentDsz = os.path.join(dpath, stu + 'current.dsz')
if os.path.islink(currentDsz):
odsz = os.readlink(currentDsz)
if odsz == bestDsz:
return bestDsz
os.unlink(currentDsz)
os.symlink(bestDsz, currentDsz)
return bestDsz
def fetch(self, pac, prefix=''):
# for use by the failure callback
self.curpac = pac
MirrorGroup._join_url = join_url
mg = MirrorGroup(self.gr, pac.urllist, failure_callback=(self.failureReport, (), {}))
if self.http_debug:
print('\nURLs to try for package \'%s\':' % pac, file=sys.stderr)
print('\n'.join(pac.urllist), file=sys.stderr)
print(file=sys.stderr)
try:
with tempfile.NamedTemporaryFile(prefix='osc_build',
delete=False) as tmpfile:
mg.urlgrab(pac.filename, filename=tmpfile.name,
text='%s(%s) %s' % (prefix, pac.project, pac.filename))
self.move_package(tmpfile.name, pac.localdir, pac)
except URLGrabError as e:
if self.enable_cpio and e.errno == 256:
self.__add_cpio(pac)
return
print()
print('Error:', e.strerror, file=sys.stderr)
print('Failed to retrieve %s from the following locations '
'(in order):' % pac.filename, file=sys.stderr)
print('\n'.join(pac.urllist), file=sys.stderr)
sys.exit(1)
finally:
if os.path.exists(tmpfile.name):
os.unlink(tmpfile.name)
def wait_for_match(self, images, similar_degree=0.98, timeout=300):
"""
Compare VM screenshot with given images, if any image in the list
matched, then return the image index, or return -1.
"""
end_time = time.time() + timeout
image_matched = False
cropped_image = os.path.join(data_dir.get_tmp_dir(), "croped.ppm")
while time.time() < end_time:
vm_screenshot = self.get_screenshot()
ppm_utils.image_crop_save(vm_screenshot, vm_screenshot)
img_index = 0
for image in images:
logging.debug("Compare vm screenshot with image %s", image)
ppm_utils.image_crop_save(image, cropped_image)
h_degree = ppm_utils.image_histogram_compare(cropped_image,
vm_screenshot)
if h_degree >= similar_degree:
logging.debug("Image %s matched", image)
image_matched = True
break
img_index += 1
if image_matched:
break
time.sleep(1)
if os.path.exists(cropped_image):
os.unlink(cropped_image)
if os.path.exists(vm_screenshot):
os.unlink(vm_screenshot)
if image_matched:
return img_index
else:
return -1
def run_merge(filenames):
"""Merges all Skype databases to a new database."""
dbs = [skypedata.SkypeDatabase(f) for f in filenames]
db_base = dbs.pop()
counts = collections.defaultdict(lambda: collections.defaultdict(int))
postbacks = Queue.Queue()
postfunc = lambda r: postbacks.put(r)
worker = workers.MergeThread(postfunc)
name, ext = os.path.splitext(os.path.split(db_base.filename)[-1])
now = datetime.datetime.now().strftime("%Y%m%d")
filename_final = util.unique_path("%s.merged.%s%s" % (name, now, ext))
print("Creating %s, using %s as base." % (filename_final, db_base))
shutil.copyfile(db_base.filename, filename_final)
db2 = skypedata.SkypeDatabase(filename_final)
chats2 = db2.get_conversations()
db2.get_conversations_stats(chats2)
for db1 in dbs:
chats = db1.get_conversations()
db1.get_conversations_stats(chats)
bar_total = sum(c["message_count"] for c in chats)
bar_text = " Processing %.*s.." % (30, db1)
bar = ProgressBar(max=bar_total, afterword=bar_text)
bar.start()
args = {"db1": db1, "db2": db2, "chats": chats,
"type": "diff_merge_left"}
worker.work(args)
while True:
result = postbacks.get()
if "error" in result:
print("Error merging %s:\n\n%s" % (db1, result["error"]))
worker = None # Signal for global break
break # break while True
if "done" in result:
break # break while True
if "diff" in result:
counts[db1]["chats"] += 1
counts[db1]["msgs"] += len(result["diff"]["messages"])
msgcounts = sum(c["message_count"] for c in result["chats"])
bar.update(bar.value + msgcounts)
if result["output"]:
log(result["output"])
if not worker:
break # break for db1 in dbs
bar.stop()
bar.afterword = " Processed %s." % db1
bar.update(bar_total)
print
if not counts:
print("Nothing new to merge.")
db2.close()
os.unlink(filename_final)
else:
for db1 in dbs:
print("Merged %s in %s from %s." %
(util.plural("message", counts[db1]["msgs"]),
util.plural("chat", counts[db1]["chats"]), db1))
print("Merge into %s complete." % db2)
def process_response(self, request, response):
if (settings.DEBUG or request.user.is_superuser) and 'prof' in request.GET:
self.prof.close()
out = StringIO.StringIO()
old_stdout = sys.stdout
sys.stdout = out
stats = hotshot.stats.load(self.tmpfile)
stats.sort_stats('time', 'calls')
stats.print_stats()
sys.stdout = old_stdout
stats_str = out.getvalue()
if response and response.content and stats_str:
response.content = "<pre>" + stats_str + "</pre>"
response.content = "\n".join(response.content.split("\n")[:40])
response.content += self.summary_for_files(stats_str)
os.unlink(self.tmpfile)
return response
def testTruncateOnWindows(self):
def bug801631():
# SF bug <http://www.python.org/sf/801631>
# "file.truncate fault on windows"
f = self.FileIO(TESTFN, 'w')
f.write(bytes(range(11)))
f.close()
f = self.FileIO(TESTFN,'r+')
data = f.read(5)
if data != bytes(range(5)):
self.fail("Read on file opened for update failed %r" % data)
if f.tell() != 5:
self.fail("File pos after read wrong %d" % f.tell())
f.truncate()
if f.tell() != 5:
self.fail("File pos after ftruncate wrong %d" % f.tell())
f.close()
size = os.path.getsize(TESTFN)
if size != 5:
self.fail("File size after ftruncate wrong %d" % size)
try:
bug801631()
finally:
os.unlink(TESTFN)
def teardown_module(module):
"""remove the temporary file created by tests in this file
this gets automatically called by nose"""
os.unlink(temp)
def delete(self):
os.unlink(self.filename)
def main():
import optparse
parser = optparse.OptionParser('usage: %prog')
parser.add_option('-m', '--mission',
default=DEFAULT_MISSION,
help='Mission [%default]')
parser.add_option('-d', '--dataDir',
default=DEFAULT_DATA_DIR,
help='Data dir [%default]')
parser.add_option('-r', '--resultsDir',
default=DEFAULT_RESULTS_DIR,
help='Results dir [%default]')
parser.add_option('--reconstructBinary',
default=DEFAULT_RECONSTRUCT_BINARY,
help='Path to reconstruct binary [%default]')
parser.add_option('-s', '--subsampleLevel',
default=DEFAULT_SUB,
help='Subsample level (can be empty) [%default]')
parser.add_option('-P', '--numProcessors',
default=DEFAULT_NUM_PROCESSORS, type='int',
help='Number of processors to use [%default]')
parser.add_option('-n', '--numFiles',
default=DEFAULT_NUM_FILES, type='int',
help='Number of files to process (0 to process all) [%default]')
parser.add_option('--showConfig',
action='store_true', default=False,
help='Show config and exit')
parser.add_option('--region',
default=DEFAULT_REGION,
help='Region to process [%default]')
parser.add_option('--extra',
default=DEFAULT_EXTRA,
help='Extra label that appears at end of name [%default]')
parser.add_option('--name',
default=DEFAULT_NAME,
help='Basename of results directory [%default]')
parser.add_option('--feb13',
action='store_true', default=False,
help='Use InitAlbedoMosaicFeb13()')
opts, args = parser.parse_args()
# stuff extra information into opts
if args:
opts.steps = args
else:
opts.steps = DEFAULT_STEPS
opts.HOME = os.environ['HOME']
opts.date = time.strftime('%Y%m%d')
opts.time = time.strftime('%H%M%S')
# set up logging params
resultsDir = expand('$resultsDir', vars(opts))
logTimeStr = datetime.datetime.now().isoformat()
logTimeStr = re.sub(r':', '', logTimeStr)
logTimeStr = re.sub(r'\.\d+$', '', logTimeStr)
logTmpl = resultsDir + '/log-%s.txt'
logFile = logTmpl % logTimeStr
opts.logFile = logFile
# debug opts if user requested it
if opts.showConfig:
import json
print json.dumps(vars(opts), indent=4, sort_keys=True)
sys.exit(0)
# start logging
print 'Logging to %s' % logFile
latestLogFile = logTmpl % 'latest'
os.system('mkdir -p %s' % resultsDir)
if os.path.lexists(latestLogFile):
os.unlink(latestLogFile)
os.symlink(os.path.basename(logFile), latestLogFile)
logging.basicConfig(filename=logFile,
level=logging.DEBUG,
format="%(levelname)-6s %(asctime)-15s %(message)s")
# do the real work
runMaster(opts)
def cleanup():
sys.modules.pop(name)
os.unlink(imp.cache_from_source(module.__file__))
def cleanup():
sys.modules.pop(name)
os.unlink(imp.cache_from_source(mapping[name]))
def DeleteApkTmpFiles(self):
for _, _, tmp_files in self.apk_info.values():
for tmp_file in tmp_files.values():
os.unlink(tmp_file)
def run(self, dry_run=False):
"""
Performs the clean-up operation by taking the following sequence of steps:
1. lists all cache files in individual corpora cache dirs
2. for all the corpora
2.1 find which files are old enough (see TTL) to be deleted
2.2 find an cache map entry in Redis and iterate over records found there
2.2.1 if a record matches a file which is waiting to be deleted, then both the
file and the record are removed
2.2.2 if a record does not match even any existing file than it is removed with
logged warning about a stale record
2.3 if there are still some files to be deleted it means that they are 'unbound'
(= there is no record in the respective cache map file);
these files are also deleted with a warning
Please note that this algorithm is unable to remove stale cache map entries as long
as there is no existing file within a matching directory (e.g. there is a bunch
of records for the "syn2010" corpus in Redis but the cache directory "/path/to/cache/syn2010"
is empty). During normal operation, the system should be able to fix such consistency
deviations. But it is a good idea to check the mapping from time to time whether there
are no stale records (e.g. after a corpus was removed/blocked).
arguments:
dry_run -- if True then no actual writing/deleting is performed
"""
num_deleted = 0
num_processed = 0
cache_files = self.list_dir()
self._log_stats(cache_files)
to_del = {}
for corpus_id, corpus_cache_files in cache_files.items(
): # processing corpus by corpus
real_file_hashes = set(
) # to be able to compare cache map with actual files
cache_key = self._entry_key_gen(corpus_id)
for cache_entry in corpus_cache_files:
num_processed += 1
item_key = os.path.basename(cache_entry[0]).rsplit('.conc')[0]
real_file_hashes.add(item_key)
if self._ttl < cache_entry[1] / 60.:
to_del[item_key] = cache_entry[0]
cache_map = self._db.hash_get_all(cache_key)
if cache_map:
try:
for item_hash, v in cache_map.items():
if item_hash in to_del:
if not dry_run:
os.unlink(to_del[item_hash])
self._db.hash_del(cache_key, item_hash)
else:
del to_del[item_hash]
num_deleted += 1
elif item_hash not in real_file_hashes:
if not dry_run:
self._db.hash_del(cache_key, item_hash)
logging.getLogger().warn(
'deleted stale cache map entry [%s][%s]' %
(cache_key, item_hash))
except Exception as ex:
logging.getLogger().warn(
'Failed to process cache map file (will be deleted): %s'
% (ex, ))
self._db.remove(cache_key)
else:
logging.getLogger().error('Cache map [%s] not found' %
cache_key)
for item_hash, unbound_file in to_del.items():
if not dry_run:
os.unlink(unbound_file)
logging.getLogger().warn('deleted unbound cache file: %s' %
unbound_file)
ans = {
'type': 'summary',
'processed': num_processed,
'deleted': num_deleted
}
logging.getLogger(__name__).info(json.dumps(ans))
return ans
def doPlots(self):
"""
"""
DATA_FILE = self.tempPedName # for haploview
INFO_FILE = self.tempMapName
fblog, blog = tempfile.mkstemp()
ste = open(blog, 'w') # to catch the blather
# if no need to rewrite - set up names for haploview call
vcl = [
javabin, '-jar', self.hvbin, '-n', '-memory',
'%d' % self.memSize, '-pairwiseTagging', '-pedfile', DATA_FILE,
'-info', INFO_FILE, '-tagrsqcounts', '-tagrsqcutoff', self.tagr2,
'-ldcolorscheme', self.ldType
]
if self.minMaf:
vcl += ['-minMaf', '%f' % self.minMaf]
if self.maxDist:
vcl += ['-maxDistance', self.maxDist]
if self.hiRes:
vcl.append('-png')
else:
vcl.append('-compressedpng')
if self.nchroms == 1:
vcl += ['-chromosome', self.chromosome]
if self.infotrack:
vcl.append('-infoTrack')
p = subprocess.Popen(' '.join(vcl),
shell=True,
cwd=self.outfpath,
stderr=ste,
stdout=self.lf)
retval = p.wait()
s = '## executing %s returned %d\n' % (' '.join(vcl), retval)
self.lf.write(s)
vcl = [self.mogrify, '-resize 800x400!', '*.PNG']
p = subprocess.Popen(' '.join(vcl),
shell=True,
cwd=self.outfpath,
stderr=self.lf,
stdout=self.lf)
retval = p.wait()
s = '## executing %s returned %d\n' % (' '.join(vcl), retval)
self.lf.write(s)
inpng = '%s.LD.PNG' % DATA_FILE # stupid but necessary - can't control haploview name mangle
inpng = inpng.replace(' ', '')
inpng = os.path.split(inpng)[-1]
tmppng = '%s.tmp.png' % self.title
tmppng = tmppng.replace(' ', '')
outpng = '1_%s.png' % self.title
outpng = outpng.replace(' ', '')
outpng = os.path.split(outpng)[-1]
vcl = [self.convert, '-resize 800x400!', inpng, tmppng]
p = subprocess.Popen(' '.join(vcl),
shell=True,
cwd=self.outfpath,
stderr=self.lf,
stdout=self.lf)
retval = p.wait()
s = '## executing %s returned %d\n' % (' '.join(vcl), retval)
self.lf.write(s)
s = "text 10,300 '%s'" % self.title[:40]
vcl = [
self.convert, '-pointsize 25', '-fill maroon',
'-draw "%s"' % s, tmppng, outpng
]
p = subprocess.Popen(' '.join(vcl),
shell=True,
cwd=self.outfpath,
stderr=self.lf,
stdout=self.lf)
retval = p.wait()
s = '## executing %s returned %d\n' % (' '.join(vcl), retval)
self.lf.write(s)
try:
os.remove(os.path.join(self.outfpath, tmppng))
except:
pass # label all the plots then delete all the .PNG files before munging
fnum = 1
if self.hmpanels:
sp = '%d' % (self.spos / 1000.) # hapmap wants kb
ep = '%d' % (self.epos / 1000.)
for panel in self.hmpanels:
if panel > '' and panel.lower(
) <> 'none': # in case someone checks that option too :)
ptran = panel.strip()
ptran = ptran.replace('+', '_')
fnum += 1 # preserve an order or else we get sorted
vcl = [
javabin, '-jar', self.hvbin, '-n', '-memory',
'%d' % self.memSize, '-chromosome', self.chromosome,
'-panel',
panel.strip(), '-hapmapDownload', '-startpos', sp,
'-endpos', ep, '-ldcolorscheme', self.ldType
]
if self.minMaf:
vcl += ['-minMaf', '%f' % self.minMaf]
if self.maxDist:
vcl += ['-maxDistance', self.maxDist]
if self.hiRes:
vcl.append('-png')
else:
vcl.append('-compressedpng')
if self.infotrack:
vcl.append('-infoTrack')
p = subprocess.Popen(' '.join(vcl),
shell=True,
cwd=self.outfpath,
stderr=ste,
stdout=self.lf)
retval = p.wait()
inpng = 'Chromosome%s%s.LD.PNG' % (self.chromosome, panel)
inpng = inpng.replace(' ', '') # mysterious spaces!
outpng = '%d_HapMap_%s_%s.png' % (fnum, ptran,
self.chromosome)
# hack for stupid chb+jpt
outpng = outpng.replace(' ', '')
tmppng = '%s.tmp.png' % self.title
tmppng = tmppng.replace(' ', '')
outpng = os.path.split(outpng)[-1]
vcl = [self.convert, '-resize 800x400!', inpng, tmppng]
p = subprocess.Popen(' '.join(vcl),
shell=True,
cwd=self.outfpath,
stderr=self.lf,
stdout=self.lf)
retval = p.wait()
s = '## executing %s returned %d\n' % (' '.join(vcl),
retval)
self.lf.write(s)
s = "text 10,300 'HapMap %s'" % ptran.strip()
vcl = [
self.convert, '-pointsize 25', '-fill maroon',
'-draw "%s"' % s, tmppng, outpng
]
p = subprocess.Popen(' '.join(vcl),
shell=True,
cwd=self.outfpath,
stderr=self.lf,
stdout=self.lf)
retval = p.wait()
s = '## executing %s returned %d\n' % (' '.join(vcl),
retval)
self.lf.write(s)
try:
os.remove(os.path.join(self.outfpath, tmppng))
except:
pass
nimages = len(glob.glob(os.path.join(
self.outfpath, '*.png'))) # rely on HaploView shouting - PNG @!
self.lf.write('### nimages=%d\n' % nimages)
if nimages > 0: # haploview may fail?
vcl = '%s -format pdf -resize 800x400! *.png' % self.mogrify
p = subprocess.Popen(vcl,
shell=True,
cwd=self.outfpath,
stderr=self.lf,
stdout=self.lf)
retval = p.wait()
self.lf.write('## executing %s returned %d\n' % (vcl, retval))
vcl = '%s *.pdf --fitpaper true --outfile alljoin.pdf' % self.pdfjoin
p = subprocess.Popen(vcl,
shell=True,
cwd=self.outfpath,
stderr=self.lf,
stdout=self.lf)
retval = p.wait()
self.lf.write('## executing %s returned %d\n' % (vcl, retval))
vcl = '%s alljoin.pdf --nup 1x%d --outfile allnup.pdf' % (
self.pdfnup, nimages)
p = subprocess.Popen(vcl,
shell=True,
cwd=self.outfpath,
stderr=self.lf,
stdout=self.lf)
retval = p.wait()
self.lf.write('## executing %s returned %d\n' % (vcl, retval))
vcl = '%s -resize x300 allnup.pdf allnup.png' % (self.convert)
p = subprocess.Popen(vcl,
shell=True,
cwd=self.outfpath,
stderr=self.lf,
stdout=self.lf)
retval = p.wait()
self.lf.write('## executing %s returned %d\n' % (vcl, retval))
ste.close() # temp file used to catch haploview blather
hblather = open(blog, 'r').readlines() # to catch the blather
os.unlink(blog)
if len(hblather) > 0:
self.lf.write('## In addition, Haploview complained:')
self.lf.write(''.join(hblather))
self.lf.write('\n')
self.lf.close()
flist = glob.glob(os.path.join(self.outfpath, '*'))
flist.sort()
ts = '!"#$%&\'()*+,-/:;<=>?@[\\]^_`{|}~' + string.whitespace
ftran = string.maketrans(ts, '_' * len(ts))
outf = file(self.outfile, 'w')
outf.write(galhtmlprefix % progname)
s = '<h4>rgenetics for Galaxy %s, wrapping HaploView</h4>' % (progname)
outf.write(s)
mainthumb = 'allnup.png'
mainpdf = 'allnup.pdf'
if os.path.exists(os.path.join(self.outfpath, mainpdf)):
if not os.path.exists(os.path.join(self.outfpath, mainthumb)):
outf.write(
'<table><tr><td colspan="3"><a href="%s">Main combined LD plot</a></td></tr></table>\n'
% (mainpdf))
else:
outf.write(
'<table><tr><td><a href="%s"><img src="%s" alt="Main combined LD image" hspace="10" align="middle">'
% (mainpdf, mainthumb))
outf.write(
'</td><td>Click the thumbnail at left to download the main combined LD image <a href=%s>%s</a></td></tr></table>\n'
% (mainpdf, mainpdf))
else:
outf.write(
'(No main image was generated - this usually means a Haploview error connecting to Hapmap site - please try later)<br/>\n'
)
outf.write('<br><div><hr><ul>\n')
for i, data in enumerate(flist):
dn = os.path.split(data)[-1]
if dn[:3] <> 'all':
continue
newdn = dn.translate(ftran)
if dn <> newdn:
os.rename(os.path.join(self.outfpath, dn),
os.path.join(self.outfpath, newdn))
dn = newdn
dnlabel = dn
ext = dn.split('.')[-1]
if dn == 'allnup.pdf':
dnlabel = 'All pdf plots on a single page'
elif dn == 'alljoin.pdf':
dnlabel = 'All pdf plots, each on a separate page'
outf.write('<li><a href="%s">%s - %s</a></li>\n' %
(dn, dn, dnlabel))
for i, data in enumerate(flist):
dn = os.path.split(data)[-1]
if dn[:3] == 'all':
continue
newdn = dn.translate(ftran)
if dn <> newdn:
os.rename(os.path.join(self.outfpath, dn),
os.path.join(self.outfpath, newdn))
dn = newdn
dnlabel = dn
ext = dn.split('.')[-1]
if dn == 'allnup.pdf':
dnlabel = 'All pdf plots on a single page'
elif dn == 'alljoin.pdf':
dnlabel = 'All pdf plots, each on a separate page'
elif ext == 'info':
dnlabel = '%s map data for Haploview input' % self.title
elif ext == 'ped':
dnlabel = '%s genotype data for Haploview input' % self.title
elif dn.find('CEU') <> -1 or dn.find('YRI') <> -1 or dn.find(
'CHB_JPT') <> -1: # is hapmap
dnlabel = 'Hapmap data'
if ext == 'TAGS' or ext == 'TESTS' or ext == 'CHAPS':
dnlabel = dnlabel + ' Tagger output'
outf.write('<li><a href="%s">%s - %s</a></li>\n' %
(dn, dn, dnlabel))
outf.write('</ol><br>')
outf.write("</div><div><hr>Job Log follows below (see %s)<pre>" %
self.logfn)
s = file(self.log_file, 'r').readlines()
s = '\n'.join(s)
outf.write('%s</pre><hr></div>' % s)
outf.write('</body></html>')
outf.close()
if self.useTemp:
try:
os.unlink(self.tempMapName)
os.unlink(self.tempPedName)
except:
pass
def handle_halt_queue(self, queue):
for halt in queue:
pid = halt.msg[0]
mtype = halt.msg[1]
if mtype == "VFS":
func_effect, func_name, abspath = halt.msg[2:]
abspathx = remove_trailing_slash(abspath)
if not abspath.startswith(self._directory):
if func_effect == "MODIFY":
halt.target.created_paths.add(abspathx)
elif func_effect == "READ":
if not abspathx in halt.target.created_paths:
if os.path.isfile(abspathx):
# halt.target.log("TODO: external dep: " + abspathx)
pass
else:
raise Exception(parts)
halt.target.release(pid)
continue
reltarget = abspath[len(self._directory)+1:]
if not reltarget: # root dir stat
halt.target.release(pid)
continue
resdef = get_crumb_def(os.path.dirname(reltarget), os.path.basename(reltarget))
if not resdef:
# We don't know how to generate this target.
if func_effect == "READ":
# must be a leaf dep
if os.path.isfile(abspath):
# halt.target.log("TODO: internal leaf dep: " + reltarget)
if reltarget not in halt.target.leaf_deps:
halt.target.log("detected leaf dep: " + reltarget)
halt.target.leaf_deps.add(reltarget)
halt.target.release(pid)
else:
if not os.path.exists(abspath):
# halt.target.log("TODO: internal leaf dep (non-existing): " + reltarget)
pass
else:
halt.target.log("TODO: internal leaf dep (non-file): " + reltarget)
halt.target.release(pid)
elif func_effect == "MODIFY":
halt.target.log("detected internal tempfile: " + reltarget)
halt.target.release(pid)
else:
raise Exception(parts)
else:
# We *do* know how to generate this target
if reltarget not in self.targets:
self.add_target(resdef)
target_path = resdef.target_path
dep_target = self.targets.get(target_path)
if func_effect == "MODIFY":
if dep_target is halt.target:
# It's us. Our output. Proceed.
halt.target.log("detected output: " + target_path)
halt.target.release(pid)
else:
# Nope, it's generating something else that doesn't belong to it
# but we know how to generate it ourselves, so something's amiss
halt.target.log("detected colliding side-effect: " + target_path)
raise Exception("can't handle it")
elif func_effect == "READ":
if dep_target.made:
# Already, done, proceed
if target_path not in halt.target.deps_already_done:
halt.target.log("dep target %s done already, proceeding" % (target_path, ))
halt.target.deps_already_done.add(target_path)
halt.target.release(pid)
else:
# Nope, we need to wait for it
if dep_target is halt.target:
# It's us. Our output. We try to probe it before generating it? No
# problem, make sure the old output isn't there, and proceed
if os.path.exists(abspath):
os.unlink(abspath)
halt.target.release(pid)
else:
halt.target.log("waiting for dep target %s to be made" % (target_path, ))
self.deps_waiting.setdefault(target_path, []).append(halt)
else:
raise Exception(parts)
else:
raise Exception(parts)
def unlock(key: str) -> None:
try:
os.unlink(f"/var/run/blueman-{key}")
except OSError:
pass
def run(test, params, env):
"""
Test command: virsh update-device.
Update device from an XML <file>.
1.Prepare test environment, adding a cdrom to VM.
2.Perform virsh update-device operation.
3.Recover test environment.
4.Confirm the test result.
"""
# Before doing anything - let's be sure we can support this test
# Parse flag list, skip testing early if flag is not supported
# NOTE: "".split("--") returns [''] which messes up later empty test
flag = params.get("updatedevice_flag", "")
flag_list = []
if flag.count("--"):
flag_list = flag.split("--")
for item in flag_list:
option = item.strip()
if option == "":
continue
if not bool(virsh.has_command_help_match("update-device", option)):
raise error.TestNAError(
"virsh update-device doesn't support --%s" % option)
# As per RH BZ 961443 avoid testing before behavior changes
if 'config' in flag_list:
# SKIP tests using --config if libvirt is 0.9.10 or earlier
if not libvirt_version.version_compare(0, 9, 10):
raise error.TestNAError("BZ 961443: --config behavior change "
"in version 0.9.10")
if 'persistent' in flag_list:
# SKIP tests using --persistent if libvirt 1.0.5 or earlier
if not libvirt_version.version_compare(1, 0, 5):
raise error.TestNAError("BZ 961443: --persistent behavior change "
"in version 1.0.5")
# Prepare initial vm state
vm_name = params.get("main_vm")
vmxml = VMXML.new_from_dumpxml(vm_name, options="--inactive")
vm = env.get_vm(vm_name)
start_vm = "yes" == params.get("start_vm", "no")
# Get the target bus/dev
target_bus = params.get("updatedevice_target_bus", "ide")
target_dev = params.get("updatedevice_target_dev", "hdc")
# Prepare tmp directory and files.
orig_iso = os.path.join(test.virtdir, "orig.iso")
test_iso = os.path.join(test.virtdir, "test.iso")
test_diff_iso = os.path.join(test.virtdir, "test_diff.iso")
update_xmlfile = os.path.join(test.tmpdir, "update.xml")
create_attach_xml(update_xmlfile, test_iso, target_bus, target_dev)
# This test needs a cdrom attached first - attach a cdrom
# to a shutdown vm. Then decide to restart or not
if vm.is_alive():
vm.destroy()
create_cdrom(vm_name, orig_iso, target_dev)
if start_vm:
vm.start()
domid = vm.get_id()
else:
domid = "domid invalid; domain is shut-off"
# Get remaining parameters for configuration.
twice = "yes" == params.get("updatedevice_twice", "no")
diff_iso = "yes" == params.get("updatedevice_diff_iso", "no")
vm_ref = params.get("updatedevice_vm_ref", "")
status_error = "yes" == params.get("status_error", "no")
extra = params.get("updatedevice_extra", "")
# OK let's give this a whirl...
try:
if vm_ref == "id":
vm_ref = domid
if twice:
# Don't pass in any flags
virsh.update_device(domainarg=domid,
filearg=update_xmlfile,
ignore_status=True,
debug=True)
if diff_iso:
# Swap filename of device backing file in update.xml
os.remove(update_xmlfile)
create_attach_xml(update_xmlfile, test_diff_iso, target_bus,
target_dev)
elif vm_ref == "uuid":
vm_ref = vmxml.uuid
elif vm_ref == "hex_id":
vm_ref = hex(int(domid))
elif vm_ref.find("updatedevice_invalid") != -1:
vm_ref = params.get(vm_ref)
elif vm_ref == "name":
vm_ref = "%s %s" % (vm_name, extra)
cmdresult = virsh.update_device(domainarg=vm_ref,
filearg=update_xmlfile,
flagstr=flag,
ignore_status=True,
debug=True)
status = cmdresult.exit_status
active_vmxml = VMXML.new_from_dumpxml(vm_name)
inactive_vmxml = VMXML.new_from_dumpxml(vm_name, options="--inactive")
finally:
vm.destroy(gracefully=False, free_mac_addresses=False)
vmxml.undefine()
vmxml.restore()
vmxml.define()
if os.path.exists(orig_iso):
os.remove(orig_iso)
if os.path.exists(test_iso):
os.remove(test_iso)
if os.path.exists(test_diff_iso):
os.remove(test_diff_iso)
# Result handling logic set errmsg only on error
errmsg = None
if status_error:
if status == 0:
errmsg = "Run successfully with wrong command!"
else: # Normal test
if status != 0:
errmsg = "Run failed with right command"
if diff_iso: # Expect the backing file to have updated
active_attached = is_attached(active_vmxml.devices, test_diff_iso,
target_dev)
inactive_attached = is_attached(inactive_vmxml.devices,
test_diff_iso, target_dev)
else: # Expect backing file to remain the same
active_attached = is_attached(active_vmxml.devices, test_iso,
target_dev)
inactive_attached = is_attached(inactive_vmxml.devices, test_iso,
target_dev)
# Check behavior of combination before individual!
if "config" in flag_list and "live" in flag_list:
if not active_attached:
errmsg = ("Active domain XML not updated when "
"--config --live options used")
if not inactive_attached:
errmsg = ("Inactive domain XML not updated when "
"--config --live options used")
elif "live" in flag_list and inactive_attached:
errmsg = ("Inactive domain XML updated when " "--live option used")
elif "config" in flag_list and active_attached:
errmsg = ("Active domain XML updated when " "--config option used")
# persistent option behavior depends on start_vm
if "persistent" in flag_list:
if start_vm:
if not active_attached or not inactive_attached:
errmsg = ("XML not updated when --persistent "
"option used on active domain")
else:
if not inactive_attached:
errmsg = ("XML not updated when --persistent "
"option used on inactive domain")
if len(flag_list) == 0:
# Not specifying any flag is the same as specifying --current
if start_vm:
if not active_attached:
errmsg = "Active domain XML not updated"
elif inactive_attached:
errmsg = ("Inactive domain XML updated when active "
"requested")
# Log some debugging info before destroying instances
if errmsg is not None:
logging.debug("Active XML:")
logging.debug(str(active_vmxml))
logging.debug("Inactive XML:")
logging.debug(str(inactive_vmxml))
logging.debug("active_attached: %s", str(active_attached))
logging.debug("inctive_attached: %s", str(inactive_attached))
logging.debug("Device XML:")
logging.debug(open(update_xmlfile, "r").read())
# clean up tmp files
del vmxml
del active_vmxml
del inactive_vmxml
os.unlink(update_xmlfile)
if errmsg is not None:
raise error.TestFail(errmsg)
async def run(self) -> MigrationResult:
await self.create_migration_table()
app_modules = self.get_app_modules()
migration_modules = {}
for app_module in app_modules:
app_config = getattr(app_module, "APP_CONFIG")
if app_config.app_name == self.app_name:
migration_modules = self.get_migration_modules(
app_config.migrations_folder_path)
break
ran_migration_ids = await Migration.get_migrations_which_ran(
app_name=self.app_name)
if len(ran_migration_ids) == 0:
# Make sure a success is returned, as we don't want this
# to appear as an error in automated scripts.
message = "No migrations to reverse!"
print(message)
return MigrationResult(success=True, message=message)
#######################################################################
if self.migration_id == "all":
earliest_migration_id = ran_migration_ids[0]
elif self.migration_id == "1":
earliest_migration_id = ran_migration_ids[-1]
else:
earliest_migration_id = self.migration_id
if earliest_migration_id not in ran_migration_ids:
message = ("Unrecognized migration name - must be one of "
f"{ran_migration_ids}")
print(message, file=sys.stderr)
return MigrationResult(success=False, message=message)
#######################################################################
latest_migration_id = ran_migration_ids[-1]
start_index = ran_migration_ids.index(earliest_migration_id)
end_index = ran_migration_ids.index(latest_migration_id) + 1
subset = ran_migration_ids[start_index:end_index]
reversed_migration_ids = list(reversed(subset))
#######################################################################
_continue = ("y" if self.auto_agree else input(
"About to undo the following migrations:\n"
f"{reversed_migration_ids}\n"
"Enter y to continue.\n"))
if _continue == "y":
print("Undoing migrations")
for migration_id in reversed_migration_ids:
print(f"Reversing {migration_id}")
migration_module = migration_modules[migration_id]
response = await migration_module.forwards()
if isinstance(response, MigrationManager):
await response.run_backwards()
await Migration.delete().where(Migration.name == migration_id
).run()
if self.clean:
os.unlink(migration_module.__file__)
return MigrationResult(success=True)
else: # pragma: no cover
message = "Not proceeding."
print(message, file=sys.stderr)
return MigrationResult(success=False, message=message)
"""
t = dbconn.run_query(q)
for s in t:
demno = str(s['id'])
#clipped_solar_raster_dir = out_path + os.sep + 'SRR_' + str(row['id'] / 1000 * 1000).zfill(4) + os.sep
clipped_solar_raster_dir = out_path + os.sep + demno + os.sep
clipped_solar_raster = clipped_solar_raster_dir + str(
row['id']) + '.img' # what raster format do we want???
if not os.path.exists(clipped_solar_raster_dir):
os.mkdir(clipped_solar_raster_dir)
if os.path.isfile(clipped_solar_raster):
print "WARNING: Overwriting existing solar raster image at " + clipped_solar_raster
os.unlink(clipped_solar_raster)
# run solar analyst
try:
solar_raster = arcpy.sa.AreaSolarRadiation(
in_surface_raster, latitude, sky_size, time_configuration,
day_interval, hour_interval, each_interval, z_factor,
slope_aspect_input_type, calculation_directions,
zenith_divisions, azimuth_divisions, diffuse_model_type,
diffuse_proportion, transmissivity,
out_direct_radiation_raster, out_diffuse_radiation_raster,
out_direct_duration_raster)
# clip to feature extent and saves output
envelope = "{0} {1} {2} {3}".format(int(row['xmin']),
int(row['ymin']),
int(row['xmax']),
def generate_checksums(repo,
output,
prg_wrap=None,
dl_path=None,
dl_mirror=None,
dl_slug=None,
curses=False,
force_cache=False,
list_files=False,
remove_prefixes=[]):
logging.info('Parsing repo...')
mods = RepoConf(repo)
mods.parse_includes()
if len(mods.mods) == 0:
logging.error('No mods found!')
return False
if not mods.validate():
logging.error('Failed to parse the repo!')
return False
cache = {'generated': 0, 'mods': {}}
start_time = time.time()
failed = False
results = {}
if os.path.isfile(output):
with open(output, 'r') as stream:
stream.seek(0, os.SEEK_END)
# Don't try to parse the file if it's empty.
if stream.tell() != 0:
stream.seek(0)
cache = json.load(stream)
c_mods = {}
for mod in cache['mods']:
if mod['id'] not in c_mods:
c_mods[mod['id']] = {}
c_mods[mod['id']][mod['version']] = mod
cache['mods'] = c_mods
# Thu, 24 Jul 2014 12:00:16 GMT
locale.setlocale(locale.LC_TIME, 'C')
tstamp = time.strftime('%a, %d %b %Y %H:%M:%S GMT',
time.gmtime(cache['generated']))
items = []
for mid, mvs in mods.mods.items():
for ver, mod in mvs.items():
c_mod = cache['mods'].get(mid, {}).get(str(ver), {})
c_pkgs = [pkg['name'] for pkg in c_mod.get('packages', [])]
for pkg in mod.packages:
my_tstamp = 0
if force_cache and pkg.name in c_pkgs:
if mid not in results:
results[mid] = {}
if ver not in results[mid]:
results[mid][ver] = {}
if pkg.name not in results[mid][ver]:
results[mid][ver][pkg.name] = {}
for name, file_ in pkg.files.items():
results[mid][ver][pkg.name][name] = {
'md5sum': 'CACHE',
'contents': [],
'size': 0
}
continue
# Only download the files if we have no checksums or they changed.
if pkg.name in c_pkgs:
my_tstamp = tstamp
for name, file_ in pkg.files.items():
# id_, links, name, archive, tstamp
items.append(
((mid, mod.version, pkg.name, name), file_.urls, name,
file_.is_archive, my_tstamp))
if len(items) < 1:
if not force_cache:
logging.error('No files found!')
failed = True
file_info = []
else:
logging.info('Generating checksums...')
init_app()
task = ChecksumTask(items, dl_path, dl_mirror, dl_slug,
remove_prefixes)
run_task(task, prg_wrap, curses)
file_info = task.get_results()
logging.info('Saving data...')
logos = {}
for id_, csum, content, size in file_info:
mid, ver, pkg, name = id_
if csum == 'FAILED':
failed = True
continue
if name == 'logo.jpg':
if mid not in logos:
logos[mid] = {}
logos[mid][ver] = csum
else:
if mid not in results:
results[mid] = {}
if ver not in results[mid]:
results[mid][ver] = {}
if pkg not in results[mid][ver]:
results[mid][ver][pkg] = {}
results[mid][ver][pkg][name] = {
'md5sum': csum,
'contents': content,
'size': size
}
outpath = os.path.dirname(output)
new_cache = {'generated': start_time, 'mods': []}
# Well, this looks horrible...
# This loop copies the Mod objects from mods, adds the data from results and places the objects in new_cache['mods'].
# It also merges in data from cache['mods'] if necessary (whenever the md5sum of a file/archive is set to CACHE).
for mid, mvs in mods.mods.items():
for ver, mod in mvs.items():
mod = mod.copy()
c_pkgs = {}
sver = str(ver)
# Retrieve our packages from cache
if mid in cache['mods'] and sver in cache['mods'][mid]:
for pkg in cache['mods'][mid][sver]['packages']:
c_pkgs[pkg['name']] = pkg.copy()
if mid in logos and ver in logos[mid] and mod.logo is None:
fd, name = tempfile.mkstemp(dir=outpath,
prefix='logo',
suffix='.' +
logos[mid][ver].split('.')[-1])
os.close(fd)
shutil.move(logos[mid][ver], name)
mod.logo = os.path.basename(name)
for pkg in mod.packages:
c_files = {}
files = {}
# Retrieve our files from cache
if pkg.name in c_pkgs:
for item in c_pkgs[pkg.name]['files']:
c_files[item['filename']] = item
for item in c_pkgs[pkg.name]['filelist']:
if item['orig_name'] is None:
continue
ar = c_files[item['archive']]
if 'contents' not in ar:
ar['contents'] = {}
ar['contents'][item['orig_name']] = item['md5sum']
# Look for files which should keep their old checksum (from cache).
if mid in results and ver in results[
mid] and pkg.name in results[mid][ver]:
files = results[mid][ver][pkg.name]
for name, info in files.items():
if info['md5sum'] == 'CACHE':
if name not in c_files:
logging.error(
'Tried to retrieve a checksum from the cache but it was\'t there! (This is a bug!)'
)
failed = True
else:
info['md5sum'] = c_files[name]['md5sum']
info['contents'] = c_files[name]['contents']
info['size'] = c_files[name]['filesize']
else:
logging.error('Checksums for "%s" are missing!', mid)
failed = True
# Copy the information from rests
for name, info in pkg.files.items():
if name not in files:
logging.error(
'Missing information for file "%s" of package "%s".',
name, pkg.name)
else:
info.md5sum = files[name]['md5sum']
info.contents = files[name]['contents']
info.filesize = files[name]['size']
mod.build_file_list()
# Check files
done = False
for pkg in mod.packages:
for item in pkg.filelist:
if item['filename'].endswith('mod.ini'):
if item['filename'] == 'mod.ini':
# All is well...
done = True
break
else:
prefix = item['filename'][:-7].strip('/')
logging.warn(
'Found mod.ini in folder %s. I assume you forgot the move action; I\'ll just add it myself.',
prefix)
mod.actions.append({
'type': 'move',
'paths': [prefix + '/*'],
'dest': '',
'glob': True
})
mod.build_file_list()
done = True
break
if done:
break
if list_files:
file_list = [
'Please make sure the following is correct:',
'If a user has FS installed in C:\\Freespace2, your mod will install the following files:',
''
]
prefix = 'C:\\Freespace2\\' + mod.folder
for pkg in mod.packages:
for item in pkg.filelist:
file_list.append(prefix + '\\' +
item['filename'].replace('/', '\\'))
logging.info('\n'.join(file_list))
new_cache['mods'].append(mod.get())
with open(output, 'w') as stream:
json.dump(new_cache, stream, separators=(',', ':'))
# Cleanup
for path in logos.values():
if os.path.isfile(path):
os.unlink(path)
if failed:
logging.error('Failed!')
return False
else:
logging.info('Done')
return True
def do_start(verb, pid_file, server, args):
if verb != 'Respawn' and pid_file == CONF.pid_file:
for pid_file, pid in pid_files(server, pid_file):
if os.path.exists('/proc/%s' % pid):
print(
_("%(serv)s appears to already be running: %(pid)s") % {
'serv': server,
'pid': pid_file
})
return
else:
print(_("Removing stale pid file %s") % pid_file)
os.unlink(pid_file)
try:
resource.setrlimit(resource.RLIMIT_NOFILE,
(MAX_DESCRIPTORS, MAX_DESCRIPTORS))
resource.setrlimit(resource.RLIMIT_DATA, (MAX_MEMORY, MAX_MEMORY))
except ValueError:
print(
_('Unable to increase file descriptor limit. '
'Running as non-root?'))
os.environ['PYTHON_EGG_CACHE'] = '/tmp'
def write_pid_file(pid_file, pid):
with open(pid_file, 'w') as fp:
fp.write('%d\n' % pid)
def redirect_to_null(fds):
with open(os.devnull, 'r+b') as nullfile:
for desc in fds: # close fds
try:
os.dup2(nullfile.fileno(), desc)
except OSError:
pass
def redirect_to_syslog(fds, server):
log_cmd = 'logger'
log_cmd_params = '-t "%s[%d]"' % (server, os.getpid())
process = subprocess.Popen([log_cmd, log_cmd_params],
stdin=subprocess.PIPE)
for desc in fds: # pipe to logger command
try:
os.dup2(process.stdin.fileno(), desc)
except OSError:
pass
def redirect_stdio(server, capture_output):
input = [sys.stdin.fileno()]
output = [sys.stdout.fileno(), sys.stderr.fileno()]
redirect_to_null(input)
if capture_output:
redirect_to_syslog(output, server)
else:
redirect_to_null(output)
@gated_by(CONF.capture_output)
def close_stdio_on_exec():
fds = [sys.stdin.fileno(), sys.stdout.fileno(), sys.stderr.fileno()]
for desc in fds: # set close on exec flag
fcntl.fcntl(desc, fcntl.F_SETFD, fcntl.FD_CLOEXEC)
def launch(pid_file, conf_file=None, capture_output=False, await_time=0):
args = [server]
if conf_file:
args += ['--config-file', conf_file]
msg = (_('%(verb)sing %(serv)s with %(conf)s') % {
'verb': verb,
'serv': server,
'conf': conf_file
})
else:
msg = (_('%(verb)sing %(serv)s') % {'verb': verb, 'serv': server})
print(msg)
close_stdio_on_exec()
pid = os.fork()
if pid == 0:
os.setsid()
redirect_stdio(server, capture_output)
try:
os.execlp('%s' % server, *args)
except OSError as e:
msg = (_('unable to launch %(serv)s. Got error: %(e)s') % {
'serv': server,
'e': e
})
sys.exit(msg)
sys.exit(0)
else:
write_pid_file(pid_file, pid)
await_child(pid, await_time)
return pid
@gated_by(CONF.await_child)
def await_child(pid, await_time):
bail_time = time.time() + await_time
while time.time() < bail_time:
reported_pid, status = os.waitpid(pid, os.WNOHANG)
if reported_pid == pid:
global exitcode
exitcode = os.WEXITSTATUS(status)
break
time.sleep(0.05)
conf_file = None
if args and os.path.exists(args[0]):
conf_file = os.path.abspath(os.path.expanduser(args[0]))
return launch(pid_file, conf_file, CONF.capture_output, CONF.await_child)
def tearDown( self):
os.unlink( self.SAMPLE_FILE)
def tearDown(self):
try:
os.unlink(TESTFN)
except OSError:
pass
def tif2kmz(args):
if args.infile is not None:
base, ext = path.splitext(args.infile)
kmltif = base + '_kml' + ext
kmlpng = base + '_kml' + '.png'
kmlfile = base + '.kml'
kmzfile = base + '.kmz'
in_ds = gdal.Open(args.infile, gdal.GA_ReadOnly)
# warp original dataset to WGS84 used by google earth
tmp_ds = gdal.Warp(base + '_kml' + ext, in_ds, dstSRS='EPSG:4326')
# get geotransform for latlonbox
gt = tmp_ds.GetGeoTransform()
cols = tmp_ds.RasterXSize
rows = tmp_ds.RasterYSize
# convert image format for use by Icon, nodata will be transparent
gdal.Translate(kmlpng, tmp_ds, format="PNG")
in_ds = None
tmp_ds = None
data = {
'folder': {
'name': base,
'description': args.description
},
'overlay': {
'name': base,
'description': args.description
},
'imgurl': kmlpng,
'north': gt[3],
'south': gt[3] + gt[5] * rows,
'west': gt[0],
'east': gt[0] + gt[1] * cols,
'rotation': 0,
'color': "{}ffffff".format(hex(int(args.alpha * 255))[2:])
}
t = Template("""<?xml version="1.0" encoding="UTF-8"?>
<kml>
<Folder>
<name>{{ folder.name }}</name>
<visibility>1</visibility>
<description>{{ folder.description }}</description>
<GroundOverlay>
<name>{{ overlay.name }}</name>
<visibility>1</visibility>
<description>{{ overlay.description }}</description>
<color>{{ color }} </color>
<Icon>
<href>{{imgurl}}</href>
</Icon>
<LatLonBox>
<north>{{north}}</north>
<south>{{south}}</south>
<east>{{east}}</east>
<west>{{west}}</west>
<rotation>{{rotation}}</rotation>
</LatLonBox>
</GroundOverlay>
</Folder>
</kml>""")
with open(kmlfile, "w") as fo:
fo.write(t.render(data))
if args.zip:
zipf = zipfile.ZipFile(kmzfile, "w", zipfile.ZIP_DEFLATED)
zipf.write(kmlfile)
zipf.write(kmlpng)
zipf.close()
if args.removetemp:
if path.exists(kmltif): os.unlink(kmltif)
if path.exists(kmlfile): os.unlink(kmlfile)
if path.exists(kmlpng): os.unlink(kmlpng)
def extract(self, project, package, srcinfo, repo, arch):
# fetch cpio headers
# check file lists for library packages
fetchlist, liblist = self.compute_fetchlist(project, package, srcinfo, repo, arch)
if not fetchlist:
msg = "no libraries found in %s/%s %s/%s"%(project, package, repo, arch)
self.logger.info(msg)
return None
# mtimes in cpio are not the original ones, so we need to fetch
# that separately :-(
mtimes= self._getmtimes(project, package, repo, arch)
self.logger.debug("fetchlist %s", pformat(fetchlist))
self.logger.debug("liblist %s", pformat(liblist))
debugfiles = set(['/usr/lib/debug%s.debug'%f for f in liblist])
# fetch binary rpms
downloaded = self.download_files(project, package, repo, arch, fetchlist, mtimes)
# extract binary rpms
tmpfile = os.path.join(CACHEDIR, "cpio")
for fn in fetchlist:
self.logger.debug("extract %s"%fn)
with open(tmpfile, 'wb') as tmpfd:
if not fn in downloaded:
raise FetchError("%s was not downloaded!"%fn)
self.logger.debug(downloaded[fn])
r = subprocess.call(['rpm2cpio', downloaded[fn]], stdout=tmpfd, close_fds=True)
if r != 0:
raise FetchError("failed to extract %s!"%fn)
tmpfd.close()
cpio = CpioRead(tmpfile)
cpio.read()
for ch in cpio:
fn = ch.filename
if fn.startswith('./'): # rpm payload is relative
fn = fn[1:]
self.logger.debug("cpio fn %s", fn)
if not fn in liblist and not fn in debugfiles:
continue
dst = os.path.join(UNPACKDIR, project, package, repo, arch)
dst += fn
if not os.path.exists(os.path.dirname(dst)):
os.makedirs(os.path.dirname(dst))
self.logger.debug("dst %s", dst)
# the filehandle in the cpio archive is private so
# open it again
with open(tmpfile, 'rb') as cpiofh:
cpiofh.seek(ch.dataoff, os.SEEK_SET)
with open(dst, 'wb') as fh:
while True:
buf = cpiofh.read(4096)
if buf is None or buf == '':
break
fh.write(buf)
os.unlink(tmpfile)
return liblist
def install(self, package, update=False):
if package.source_type == "directory":
self.install_directory(package)
return
if package.source_type == "git":
self.install_git(package)
return
args = ["install", "--no-deps"]
if package.source_type == "legacy" and package.source_url:
parsed = urlparse.urlparse(package.source_url)
if parsed.scheme == "http":
self._io.write_error(
" <warning>Installing from unsecure host: {}</warning>".format(
parsed.hostname
)
)
args += ["--trusted-host", parsed.hostname]
auth = get_http_basic_auth(
Config.create("auth.toml"), package.source_reference
)
if auth:
index_url = "{scheme}://{username}:{password}@{netloc}{path}".format(
scheme=parsed.scheme,
username=auth[0],
password=auth[1],
netloc=parsed.netloc,
path=parsed.path,
)
else:
index_url = package.source_url
args += ["--index-url", index_url]
if update:
args.append("-U")
if package.hashes and not package.source_type:
# Format as a requirements.txt
# We need to create a requirements.txt file
# for each package in order to check hashes.
# This is far from optimal but we do not have any
# other choice since this is the only way for pip
# to verify hashes.
req = self.create_temporary_requirement(package)
args += ["-r", req]
try:
self.run(*args)
finally:
os.unlink(req)
else:
req = self.requirement(package)
if not isinstance(req, list):
args.append(req)
else:
args += req
self.run(*args)
def close(self):
'''Close the workspace and delete the temporary measurements file'''
if self.measurements is not None and self.__filename is not None:
self.measurements.close()
os.unlink(self.__filename)
def tearDown(self):
os.close(self.db_fd)
os.unlink(flaskr.app.config['DATABASE'])
def tearDown(self):
"delete the file we have created"
import os
os.unlink("__unittests.xml")
def cleanup():
if os.path.exists(old_dump):
os.unlink(old_dump)
if os.path.exists(new_dump):
os.unlink(new_dump)
def process(tsv, metric = None, query = None, enumsize = None):
with open(tsv) as inf:
lines = inf.readlines()
colnames = lines[0][1:].strip().split('\t') # first line is a header that starts with '#'
column = colnames.index # helper function to find column number by name
data = [line.strip().split('\t') for line in lines[1:]]
for i in xrange(len(data)):
data[i][1] = long(data[i][1]) # store size is a long
for j in xrange(5, len(colnames)):
data[i][j] = float(data[i][j]) # all metrics have float values
unique_vals = lambda colnum: list(OrderedDict.fromkeys(row[colnum] for row in data))
ages = unique_vals(column('age class'))
lengths = unique_vals(column('length class'))
try:
if metric is None:
assert len(colnames) == 6
metric = colnames[5]
else:
assert metric in colnames[5:]
except AssertionError:
raise AssertionError('-metric must be specified as one of %s' % ' '.join(colnames[5:]))
try:
all_queries = unique_vals(column('query'))
if query is None:
assert len(all_queries) == 1
query = all_queries[0]
else:
assert query in all_queries
except AssertionError:
raise AssertionError('-query must be specified as one of %s' % ' '.join(all_queries))
pdf_files = 'tmp-A%s-L%s.pdf'
dat_file = 'tmp-data.tsv'
gp_file = 'tmp.gp'
tex_file = 'plot.tex'
tex_aux_files = ['plot.aux', 'plot.log']
all_files = [pdf_files % (age, length) for age in ages for length in lengths] + [dat_file, gp_file, tex_file] + tex_aux_files
pdfs = defaultdict(dict)
for age in ages:
for length in lengths:
have_data = False
with open(dat_file, 'w') as outf:
for row in data:
if row[column('query')] == query and row[column('age class')] == age and row[column('length class')] == length:
have_data = True
outf.write("%s\t%s\n" % (row[1], row[column(metric)]))
if have_data:
out_file = pdf_files % (age, length)
pdfs[age][length] = out_file
with open(gp_file, 'w') as outf:
if enumsize is not None:
normfact = float(enumsize)
plotx = 'set xrange [0:1]'
else:
normfact = 1
plotx = 'set autoscale x'
outf.write("""load "template.gp"
set term pdfcairo lw 8
unset grid
unset xtics
unset ytics
set output "{out_file}"
{plotx}
set yrange [0:100]
plot "{dat_file}" u ($1/{normfact}):($2*100) w lp ls 1 notitle
""".format(**locals()))
gnuplot(gp_file)
with open(tex_file, 'w') as outf:
outf.write("""\documentclass[12pt]{article}
\usepackage{graphicx}
\usepackage{palatino}
\usepackage{diagbox}
\usepackage{pdflscape}
\\begin{document}
\\thispagestyle{empty}
\\begin{landscape}
\\begin{table}
\centering
\\begin{tabular}{l|%s}
%%\hline
\\diagbox{age}{length} & %s \\\\
\hline
""" % (\
''.join('l' for i in xrange(len(lengths))),\
' & '.join(lengths)))
for age in ages:
if age not in pdfs:
continue
outf.write("%s" % age)
for length in lengths:
if length not in pdfs[age]:
continue
outf.write(" & \includegraphics[width=0.15\\textwidth]{%s}" % pdfs[age][length])
outf.write("""\\\\
%\hline
""")
if enumsize is not None:
xaxis = 'sstore/enumeration bytes [0:1]'
else:
xaxis = '\# sstore windows'
if metric[0] == 'p' and is_float(metric[1:]):
yaxis = '%sth percentile' % metric[1:]
else:
yaxis = metric
yaxis += ' percent error [0:100]'
outf.write("""
\end{tabular}
\caption{%s query: x-axis %s, y-axis %s}
\end{table}
\end{landscape}
\end{document}""" % (query, xaxis, yaxis))
pdflatex(tex_file)
for fil in all_files:
try:
os.unlink(fil)
except OSError: # file doesn't exist
pass
def handle(self, *args, **options):
start = datetime.now() # Measure the time it takes to run the script.
day = options['date']
if not day:
day = (datetime.now() - timedelta(days=1)).strftime('%Y-%m-%d')
folder = options['folder_name']
folder = path.join(settings.TMP_PATH, folder, day)
sep = options['separator']
filepath = path.join(folder, 'download_counts.hive')
# Make sure we're not trying to update with mismatched data.
if get_date_from_file(filepath, sep) != day:
raise CommandError('%s file contains data for another day' %
filepath)
# First, make sure we don't have any existing counts for the same day,
# or it would just increment again the same data.
DownloadCount.objects.filter(date=day).delete()
# Memoize the files to addon relations and the DownloadCounts.
download_counts = {}
# Perf: preload all the files and slugs once and for all.
# This builds two dicts:
# - One where each key (the file_id we get from the hive query) has
# the addon_id as value.
# - One where each key (the add-on slug) has the add-on_id as value.
files_to_addon = dict(File.objects.values_list('id',
'version__addon_id'))
slugs_to_addon = dict(Addon.objects.public().values_list('slug', 'id'))
# Only accept valid sources, which are listed in the DownloadSource
# model. The source must either be exactly one of the "full" valid
# sources, or prefixed by one of the "prefix" valid sources.
fulls = set(DownloadSource.objects.filter(type='full').values_list(
'name', flat=True))
prefixes = DownloadSource.objects.filter(type='prefix').values_list(
'name', flat=True)
with codecs.open(filepath, encoding='utf8') as count_file:
for index, line in enumerate(count_file):
if index and (index % 1000000) == 0:
log.info('Processed %s lines' % index)
splitted = line[:-1].split(sep)
if len(splitted) != 4:
log.debug('Badly formatted row: %s' % line)
continue
day, counter, id_or_slug, src = splitted
try:
# Clean up data.
id_or_slug = id_or_slug.strip()
counter = int(counter)
except ValueError:
# Ignore completely invalid data.
continue
if id_or_slug.strip().isdigit():
# If it's a digit, then it should be a file id.
try:
id_or_slug = int(id_or_slug)
except ValueError:
continue
# Does this file exist?
if id_or_slug in files_to_addon:
addon_id = files_to_addon[id_or_slug]
# Maybe it's an add-on ?
elif id_or_slug in files_to_addon.values():
addon_id = id_or_slug
else:
# It's an integer we don't recognize, ignore the row.
continue
else:
# It's probably a slug.
if id_or_slug in slugs_to_addon:
addon_id = slugs_to_addon[id_or_slug]
else:
# We've exhausted all possibilities, ignore this row.
continue
if not is_valid_source(src, fulls=fulls, prefixes=prefixes):
continue
# Memoize the DownloadCount.
if addon_id in download_counts:
dc = download_counts[addon_id]
else:
dc = DownloadCount(date=day, addon_id=addon_id, count=0)
download_counts[addon_id] = dc
# We can now fill the DownloadCount object.
dc.count += counter
dc.sources = update_inc(dc.sources, src, counter)
# Create in bulk: this is much faster.
DownloadCount.objects.bulk_create(download_counts.values(), 100)
for download_count in download_counts.values():
save_stats_to_file(download_count)
log.info('Processed a total of %s lines' % (index + 1))
log.debug('Total processing time: %s' % (datetime.now() - start))
# Clean up file.
log.debug('Deleting {path}'.format(path=filepath))
unlink(filepath)
def handle_func(self, func, data):
'''
Execute this method in a multiprocess or thread
'''
if salt.utils.is_windows():
self.functions = salt.loader.minion_mods(self.opts)
self.returners = salt.loader.returners(self.opts, self.functions)
ret = {
'id': self.opts.get('id', 'master'),
'fun': func,
'schedule': data['name'],
'jid': '{0:%Y%m%d%H%M%S%f}'.format(datetime.datetime.now())
}
proc_fn = os.path.join(salt.minion.get_proc_dir(self.opts['cachedir']),
ret['jid'])
# Check to see if there are other jobs with this
# signature running. If there are more than maxrunning
# jobs present then don't start another.
# If jid_include is False for this job we can ignore all this
# NOTE--jid_include defaults to True, thus if it is missing from the data
# dict we treat it like it was there and is True
if 'jid_include' not in data or data['jid_include']:
jobcount = 0
for basefilename in os.listdir(
salt.minion.get_proc_dir(self.opts['cachedir'])):
fn_ = os.path.join(
salt.minion.get_proc_dir(self.opts['cachedir']),
basefilename)
with salt.utils.fopen(fn_, 'r') as fp_:
job = salt.payload.Serial(self.opts).load(fp_)
if job:
if 'schedule' in job:
log.debug(
'schedule.handle_func: Checking job against '
'fun {0}: {1}'.format(ret['fun'], job))
if ret['schedule'] == job[
'schedule'] and os_is_running(job['pid']):
jobcount += 1
log.debug(
'schedule.handle_func: Incrementing jobcount, now '
'{0}, maxrunning is {1}'.format(
jobcount, data['maxrunning']))
if jobcount >= data['maxrunning']:
log.debug(
'schedule.handle_func: The scheduled job {0} '
'was not started, {1} already running'.
format(ret['schedule'],
data['maxrunning']))
return False
else:
try:
log.info('Invalid job file found. Removing.')
os.remove(fn_)
except OSError:
log.info('Unable to remove file: {0}.'.format(fn_))
salt.utils.daemonize_if(self.opts)
ret['pid'] = os.getpid()
if 'jid_include' not in data or data['jid_include']:
log.debug('schedule.handle_func: adding this job to the jobcache '
'with data {0}'.format(ret))
# write this to /var/cache/salt/minion/proc
with salt.utils.fopen(proc_fn, 'w+') as fp_:
fp_.write(salt.payload.Serial(self.opts).dumps(ret))
args = None
if 'args' in data:
args = data['args']
kwargs = None
if 'kwargs' in data:
kwargs = data['kwargs']
try:
if args and kwargs:
ret['return'] = self.functions[func](*args, **kwargs)
if args and not kwargs:
ret['return'] = self.functions[func](*args)
if kwargs and not args:
ret['return'] = self.functions[func](**kwargs)
if not kwargs and not args:
ret['return'] = self.functions[func]()
data_returner = data.get('returner', None)
if data_returner or self.schedule_returner:
if 'returner_config' in data:
ret['ret_config'] = data['returner_config']
rets = []
for returner in [data_returner, self.schedule_returner]:
if isinstance(returner, str):
rets.append(returner)
elif isinstance(returner, list):
rets.extend(returner)
# simple de-duplication with order retained
rets = OrderedDict.fromkeys(rets).keys()
for returner in rets:
ret_str = '{0}.returner'.format(returner)
if ret_str in self.returners:
ret['success'] = True
self.returners[ret_str](ret)
else:
log.info(
'Job {0} using invalid returner: {1}. Ignoring.'.
format(func, returner))
except Exception:
log.exception("Unhandled exception running {0}".format(ret['fun']))
# Although catch-all exception handlers are bad, the exception here
# is to let the exception bubble up to the top of the thread context,
# where the thread will die silently, which is worse.
finally:
try:
os.unlink(proc_fn)
except OSError as exc:
if exc.errno == errno.EEXIST:
# EEXIST is OK because the file is gone and that's what
# we wanted
pass
else:
log.error("Failed to delete '{0}': {1}".format(
proc_fn, exc.errno))
# Otherwise, failing to delete this file is not something
# we can cleanly handle.
raise
def _clean_up(self):
try:
BlockDev.loop_teardown(self.loop)
except:
pass
os.unlink(self.dev_file)
def source(self):
download("https://github.com/cpputest/cpputest/releases/download/v3.8/cpputest-3.8.zip", "cpputest-3.8.zip")
unzip("cpputest-3.8.zip")
os.unlink("cpputest-3.8.zip")
def extract_zips(unzip_dir, files):
# Handles de-duping; if there are multiple files with the same data,
# we will use only one of the equivalent set. If there is an entry in
# the filename dictionary, that takes precedence.
md5sums = [md5(fn) for fn in files]
mdict = defaultdict(list)
for md, fn in zip(md5sums, files):
mdict[md].append(fn)
# Try to parse out at least the kit number by trying a series of regular
# expressions. Adding regular expressions at the end of this list is safer
# than at the beginning. Order is important - rules at top are matched
# first.
# constants used in filename regular expressions
# groupings (?:xxx) are ignored
ws = r'^[_]?'
nam1 = r"[a-z]{0,20}|O\'[a-z]{3,20}|O['][a-z]{3,20}"
cname = r'([\w]{1,20})' #matches unicode chars; also matches digits though
pnam = r'\(' + nam1 + r'\)'
nam2 = r'(?:' + nam1 + '|' + pnam + r')'
ndate = r'(?:(201[1-8][\d]{4}|201[1-8]-\d\d-\d\d|\d{4}201[1-8]))'
sep = r'[\-\s\._]'
seps = r'[\-\s\._]?'
sepp = r'[\-\s\._]+'
sepp = r'_'
sept = r'[\-\s\._]{3}'
bigy = r'(?:big' + seps + r'y(?:data)?|ydna)'
rslt = r'(?:results|data|rawdata|vcfdata|raw data|csvexport|raw_data|raw|bigyrawdata)'
name = r'((?:' + nam2 + seps + '){1,3})'
kit = r'(?:(?:kit|ftdna)?[ #]?)?([enhb1-9][0-9]{3,6})'
rzip = r'zip(?:.zip)?'
snps = r'(?:[\-_]{,3}(?:r[\-_]?)?(?:cts\d{3,6}|fcg\d{4,5}|fgc\d{3,5}x?|p312|z\d{3,5}|df\d{2,3}x?|l\d{2,3}x?|u152|rs\d{4}|d27|sry\d{4}|m222|l\d{4}|s\d{4,6}|mc14|a\d{3,5}|zz\d{2}|zp\d{2}|z\d{2,3}|s\d{3}|pf\d{4}|by\d{3,5}|u106|l2|y\d{4,5}|yp\d{4,5})){1,3}'
plac = r'(?:Germany|England|UnknownOrigin|Sweden|France|United_?Kingdom|Scotland|Ireland|Netherlands|Europe|Luxembour?g|Wales|Poland|Italy|CzechRepublic|Russia|Puerto-Rico|Switzerland|Algeria|Denmark|Slovakia|US|USA)?'
name_re = [
#0 e.g. bigy-Treece-N4826.zip
(re.compile(ws + sepp.join([bigy, name, kit, rzip]),
re.I), 'name', 'kit'),
#1 e.g. bigy-N4826-Treece.zip
(re.compile(ws + sepp.join([bigy, kit, name, rzip]),
re.I), 'kit', 'name'),
#2 e.g. N4826-bigy-Treece.zip
(re.compile(ws + sepp.join([kit, bigy, name, rzip]),
re.I), 'kit', 'name'),
#3 e.g. Treece - N4826 - bigy.zip
(re.compile(ws + name + sept + kit + sept + bigy + sep + r'?\.zip',
re.I), 'name', 'kit'),
#4 e.g. Treece N4826 bigy.zip
(re.compile(ws + sepp.join([name, kit, bigy, rzip]),
re.I), 'name', 'kit'),
#5 e.g. Treece N4826 bigy results 20140808.zip
(re.compile(ws + sepp.join([name, kit, bigy, rslt, ndate, rzip]),
re.I), 'name', 'kit'),
#6 e.g. bigy-Treece-N4826-FGC1233.zip
(re.compile(ws + sepp.join([bigy, name, kit, snps, rzip]),
re.I), 'name', 'kit'),
#7 e.g. FGC1234-N4826-Treece-England-bigy-rawdata-20140708.zip
(re.compile(
ws + sepp.join([snps, kit, name, plac, bigy, rslt, ndate, rzip]),
re.I), 'kit', 'name'),
#8 e.g. FGC1234-N4826-Treece-bigy-rawdata-20140708.zip
(re.compile(ws + sepp.join([snps, kit, name, bigy, rslt, ndate, rzip]),
re.I), 'kit', 'name'),
#9 e.g. FGC1234-Treece-N4826-bigy-rawdata-20140708.zip
(re.compile(ws + sepp.join([snps, name, kit, bigy, rslt, ndate, rzip]),
re.I), 'name', 'kit'),
#10 e.g. FGC1234-N4826-Treece-bigy-rawdata.zip
(re.compile(ws + sepp.join([snps, kit, name, bigy, rslt, rzip]),
re.I), 'kit', 'name'),
#11 e.g. FGC1234-N4826-Treece-bigy-rawdata.zip
(re.compile(ws + sepp.join([snps, kit, name, bigy, rzip]),
re.I), 'kit', 'name'),
#12 e.g. FGC1234-N4826-Treece-England-bigy-rawdata.zip
(re.compile(ws + sepp.join([snps, kit, name, plac, rslt, ndate, rzip]),
re.I), 'kit', 'name'),
#13 e.g. N4826_Treece_US_BigY_RawData_2018-01-03.zip
(re.compile(
ws + sepp.join([kit, name, plac, bigy, rslt, ndate]) + '.zip',
re.I), 'kit', 'name'),
#14 e.g. FGC1234-N4826-Treece-bigy-rawdata.zip
(re.compile(ws + sepp.join([snps, kit, name, rslt, ndate, rzip]),
re.I), 'kit', 'name'),
#15 e.g. FGC1234-N4826-Treece-bigy-20140708.zip
(re.compile(ws + sepp.join([snps, kit, name, bigy, ndate, rzip]),
re.I), 'kit', 'name'),
#16 e.g. FGC1234-N4826-Treece-20140708.zip
(re.compile(ws + sepp.join([snps, kit, name, ndate, rzip]),
re.I), 'kit', 'name'),
#17 e.g. N4826-Treece-bigy-data-20140708.zip
(re.compile(ws + sepp.join([kit, name, bigy, rslt, ndate, rzip]),
re.I), 'kit', 'name'),
#18 e.g. N4826-bigy-Treece-20140708.zip
(re.compile(ws + sepp.join([kit, bigy, name, ndate, rzip]),
re.I), 'kit', 'name'),
#19 e.g. FGC1234-Treece-N4826.zip
(re.compile(ws + sepp.join([snps, name, kit, rzip]),
re.I), 'name', 'kit'),
#20 e.g. FGC1234-Treece-N4826-bigy-rawdata.zip
(re.compile(ws + sepp.join([snps, name, kit, bigy, rslt, rzip]),
re.I), 'name', 'kit'),
#21 e.g. bigy-Lindström-548872.zip
(re.compile(ws + sepp.join([bigy, cname, kit, rzip]),
re.I), 'name', 'kit'),
]
trace(2, 'File names mapped, according to which regular expression:')
# track counts - only for diagnostics
cnt = defaultdict(int)
# list of non-matching files
nomatch = []
# all of the file names we could parse
fname_dict = {}
bed_re = re.compile(r'(\b(?:\w*[^_/])?regions(?:\[\d\])?\.bed)')
vcf_re = re.compile(r'(\b(?:\w*[^_/])?variants(?:\[\d\])?\.vcf)')
zip_re = re.compile(r'(\b(?:\w*[^_/])?bigy.*\.zip)')
mgroup = []
for md, fnames in mdict.items():
if md in rename_dict:
kkit, nname = rename_dict[md]
pathname = fnames[0]
fname = os.path.split(pathname)[-1]
rule = 'dm'
else:
for pathname in fnames:
fname = os.path.split(pathname)[-1]
if fname in rename_dict:
kkit, nname = rename_dict[fname]
rule = 'df'
break
else:
kkit = nname = None
if not keepfile(md):
trace(2, '{} skipped because of subsetting'.format(fname))
continue
if kkit:
if kkit in ('None', '') and nname in ('None', ''):
kkit = nname = None
trace(
2, '{} skipped because of entry in mappings dictionary'.
format(fname))
else:
trace(
2, '{3:>2} {0:<50s} {1:<15s} {2:<10s}'.format(
fname, nname, kkit, rule))
cnt[rule] += 1
fname_dict[pathname] = kkit, nname
else:
pathname = fnames[0] # only use one of the equivalent set
if os.path.splitext(pathname)[-1].lower() != '.zip':
trace(
2,
'Found foreigner hanging out in zip directory: {0}'.format(
fname))
continue
d = {}
fname = os.path.split(pathname)[-1]
for ii, (r, k1, k2) in enumerate(name_re):
s = r.search(fname)
if s:
d[k1] = s.groups()[0]
if k2:
d[k2] = s.groups()[1]
else:
d['name'] = 'Unknown'
d['name'] = name_preference(d['name'])
try:
trace(
2, '{3:>2} {0:<50s} {1:<15s} {2:<10s}'.format(
fname, d['name'], d['kit'], ii))
cnt[ii] += 1
fname_dict[pathname] = d['kit'], d['name']
except:
trace(0, 'FAILURE on filename:', fname)
break
else:
nomatch.append(pathname)
if len(fnames) > 1:
for eq in [p for p in fnames if p != pathname]:
trace(2, ' -same: {}'.format(os.path.split(eq)[-1]))
trace(2, 'Number of filenames not matched: {0}'.format(len(nomatch)))
trace(2, 'Which expressions were matched:')
def keyfunc(v):
return '{0!s:0>2}'.format(v)
for nn in sorted(cnt, key=keyfunc):
trace(2, '{0:>2}: {1:>4}'.format(nn, cnt[nn]))
if nomatch:
trace(1, 'Files that did not match:')
for ll in nomatch:
trace(1, ll.strip())
else:
trace(1, 'All files matched a rule')
zipcount = 0
# keep track of what needs to be cleaned up
emptydirs = []
import zipfile
for fname in fname_dict:
kitnumber, kitname = fname_dict[fname]
if kitnumber == None:
continue
if keep_files:
vcffile = os.path.join(unzip_dir,
'%s-%s.vcf' % (kitname, kitnumber))
bedfile = os.path.join(unzip_dir,
'%s-%s.bed' % (kitname, kitnumber))
# no checking to see if the contents are good, but that's what was asked for
if os.path.isfile(vcffile) and os.path.isfile(bedfile):
trace(
1, '%s-%s already exists and keep flag - skipping' %
(kitname, kitnumber))
continue
try:
zf = zipfile.ZipFile(fname)
except:
trace(
0,
'WARN: not a zip file: {} from {}'.format(fname, os.getcwd()))
continue
listfiles = zf.namelist()
bedfile = vcffile = zipfname = None
for ff in listfiles:
dirname, basename = os.path.split(ff)
if bed_re.search(basename):
bedfile = ff
elif vcf_re.search(basename):
vcffile = ff
elif zip_re.search(basename):
zipfname = ff
if dirname and (dirname not in emptydirs):
emptydirs.append(dirname)
if (not bedfile) or (not vcffile):
if not zipfname:
trace(0, 'WARN: missing data in ' + fname)
continue
else:
try:
zf.extractall(unzip_dir, [
zipfname,
])
emptydirs.append(os.path.join(unzip_dir, zipfname))
bedfile = 'regions.bed'
vcffile = 'variants.vcf'
zf = zipfile.ZipFile(os.path.join(unzip_dir, zipfname))
except:
trace(0, 'WARN: missing data in ' + fname)
continue
try:
zf.extractall(unzip_dir, [bedfile, vcffile])
except RuntimeError:
trace(0, 'WARN: {} would not extract - encrypted?'.format(base))
base = '%s-%s' % (kitname, kitnumber)
fpath = os.path.join(unzip_dir, '%s')
trace(3, fpath % base)
if vars(namespace)['rename']:
try:
os.link(fname, (fpath % base) + '.zip')
trace(1, 'ln {} {}.zip'.format(fname, (fpath % base)))
except:
shutil.copy2(fname, (fpath % base) + '.zip')
trace(1, 'cp -p {} {}.zip'.format(fname, (fpath % base)))
emptydirs.append(fpath % bedfile)
emptydirs.append(fpath % vcffile)
else:
try:
os.rename(fpath % bedfile, (fpath % base) + '.bed')
os.rename(fpath % vcffile, (fpath % base) + '.vcf')
except:
trace(
0,
'WARN: could not identify VCF and/or BED file for ' + base)
zipcount += 1
trace(0, '%d new files extracted' % zipcount)
# clean up any empty dirs unzip created
if emptydirs:
trace(3, 'Trying to remove droppings:')
for dir in emptydirs:
if os.path.isfile(dir):
os.unlink(dir)
for dir in emptydirs:
if os.path.isfile(dir):
continue
try:
dp = os.path.join(unzip_dir, dir)
os.removedirs(dp)
trace(3, ' {0}'.format(dp))
except FileNotFoundError:
pass
except:
trace(3, ' W! could not remove {0}'.format(dp))
pass
# list of file names we unzipped
files = os.listdir(unzip_dir)
return files
