def test_generate_target_from_supercells(self):
"""Test generation of a structure based on a target cluster vector and a list
of supercells"""
target_cv = [1., 0., 0., -1., 0., 1., 0., 0., 0., 0., 0., 0., 0.]
target_conc = {'Au': 0.5, 'Pd': 0.5}
kwargs = dict(cluster_space=self.cs,
target_concentrations=target_conc,
target_cluster_vector=target_cv,
n_steps=500,
random_seed=42,
optimality_weight=0.3)
supercells = [self.prim.repeat((2, 2, 1)), self.prim.repeat((2, 1, 1))]
# This should be simple enough to always work
structure = generate_target_structure_from_supercells(
supercells=supercells, **kwargs)
self.assertTrue(
np.allclose(self.cs.get_cluster_vector(structure), target_cv))
# Log output to StringIO stream
for handler in logger.handlers:
handler.close()
logger.removeHandler(handler)
# Use supercells that do not fit
supercells = [self.prim.repeat((2, 2, 1)), self.prim.repeat((3, 1, 1))]
logfile = NamedTemporaryFile(mode='w+', encoding='utf-8')
set_log_config(filename=logfile.name)
structure = generate_target_structure_from_supercells(
supercells=supercells, **kwargs)
logfile.seek(0)
lines = logfile.readlines()
logfile.close()
self.assertIn('At least one supercell was not commensurate', lines[0])
self.assertTrue(
np.allclose(self.cs.get_cluster_vector(structure), target_cv))
# Use two supercells that do not fit
supercells = [self.prim.repeat((3, 3, 1)), self.prim.repeat((3, 1, 1))]
logfile = NamedTemporaryFile(mode='w+', encoding='utf-8')
set_log_config(filename=logfile.name)
with self.assertRaises(ValueError) as cm:
generate_target_structure_from_supercells(supercells=supercells,
**kwargs)
logfile.seek(0)
lines = logfile.readlines()
logfile.close()
self.assertEqual(len(lines), 1) # Warning should be issued once
self.assertIn('At least one supercell was not commensurate', lines[0])
self.assertIn('No supercells that may host the specified',
str(cm.exception))
def solve(self, cnf):
s = Solution()
infile = NamedTemporaryFile(mode='w')
outfile = NamedTemporaryFile(mode='r')
io = DimacsCnf()
infile.write(io.tostring(cnf))
infile.flush()
ret = call(self.command % (infile.name, outfile.name), shell=True)
infile.close()
if ret != 10:
return s
s.success = True
lines = outfile.readlines()[1:]
for line in lines:
varz = line.split(" ")[:-1]
for v in varz:
v = v.strip()
value = v[0] != '-'
v = v.lstrip('-')
vo = io.varobj(v)
s.varmap[vo] = value
# Close deletes the tmp files
outfile.close()
return s
def test_generate_sqs_from_supercells(self):
"""Test generation of SQS structure from list of supercells"""
target_conc = {'Au': 0.5, 'Pd': 0.5}
kwargs = dict(cluster_space=self.cs,
target_concentrations=target_conc,
n_steps=500,
random_seed=42,
optimality_weight=0.0)
supercells = [self.prim.repeat((2, 2, 1)), self.prim.repeat((2, 1, 1))]
structure = generate_sqs_from_supercells(supercells=supercells,
**kwargs)
target_cv = [1., 0., 0., -1., 0., 1., 0., 0., 0., 0., 0., 0., 0.]
self.assertTrue(
np.allclose(self.cs.get_cluster_vector(structure), target_cv))
# Log output to StringIO stream
for handler in logger.handlers:
handler.close()
logger.removeHandler(handler)
# Test with supercell that does not match
supercells = [self.prim]
logfile = NamedTemporaryFile(mode='w+', encoding='utf-8')
set_log_config(filename=logfile.name)
with self.assertRaises(ValueError) as cm:
generate_sqs_from_supercells(supercells=supercells, **kwargs)
logfile.seek(0)
lines = logfile.readlines()
logfile.close()
self.assertEqual(len(lines), 1)
self.assertIn('At least one supercell was not commensurate', lines[0])
self.assertIn('No supercells that may host the specified',
str(cm.exception))
def test_AllTypes_RandomDiam(self):
for rule_type in ['all', 'retro', 'forward']:
# for i in range(len(self.diameters)):
i = 3
diams = list(
combinations(
self.diameters,
i+1
)
)
seed(2)
sub_diams = sample(diams, 1)
for diam in sub_diams:
with self.subTest(rule_type=rule_type, diam=diam):
outfile = NamedTemporaryFile(delete=False)
parse_rules(
rules_file = 'retrorules',
outfile = outfile.name,
rule_type = rule_type,
diameters = ','.join(diam)
)
# Test if outfile has more than one single line (header)
self.assertGreater(
len(outfile.readlines()),
1
)
outfile.close()
unlink(outfile.name)
def cargo_toml_context():
tmp_file = NamedTemporaryFile(buffering=False)
with open("Cargo.toml", "rb") as f:
tmp_file.writelines(f.readlines())
cargo_file = toml.load("Cargo.toml")
cargo_file.setdefault("patch",
{}).setdefault("crates-io", {})["jsonschema"] = {
"path":
os.environ["UNRELEASED_JSONSCHEMA_PATH"],
}
with open("Cargo.toml", "w") as f:
toml.dump(cargo_file, f)
try:
print(
"Modified Cargo.toml file by patching jsonschema dependency to {}"
.format(os.environ["UNRELEASED_JSONSCHEMA_PATH"]),
file=sys.stderr,
)
yield
except:
print("Cargo.toml used during the build", file=sys.stderr)
with open("Cargo.toml", "r") as f:
print(f.read(), file=sys.stderr)
raise
finally:
with open("Cargo.toml", "wb") as f:
tmp_file.seek(0)
f.writelines(tmp_file.readlines())
def main():
parser = argparse.ArgumentParser(description="Check files.")
parser.add_argument('infile',
type=str,
nargs=1,
help='Specify the path to the file of interest.')
parser.add_argument('-out',
'--outfile',
type=str,
nargs=1,
default=[None],
help='Specify the path to the outfile.')
args = parser.parse_args()
infile = args.infile[0]
new_file = NamedTemporaryFile(delete=False)
with open(infile, 'rb') as f:
for line in f:
if not line.startswith('#'):
new_file.write(line)
for line in new_file.readlines():
if not is_number(line.rstrip().split('\t')[-1]):
print('du', line)
print('no errors')
fs = FileSort(new_file, args.outfile[0])
fs.sort()
def modify_guest(self):
# We swap out the default kickstart for one that forces mptspi
# This results in a bootable guest both during KVM customize and when
# run on vsphere
# By adding this at JEOS install time we leave open the possibility
# of modifying the module/boot environment during customization,
# for example, via the install of vmware tools.
# Use this just to get a known unique temp file name
new_kickstart = NamedTemporaryFile(delete = False)
new_kickstart_name = new_kickstart.name
new_kickstart.close()
# This step does the rootpw substitution for us
# modified file ends up in new_kickstart_name
# This is slightly naughty since we use an internal Oz function
self.guest._copy_kickstart(new_kickstart_name)
# read the now root-pw-substituted kickstart
new_kickstart = open(new_kickstart_name, "r")
ks_lines = new_kickstart.readlines()
new_kickstart.close()
# Add our device line
new_kickstart = open(new_kickstart_name, "w")
for ks_line in ks_lines:
new_kickstart.write(ks_line)
if re.match("install", ks_line):
new_kickstart.write("device scsi mptspi\n")
new_kickstart.close()
# Tell Oz that we want it to use this ks.cfg rather than the built in one
self.guest.auto = new_kickstart_name
def _convertDiff(filename, do_diff=False):
m = Mesh(3)
sf = filename[filename.rindex('.'):]
if sf == '.smesh':
sf_o = '.ply'
else:
sf_o = '.smesh'
try:
m.parse(filename)
t = NamedTemporaryFile(suffix=sf_o)
m.write(t.name)
m.parse(t.name)
t2 = NamedTemporaryFile(suffix=sf)
fn2 = m.write(t2.name)
except Exception as e:
t.seek(0)
open('/tmp/error.log', 'w').writelines(t.readlines())
print((traceback.format_exc()))
raise e
#diff = difflib.unified_diff( open( filename ).readlines(), open( fn2 ).readlines() )
if do_diff:
print('-' * 15, 'generating diff output, convertDiff')
diff = difflib.HtmlDiff().make_file(
open(filename).readlines(),
open(fn2).readlines())
open('%s.convert.diff.html' % filename, 'w').writelines(diff)
def runDCBScodeInMatlab(fastaPath, cdsCount):
fout = NamedTemporaryFile(mode='r', delete=(not debugMode))
print("Calculating DCBS...")
# /usr/local.cc/bin/matlab -nodisplay -nodesktop -nosplash -singleCompThread -nojvm -r "cds=fastaread('/var/tmp/pbs.9025895.power8.tau.ac.il/tmpIzkenm');csvwrite('/var/tmp/pbs.9025895.power8.tau.ac.il/tmpIzkenm.csv',Compute_DCBS({cds.Sequence}));quit()"
cmdline = ((
config.MatlabPath, "-nodisplay", "-nodesktop", "-nosplash",
"-singleCompThread", "-nojvm", "-r",
"cds=fastaread('{}');csvwrite('{}',Compute_DCBS({{cds.Sequence}}));quit()"
.format(fastaPath, fout.name)))
print(" ".join(cmdline))
out = subprocess.call(cmdline, shell=False)
lastVal = None
lineNum = 0
for line in fout.readlines():
lastVal = float(line)
lineNum += 1
if lineNum != cdsCount + 1:
raise Exception(
"Expected to find DCBS values for {} proteins; found {} instead".
format(cdsCount, lineNum))
return lastVal
def solve(self, cnf):
s = Solution()
infile = NamedTemporaryFile(mode='w')
outfile = NamedTemporaryFile(mode='r')
io = DimacsCnf()
infile.write(io.tostring(cnf))
infile.flush()
ret = call(self.command % (infile.name, outfile.name), shell=True)
infile.close()
if ret != 10:
return s
s.success = True
lines = outfile.readlines()[1:]
for line in lines:
varz = line.split(" ")[:-1]
for v in varz:
v = v.strip()
value = v[0] != '-'
v = v.lstrip('-')
vo = io.varobj(v)
s.varmap[vo] = value
# Close deletes the tmp files
outfile.close()
return s
def _test(*, num_lines: int, max_length: int):
in_file = NamedTemporaryFile(mode='r+b')
out_file = NamedTemporaryFile(mode='w+b')
write_lines_to_file(in_file, num_lines, max_length)
merge_sort(in_file, out_file, chunk_size=1_000)
out_file.seek(0)
src = out_file.readlines()
in_file.seek(0)
dst = sorted(in_file.readlines())
try:
assert src == dst
finally:
in_file.close()
out_file.close()
def test_newline_at_EOF():
"""http://stackoverflow.com/a/729795"""
f = NamedTemporaryFile(delete=False)
prop = Properties(OrderedDict([("a", "b"), ("c", "d"), ("e", "f")]))
prop.save(f.name)
with open(f.name) as f:
lastline = f.readlines()[-1]
os.remove(f.name)
assert lastline.endswith("\n")
def test_export_table(self):
Account.objects.create(account='TEST',
type='Asset',
last_updated=datetime.datetime.now())
f = NamedTemporaryFile()
path = f.name
f.close()
export_table(Account, path)
with open(path, 'r') as f:
lines = f.readlines()
os.remove(path)
self.assertEqual(2, len(lines))
def test_newline_at_EOF():
"""http://stackoverflow.com/a/729795"""
f = NamedTemporaryFile(delete=False)
prop = Properties(OrderedDict([
("a", "b"),
("c", "d"),
("e", "f")
]))
prop.save(f.name)
with open(f.name) as f:
lastline = f.readlines()[-1]
os.remove(f.name)
assert lastline.endswith("\n")
def call_minisat(n, clauses):
import os
dir_path = os.path.dirname(os.path.realpath(__file__))
infile = NamedTemporaryFile(mode='w')
outfile = NamedTemporaryFile(mode='r')
Dimacs.Dimacs.write(n, clauses, infile)
call("%s/minisat %s %s" % (dir_path, infile.name, outfile.name), shell=True)
infile.close()
lines = outfile.readlines()
if lines[0].strip().startswith('SAT'):
return [int(x) for x in lines[1].strip().split()]
else:
return "UNSAT"
def test_get_skll_logger_with_warning():
reset()
temp_file = NamedTemporaryFile("w", delete=False)
temp_file.close()
TEMP_FILES.append(temp_file)
TEMP_FILE_PATHS.append(temp_file.name)
logger = get_skll_logger("test_get_skll_logger_with_warning",
temp_file.name)
LOGGERS.append(logger)
# Send a regular log message
msg1 = "message 1"
logger.info(msg1)
# Trigger an ``sklearn`` warning
trigger_sklearn_warning()
# Send a regular log message
msg2 = "message 2"
logger.info(msg2)
with open(temp_file.name) as temp_file:
log_lines = temp_file.readlines()
assert log_lines[0].endswith("INFO - {}\n".format(msg1))
sklearn_warning_re = \
re.compile(r"WARNING - [^\n]+sklearn.metrics.ranking.py:\d+: "
r"UndefinedMetricWarning:No negative samples in "
r"y_true, false positive value should be "
r"meaningless")
assert sklearn_warning_re.search("".join(log_lines[1]))
assert log_lines[-1].endswith("INFO - {}\n".format(msg2))
# Now make sure that warnings.showwarning works the way
# it normally works (writes to STDERR) by issuing a warning,
# capturing it, and, finally, making sure the expected
# warning shows up correctly in the STDERR stream and,
# additionally, not in the log file.
old_stderr = sys.stderr
try:
msg3 = "message 3"
sys.stderr = mystderr = StringIO()
warnings.warn(msg3)
err = mystderr.getvalue()
assert "UserWarning: {}".format(msg3) in err
with open(temp_file.name) as log_file:
assert "UserWarning:{}".format(msg3) not in log_file.read()
finally:
sys.stderr = old_stderr
close_and_remove_logger_handlers(logger)
def patch(binary, addForSub, subForAdd):
patchBytes = NamedTemporaryFile()
r = r2pipe.open(binary)
# Disassemble binary
r.cmd('aa')
if (addForSub):
# Find all 4-byte 'add rsp, const8' instructions
r.cmd('/c add rsp | grep " # 4:" > ' + patchBytes.name)
if (subForAdd):
# Find all 4-byte 'sub rsp, const8' instructions
r.cmd('/c sub rsp | grep " # 4:" >> ' + patchBytes.name)
count = 0
with open(binary, 'r+b') as binfile:
for line in patchBytes.readlines():
# Address to patch
patchAddr = int(line.split()[0], 16)
# Seek to byte that must be inverted
binfile.seek(patchAddr + 3)
originalConst = ord(binfile.read(1))
newConst = (~originalConst + 1) & 0xff
# Seek back to byte to 'add'/'sub' opcode
binfile.seek(patchAddr+2)
opcode = ord(binfile.read(1))
# 'add rsp' -> 'sub rsp'
if (addForSub and opcode == 0xc4):
binfile.seek(patchAddr+2)
# 'sub rsp'
binfile.write('\xec')
# Write inverted byte
binfile.write(chr(newConst))
count += 1
# 'sub rsp' -> 'add rsp'
if (subForAdd and opcode == 0xec):
binfile.seek(patchAddr+2)
# 'add rsp'
binfile.write('\xc4')
# Write inverted byte
binfile.write(chr(newConst))
count += 1
filename = os.path.basename(binary)
print("Patched %d instructions in: %s" %(count, filename))
def test_startup_script(self):
self.command.contact_group_info = [{
'account': '*****@*****.**',
'key': 'conv1',
'contacts_csv': 'contacts.csv',
}]
self.command.conversation_info = [{
'account': '*****@*****.**',
'key': 'conv1',
'start': True,
}, {
'account': '*****@*****.**',
'key': 'conv2',
'start': False,
}]
self.command.router_info = [{
'account': '*****@*****.**',
'key': 'router1',
'start': False,
}, {
'account': '*****@*****.**',
'key': 'router2',
'start': True,
}]
startup_tmpfile = NamedTemporaryFile()
self.command.mk_filename = lambda fn, s: startup_tmpfile.name
self.command.write_startup_script()
startup_tmpfile.flush()
lines = [
l.strip('\n') for l in startup_tmpfile.readlines()[3:]
if l.strip() != ''
]
self.assertEqual(
lines,
[
'#!/bin/bash',
'./go-admin.sh go_import_contacts ' # cont.
'--email-address [email protected] \\',
' --contacts contacts.csv --group conv1',
'echo "Starting conversation: conv1"',
'./go-admin.sh go_manage_conversation ' # cont.
'--email-address [email protected] \\',
' --conversation-key conv1 --start',
'echo "Starting router: router2"',
'./go-admin.sh go_manage_router ' # cont.
'--email-address [email protected] \\',
' --router-key router2 --start',
])
def test_BadDiametersArgument(self):
for diam in ['3']:
with self.subTest(diam=diam):
outfile = NamedTemporaryFile(delete=False)
parse_rules(
rules_file=self.rules_file,
rule_type='retro',
diameters=diam,
outfile=outfile.name
)
# Test if outfile has one single line (header)
self.assertEqual(len(outfile.readlines()), 1)
outfile.close()
unlink(outfile.name)
def test_startup_script(self):
self.command.contact_group_info = [{
'account': '*****@*****.**',
'key': 'conv1',
'contacts_csv': 'contacts.csv',
}]
self.command.conversation_info = [{
'account': '*****@*****.**',
'key': 'conv1',
'start': True,
}, {
'account': '*****@*****.**',
'key': 'conv2',
'start': False,
}]
self.command.router_info = [{
'account': '*****@*****.**',
'key': 'router1',
'start': False,
}, {
'account': '*****@*****.**',
'key': 'router2',
'start': True,
}]
startup_tmpfile = NamedTemporaryFile()
self.command.mk_filename = lambda fn, s: startup_tmpfile.name
self.command.write_startup_script()
startup_tmpfile.flush()
lines = [l.strip('\n') for l in startup_tmpfile.readlines()[3:]
if l.strip() != '']
self.assertEqual(lines, [
'#!/bin/bash',
'./go-admin.sh go_import_contacts ' # cont.
'--email-address [email protected] \\',
' --contacts contacts.csv --group conv1',
'echo "Starting conversation: conv1"',
'./go-admin.sh go_manage_conversation ' # cont.
'--email-address [email protected] \\',
' --conversation-key conv1 --start',
'echo "Starting router: router2"',
'./go-admin.sh go_manage_router ' # cont.
'--email-address [email protected] \\',
' --router-key router2 --start',
])
def main():
parser = argparse.ArgumentParser(description="Check files.")
parser.add_argument('infile', type=str, nargs=1, help='Specify the path to the file of interest.')
parser.add_argument('-out', '--outfile', type=str, nargs=1, default=[None], help='Specify the path to the outfile.')
args = parser.parse_args()
infile = args.infile[0]
new_file = NamedTemporaryFile(delete=False)
with open(infile, 'rb') as f:
for line in f:
if not line.startswith('#'):
new_file.write(line)
for line in new_file.readlines():
if not is_number(line.rstrip().split('\t')[-1]):
print('du', line)
print('no errors')
fs = FileSort(new_file, args.outfile[0])
fs.sort()
def solve(self,
cnf,
variable=None,
value=True,
translator=AIMA_to_Dimacs_Translator):
# if there are no clauses, then can't infer anything, so by default query result is unknown
# return Solution with success == None
# Note that this could be treated the same as failure.
# In PropKB_SAT.ask, this is OK as it will test if sT.success == sF.success
# and therefore will also return None
if not cnf: return Solution(None)
s = Solution()
infile = NamedTemporaryFile(mode='w')
outfile = NamedTemporaryFile(mode='r')
io = translator()
if variable:
dimacs = io.to_dimacs_string_set_variable_value(
cnf, variable, value)
if dimacs:
infile.write(dimacs)
else:
return s
else:
infile.write(io.to_dimacs_string(cnf))
infile.flush()
ret = call(self.command % (infile.name, outfile.name), shell=True)
infile.close()
if ret != 10:
return s
s.success = True
lines = outfile.readlines()[1:]
for line in lines:
varz = line.split(' ')[:-1]
for v in varz:
v = v.strip()
value = v[0] != '-'
v = v.lstrip('-')
vo = io.varobj(v)
s.varmap[vo] = value
outfile.close()
return s
def test_convert_excel_to_datafile(self):
read_strategy = ReadExcel()
write_strategy = WriteDatafile()
context = Context(read_strategy, write_strategy)
tmpfile = NamedTemporaryFile()
from_path = os.path.join("tests", "fixtures", "combined_inputs.xlsx")
context.convert(from_path, tmpfile.name)
tmpfile.seek(0)
actual = tmpfile.readlines()
tmpfile.close()
assert actual[-1] == b"end;\n"
assert actual[0] == b"# Model file written by *otoole*\n"
assert actual[2] == b"09_ROK d_bld_2_coal_products 2017 20.8921\n"
assert actual[8996] == b"param default 1 : DepreciationMethod :=\n"
def _convertDiff(filename,do_diff=False):
m = Mesh(3)
sf = filename[filename.rindex('.'):]
if sf == '.smesh':
sf_o = '.ply'
else:
sf_o = '.smesh'
try:
m.parse( filename )
t = NamedTemporaryFile(suffix=sf_o)
m.write( t.name )
m.parse( t.name )
t2 = NamedTemporaryFile(suffix=sf)
fn2 = m.write( t2.name )
except Exception, e:
t.seek(0)
open( '/tmp/error.log', 'w').writelines( t.readlines() )
print traceback.format_exc()
raise e
def test_multiline_value_unicode_to_file(self):
disk_file = NamedTemporaryFile(mode='ab+')
handler = FileHandler(disk_file.name)
handler.setFormatter(ShoehornFormatter())
self.logger.addHandler(handler)
try:
1 / 0
except:
get_logger().exception('bad', short='x', diff=u'foo\n\U0001F4A9')
disk_file.seek(0)
compare(disk_file.readlines()[:5],
expected=[
b"bad short='x'\n",
b'diff:\n',
b'foo\n',
b'\xf0\x9f\x92\xa9\n',
b'Traceback (most recent call last):\n',
])
def process(infile):
fin = open(infile, "r")
cmd = []
for line in fin:
cmd.append(line.split('#')[0].strip())
a = NamedTemporaryFile()
foutName = a.name
a.close()
cmd.append("ELn %s.xml" % foutName)
ai = athenaObj.Interpreter('cgi')
for c in cmd:
ai.cmd(c)
a = open(foutName + ".sco", "r")
for line in a.readlines():
print(line)
def process(infile):
fin = open(infile, "r")
cmd = []
for line in fin:
cmd.append(line.split('#')[0].strip())
a = NamedTemporaryFile()
foutName = a.name
a.close()
cmd.append("ELn %s.xml"%foutName)
ai = athenaObj.Interpreter('cgi')
for c in cmd:
ai.cmd(c)
a = open(foutName + ".sco", "r")
for line in a.readlines():
print line
def load_target_types(ccdb_context, run_number):
mapping = {}
# make temp file to store CCDB info in
f = NamedTemporaryFile()
ccdb_context.process_command_line("dump /TARGET/target_type_list:" + str(run_number) + " > " + f.name)
# read in info
f.flush()
lines = f.readlines()
if len(lines) < 2:
print "Problem writing out CCDB table: /TARGET/target_type_list"
else:
# skip the first line, which is junk
for x in range(1, len(lines)):
vals = lines[x].split()
# index = int(vals[0])
mapping[int(vals[0])] = " ".join(vals[1:])
return mapping
def solve(self, cnf, variable = None, value = True,
translator = AIMA_to_Dimacs_Translator):
# if there are no clauses, then can't infer anything, so by default query result is unknown
# return Solution with success == None
# Note that this could be treated the same as failure.
# In PropKB_SAT.ask, this is OK as it will test if sT.success == sF.success
# and therefore will also return None
if not cnf: return Solution(None)
s = Solution()
infile = NamedTemporaryFile(mode='w')
outfile = NamedTemporaryFile(mode='r')
io = translator()
if variable:
dimacs = io.to_dimacs_string_set_variable_value(cnf, variable, value)
if dimacs:
infile.write(dimacs)
else:
return s
else:
infile.write(io.to_dimacs_string(cnf))
infile.flush()
ret = call(self.command % (infile.name, outfile.name), shell=True)
infile.close()
if ret != 10:
return s
s.success = True
lines = outfile.readlines()[1:]
for line in lines:
varz = line.split(' ')[:-1]
for v in varz:
v = v.strip()
value = v[0] != '-'
v = v.lstrip('-')
vo = io.varobj(v)
s.varmap[vo] = value
outfile.close()
return s
def load_target_types(ccdb_context, run_number):
mapping = {}
# make temp file to store CCDB info in
f = NamedTemporaryFile()
ccdb_context.process_command_line("dump /TARGET/target_type_list:" +
str(run_number) + " > " + f.name)
# read in info
f.flush()
lines = f.readlines()
if len(lines) < 2:
print "Problem writing out CCDB table: /TARGET/target_type_list"
else:
# skip the first line, which is junk
for x in range(1, len(lines)):
vals = lines[x].split()
#index = int(vals[0])
mapping[int(vals[0])] = " ".join(vals[1:])
return mapping
class TmpFile:
mode = dict(mode='rt', encoding='utf-8')
def __init__(self):
pass
def __exit__(self, type, value, tb):
self.f.close()
os.remove(self.f.name)
def __enter__(self):
self.f = NamedTemporaryFile(delete=False, **TmpFile.mode)
return self
def borrow_name(self, proc):
self.f.close()
proc(self.f.name)
self.f = open(self.f.name, **TmpFile.mode)
return self
def readlines(self):
return self.f.readlines()
def test_mapping(structure,
reference=None,
expected_drmax=0.276249887,
expected_dravg=0.139034051,
expected_chemical_formula='H3Au6Pd2X5',
**kwargs):
"""
Convenience wrapper for testing mapping.
"""
if reference is None:
reference = self.reference
logfile = NamedTemporaryFile(mode='w+', encoding='utf-8')
set_log_config(filename=logfile.name)
mapped, info = map_structure_to_reference(structure, reference,
**kwargs)
self.assertEqual(len(info), 6)
self.assertAlmostEqual(info['drmax'], expected_drmax)
self.assertAlmostEqual(info['dravg'], expected_dravg)
self.assertEqual(mapped.get_chemical_formula(),
expected_chemical_formula)
logfile.seek(0)
lines = logfile.readlines()
logfile.close()
return lines, info
def createEncPrimeReport(fastaPath, geneticCode):
fout = NamedTemporaryFile(mode='r', delete=(not debugMode))
print("Creating codon counts...")
#SeqCount -c ExSeqs.fasta 9
print((ENCprimeMain,
"{}.codcnt".format(fastaPath), "{}.acgtfreq".format(fastaPath),
str(geneticCode), fout.name, "0", "-q"))
out = subprocess.check_output((ENCprimeMain, "{}.codcnt".format(fastaPath),
"{}.acgtfreq".format(fastaPath),
str(geneticCode), fout.name, "0", "-q"),
shell=False)
print(out)
for line in fout.readlines():
print("= {}".format(line))
if line.find("Totals: ") == 0:
fields = line.split(" ")
ENc = float(fields[1])
ENc_prime = float(fields[2])
print("ENc: {}\tENc': {}".format(ENc, ENc_prime))
return (ENc, ENc_prime)
def _convertDiff(filename,do_diff=False):
m = Mesh(3)
sf = filename[filename.rindex('.'):]
if sf == '.smesh':
sf_o = '.ply'
else:
sf_o = '.smesh'
try:
m.parse( filename )
t = NamedTemporaryFile(suffix=sf_o)
m.write( t.name )
m.parse( t.name )
t2 = NamedTemporaryFile(suffix=sf)
fn2 = m.write( t2.name )
except Exception as e:
t.seek(0)
open( '/tmp/error.log', 'w').writelines( t.readlines() )
print((traceback.format_exc()))
raise e
#diff = difflib.unified_diff( open( filename ).readlines(), open( fn2 ).readlines() )
if do_diff:
print('-'*15 , 'generating diff output, convertDiff')
diff = difflib.HtmlDiff().make_file( open( filename ).readlines(), open( fn2 ).readlines() )
open( '%s.convert.diff.html'%filename ,'w' ).writelines(diff)
def test_get_skll_logger():
reset()
temp_file = NamedTemporaryFile("w", delete=False)
temp_file.close()
TEMP_FILES.append(temp_file)
TEMP_FILE_PATHS.append(temp_file.name)
logger = get_skll_logger("test_get_skll_logger", temp_file.name)
LOGGERS.append(logger)
# Send a regular log message
msg1 = "message 1"
logger.info(msg1)
# Send a regular log message
msg2 = "message 2"
logger.info(msg2)
with open(temp_file.name) as temp_file:
log_lines = temp_file.readlines()
assert log_lines[0].endswith("INFO - {}\n".format(msg1))
assert log_lines[1].endswith("INFO - {}\n".format(msg2))
close_and_remove_logger_handlers(logger)
class _FileLogger:
"""File logging class wrapper.
Class wrapping is needed manly for safety of log file removal
after Blender is shut down.
Registering fuction for atexit module makes sure than,
file is deleted if Blender is closed normally.
However file is not deleted if process is killed in Linux.
On Windows, on the other hand, file gets deleted even if Blender
is closed from Task Manager -> End Task/Process
"""
__log_file = None
def __init__(self):
self.__log_file = NamedTemporaryFile(mode="w+", suffix=".log.txt", delete=True)
# instead of destructor we are using delete method,
# to close and consequentially delete log file
atexit.register(self.delete)
def delete(self):
"""Closes file and consiquentally deletes it as log file was created in that fashion.
"""
# close file only if it's still exists in class variable
if self.__log_file is not None:
self.__log_file.close()
self.__log_file = None
def write(self, msg_object):
"""Writes message to the log file.
:param msg_object: message to be written to file
:type msg_object: object
"""
self.__log_file.write(msg_object)
def flush(self):
"""Flushes written content to file on disk."""
self.__log_file.flush()
def get_log(self):
"""Gets current content of temporary SCS BT log file,
which was created at startup and is having log of BT session.
:return: current content of log file as string
:rtype: str
"""
# firstly move to start of the file
self.__log_file.seek(0)
log = ""
for line in self.__log_file.readlines():
log += line.replace("\t ", "\t\t ") # replace for Blender text editor to be aligned the same as in console
return log
def main(argv):
# configuration vars
RUN_PERIOD = "RunPeriod-2014-10"
CONDITION_FILE_FORMAT = "/work/halld/online_monitoring/conditions/run_conditions%06d.dat"
# assume that the files are loaded on the cache disk
RAWDATA_DIR_FORMAT = "/cache/mss/halld/RunPeriod-2014-10/rawdata/Run%06d"
# read in run number from command line
try:
run_number = int(argv[0])
except:
print "Need to pass the run number to process as a command line argument!"
return
run_properties = init_property_mapping()
run_conditions = parse_condition_file(CONDITION_FILE_FORMAT % (run_number))
if run_conditions is None:
return
# start extracting saved EPICS values
# run_number = run_conditions['RUN'] ## check this?
run_properties["beam_current"] = run_conditions["IBCAD00CRCUR6"]
run_properties["start_time"] = run_conditions["TIME"]
run_properties["solenoid_current"] = run_conditions["HallD-PXI:Data:I_Shunt"]
# figure out which radiator was used
# save luminosity factor = current * radiator thickness
amorphous_radiator_position = float(run_conditions["hd:radiator:motor.RBV"])
if fabs(amorphous_radiator_position - 135.948) < RADIATOR_TOLERANCE:
run_properties["radiator_type"] = "2x10-5 RL"
run_properties["luminosity"] = 1.7e-5 * float(run_properties["beam_current"])
elif fabs(amorphous_radiator_position - 166.095) < RADIATOR_TOLERANCE:
run_properties["radiator_type"] = "1x10-4 RL"
run_properties["luminosity"] = 11.2e-5 * float(run_properties["beam_current"])
elif fabs(amorphous_radiator_position - 196.262) < RADIATOR_TOLERANCE:
run_properties["radiator_type"] = "3x10-4 RL"
run_properties["luminosity"] = 22.7e-5 * float(run_properties["beam_current"])
else:
run_properties["radiator_type"] = "None"
# run_properties['luminosity'] = run_properties['beam_current']
run_properties["luminosity"] = 0.0
# parse EVIO files to extract useful information
# eventually the DAQ will report this properly?
rawdata_evio_dir = RAWDATA_DIR_FORMAT % (run_number)
if os.path.isdir(rawdata_evio_dir):
filelist = [
join(rawdata_evio_dir, f)
for f in listdir(rawdata_evio_dir)
if ((f[:10] == "hd_rawdata" or f[:6] == "hd_raw") and (f[-5:] == ".evio"))
]
filelist.sort()
file_properties = ParseEVIOFiles(filelist)
if len(file_properties) > 0:
run_properties["num_events"] = file_properties["num_events"]
run_properties["num_files"] = file_properties["num_files"]
run_properties["start_time"] = file_properties["start_time"]
run_properties["end_time"] = file_properties["end_time"]
# pull out target information from the CCDB
# load CCDB connection
ccdb_context = InitCCDB()
# read target index -> name mapping definition in from the CCDB
target_types = load_target_types(ccdb_context, run_number)
# make temp file to store CCDB info in
fconst = NamedTemporaryFile()
ccdb_context.process_command_line("dump /TARGET/target_parms:" + str(run_number) + " > " + fconst.name)
# read in info
fconst.flush()
const_lines = fconst.readlines()
if len(const_lines) < 2:
print "Problem writing out CCDB constants to file!"
else:
# the first line of the output file from CCDB is junk, and our numbers are on the second line
vals = const_lines[1].split()
target_index = int(vals[0])
if target_index in target_types:
run_properties["target_type"] = target_types[target_index]
else:
print "Invalid target index from CCDB = " + str(target_index)
fconst.close()
if VERBOSE:
print "RUN PROPERTIES FOR RUN " + str(run_number)
print str(run_properties)
# Add information to DB
## initialize DB
db = datamon_db()
## add blank run to DB if it doesn't exist
if db.GetRunID(run_number) < 0:
db.CreateRun(run_number)
db.UpdateRunInfo(run_number, run_properties)
class BasicTranscludeTests(unittest.TestCase):
def setUp(self):
"""Create tempfile as target, add cleanup methods to close and unlink tempfiles."""
self.target = NamedTemporaryFile(delete=False)
self.addCleanup(self.target.close)
self.addCleanup(os.unlink, self.target.name)
def compare_results(self, correct_path):
"""Compare the actual result with the correct result."""
with file(correct_path, 'r+') as correct:
c = correct.readlines()
self.target.seek(0)
t = self.target.readlines()
self.assertEqual(c, t)
def test_no_transclusion(self):
"""Transcluding a file without transclude directive returns the original file."""
transclude_file(make_path("simple-test-result.md"), self.target, 'md')
self.compare_results(make_path("simple-test-result.md"))
def test_simple_transclude(self):
"""Transclude replaces directive {{some_other_file.txt}} with contents of some_other_file.txt."""
"""transclude looks for files in parent folder of source"""
transclude_file(make_path("simple-transclusion.md"), self.target, 'md')
self.compare_results(make_path("simple-test-result.md"))
def test_recursive_transclude(self):
"""Transclude is recursive."""
transclude_file(
make_path("recursive-transclusion.md"), self.target, 'md')
self.compare_results(make_path("simple-test-result.md"))
def test_two_transclusions_in_one_line(self):
"""Two transclusion directives in one file are handled correctly."""
transclude_file(make_path("double-transclusion.md"), self.target, 'md')
self.compare_results(make_path("simple-test-result.md"))
def test_wildcard_transclusion(self):
"""Wildcard transclusion {{foo.*}} wildcard is set according to type (tex, html, )"""
transclude_file(
make_path("wildcard-transclusion.md"), self.target, 'html')
self.compare_results(make_path("simple-test-result.md"))
def test_missing_file_raises_error(self):
"""transclude outputs an error when a file to transclude is not found."""
self.assertRaises(MissingFileException,
transclude_file,
make_path("missing-transclusion.md"),
self.target,
'md')
def test_transclude_base(self):
"""If metadata "Transclude Base" is set, transclude looks there for files."""
"""metadata "Transclude Base" is only evaluated in the first file."""
transclude_file(
make_path("new-transclude-base.md"), self.target, 'html')
self.compare_results(make_path("new-transclude-base-result.md"))
def test_recursion_nested_folders(self):
"""Transclude ignores metadata in transculded file."""
"""with recursion, transclude looks for files relative to the file which transludes them."""
"""after recursion, transclude looks for files again relative to source."""
"""metadata of recursed files is ignored in result."""
"""metadata of source file is included in result"""
# TODO: split into smaller tests
transclude_file(
make_path("recursive-subfolder.md"), self.target, 'html')
self.compare_results(make_path("recursive-subfolder-result.md"))
class FileStore(object):
def __init__(self, proxy, processId, mode= 'r'):
'''
Gives r+w access to file on static instance via local tempfile.NamedTemproraryFile
'''
self.proxy= proxy
self.processId= processId
self.mode= mode
self.tmpFile= NamedTemporaryFile(mode= 'r+', prefix= self.processId, delete= True)
if 'r' in mode:
self.__get__()
def __get__(self):
'''
Get contents of static instance file and save to local temp file
'''
data= self.proxy.getFileContents(self.processId)
self.tmpFile.write(data.tostring())
self.tmpFile.seek(0)
def __post__(self):
'''
Posts contents of local temp file to static instance file
'''
self.tmpFile.seek(0)
data= self.tmpFile.read()
self.proxy.setFileContents(self.processId, data)
self.tmpFile.seek(0)
def getName(self):
return self.processId
def getLocalName(self):
return self.tmpFile.name
def write(self, data):
'''
Writes data to local tempfile
'''
if 'w' not in self.mode:
raise Exception('file open for read only')
self.tmpFile.write(dumps(data))
def read(self, size= -1):
'''
Reads data from local tempfile. See file read() for more details.
'''
if 'r' not in self.mode:
raise Exception('file open for write only')
return loads(self.tmpFile.read(size))
def readlines(self):
'''
Reads lines from local tempfile. See file readlines() for more detals.
'''
if 'r' not in self.mode:
raise Exception('file open for write only')
return loads(self.tmpFile.readlines())
def readline(self):
'''
Reads line from local tempfile. See file readline() for more details.
'''
if 'r' not in self.mode:
raise Exception('file open for write only')
return loads(self.tmpFile.readline())
def close(self, delete= False):
'''
Saves the contents of the local tempfile and then closes/destroys the local tempfile. See self.__post__() and python tempfile for more details.
'''
if 'w' in self.mode:
self.__post__()
elif 'r' in self.mode:
# if delete requested -- remove file form static instance
if delete:
self.proxy.deleteFile(self.processId)
self.tmpFile.close()
class History(TestCase):
def setUp(self):
from kamaki.cli.history import History as HClass
self.HCLASS = HClass
self.file = NamedTemporaryFile()
def tearDown(self):
self.file.close()
def test__match(self):
self.assertRaises(AttributeError, self.HCLASS._match, 'ok', 42)
self.assertRaises(TypeError, self.HCLASS._match, 2.71, 'ok')
for args, expected in (
(('XXX', None), True),
((None, None), True),
(('this line has some terms', 'some terms'), True),
(('this line has some terms', 'some bad terms'), False),
(('small line', 'not so small line terms'), False),
((['line', 'with', 'some', 'terms'], 'some terms'), True),
((['line', 'with', 'some terms'], 'some terms'), False)):
self.assertEqual(self.HCLASS._match(*args), expected)
def test_add(self):
history = self.HCLASS(self.file.name)
some_strings = ('a brick', 'two bricks', 'another brick', 'A wall!')
for i, line in enumerate(some_strings):
history.add(line)
self.file.seek(0)
self.assertEqual(
self.file.read(), '\n'.join(some_strings[:(i + 1)]) + '\n')
def test_empty(self):
content = 'a brick\ntwo bricks\nanother brick\nA wall!\n'
self.file.write(content)
self.file.flush()
self.file.seek(0)
self.assertEqual(self.file.read(), content)
history = self.HCLASS(self.file.name)
history.empty()
self.file.seek(0)
self.assertEqual(self.file.read(), '0\n')
def test_retrieve(self):
sample_history = (
'0\n',
'kamaki history show\n',
'kamaki file list\n',
'kamaki file create /pithos/f1\n',
'kamaki file info /pithos/f1\n',
'last command is always excluded')
self.file.write(''.join(sample_history))
self.file.flush()
history = self.HCLASS(self.file.name)
self.assertRaises(ValueError, history.retrieve, 'must be number')
self.assertRaises(TypeError, history.retrieve, [1, 2, 3])
for i in (0, len(sample_history) + 1, - len(sample_history) - 1):
self.assertEqual(history.retrieve(i), None)
for i in range(1, len(sample_history)):
self.assertEqual(history.retrieve(i), sample_history[i])
self.assertEqual(history.retrieve(- i), sample_history[- i])
def test_limit(self):
sample_history = (
'0\n',
'kamaki history show\n',
'kamaki file list\n',
'kamaki file create /pithos/f1\n',
'kamaki file info /pithos/f1\n',
'last command is always excluded')
sample_len = len(sample_history)
self.file.write(''.join(sample_history))
self.file.flush()
history = self.HCLASS(self.file.name)
for value, exp_e in (
(-2, ValueError),
('non int', ValueError),
(None, TypeError)):
try:
history.limit = value
except Exception as e:
self.assertTrue(isinstance(e, exp_e))
history.limit = 10
self.assertEqual(history.limit, 10)
self.file.seek(0)
self.assertEqual(len(self.file.readlines()), sample_len)
history.limit = sample_len - 1
self.assertEqual(history.limit, sample_len - 1)
self.file.seek(0)
self.assertEqual(len(self.file.readlines()), sample_len)
class _FileLogger:
"""File logging class wrapper.
Class wrapping is needed manly for safety of log file removal
after Blender is shut down.
Registering fuction for atexit module makes sure than,
file is deleted if Blender is closed normally.
However file is not deleted if process is killed in Linux.
On Windows, on the other hand, file gets deleted even if Blender
is closed from Task Manager -> End Task/Process
"""
__log_file = None
def __init__(self):
self.__log_file = NamedTemporaryFile(mode="w+",
suffix=".log.txt",
delete=True)
# instead of destructor we are using delete method,
# to close and consequentially delete log file
atexit.register(self.delete)
def delete(self):
"""Closes file and consiquentally deletes it as log file was created in that fashion.
"""
# close file only if it's still exists in class variable
if self.__log_file is not None:
self.__log_file.close()
self.__log_file = None
def write(self, msg_object):
"""Writes message to the log file.
:param msg_object: message to be written to file
:type msg_object: object
"""
self.__log_file.write(msg_object)
def flush(self):
"""Flushes written content to file on disk."""
self.__log_file.flush()
def get_log(self):
"""Gets current content of temporary SCS BT log file,
which was created at startup and is having log of BT session.
:return: current content of log file as string
:rtype: str
"""
# firstly move to start of the file
self.__log_file.seek(0)
log = ""
for line in self.__log_file.readlines():
log += line.replace(
"\t ", "\t\t "
) # replace for Blender text editor to be aligned the same as in console
return log
def do_align(f_name, rev_f_name, seed=None):
print('Reading source/target sentences from %s...' %
f_name,
file=sys.stderr, flush=True)
with open(f_name, 'r', encoding='utf-8') as f:
src_sents_text = []
trg_sents_text = []
for i, line in enumerate(f):
fields = line.strip().split(' ||| ')
if len(fields) != 2:
print('ERROR: line %d of %s does not contain a single |||'
' separator, or sentence(s) are empty!' % (
i+1, args.joint_filename),
file=sys.stderr, flush=True)
sys.exit(1)
src_sents_text.append(fields[0])
trg_sents_text.append(fields[1])
src_text = '\n'.join(src_sents_text) + '\n'
trg_text = '\n'.join(trg_sents_text) + '\n'
src_sents_text = None
trg_sents_text = None
source_prefix_len = 0
source_suffix_len = 0
target_prefix_len = 0
target_suffix_len = 0
with io.StringIO(src_text) as f:
src_sents, src_index = read_text(
f, True, source_prefix_len, source_suffix_len)
n_src_sents = len(src_sents)
src_voc_size = len(src_index)
srcf = NamedTemporaryFile('wb')
write_text(srcf, tuple(src_sents), src_voc_size)
src_sents = None
src_text = None
with io.StringIO(trg_text) as f:
trg_sents, trg_index = read_text(
f, True, target_prefix_len, target_suffix_len)
trg_voc_size = len(trg_index)
n_trg_sents = len(trg_sents)
trgf = NamedTemporaryFile('wb')
write_text(trgf, tuple(trg_sents), trg_voc_size)
trg_sents = None
trg_text = None
"""
print("source")
with open(f_name, 'r', encoding='utf-8') as f:
src_sents, src_index = read_text(
f, True, 0, 0)
n_src_sents = len(src_sents)
src_voc_size = len(src_index)
srcf = NamedTemporaryFile('wb')
write_text(srcf, tuple(src_sents), src_voc_size)
src_sents = None
print("target")
with open(rev_f_name, 'r', encoding='utf-8') as f:
trg_sents, trg_index = read_text(
f, True, 0, 0)
trg_voc_size = len(trg_index)
n_trg_sents = len(trg_sents)
trgf = NamedTemporaryFile('wb')
write_text(trgf, tuple(trg_sents), trg_voc_size)
trg_sents = None
"""
fwd_links_file = NamedTemporaryFile('r+')
rev_links_file = NamedTemporaryFile('r+')
stat_file = NamedTemporaryFile('r+')
print("start align")
align(srcf.name, trgf.name, statistics_filename = stat_file.name, quiet=False, links_filename_fwd=fwd_links_file.name, links_filename_rev=rev_links_file.name)
# Not using stat_file at the moment
result = fwd_links_file.readlines()
rev_result = rev_links_file.readlines()
fwd_links_file.close()
rev_links_file.close()
stat_file.close()
srcf.close()
trgf.close()
"""
if discretize:
ibm_print(aaa, reverse, output.fileno())
else: # Not used at the moment, but keeping this for the future
with open(output_prob, 'wb') as f:
pickle.dump(aaa, f, -1)
output.seek(0)
result = []
for line in output:
result.append(line.decode('ascii').strip())
"""
return result, rev_result
def test_memory(self):
"""
Memory test
"""
def cleanup(supress=False):
""" cleanup """
logging.debug("test_memory: Cleanup")
err = ""
if item.rm_cgroup(pwd):
err += "\nCan't remove cgroup directory"
utils.system("swapon -a")
if err:
if supress:
logging.warn("Some parts of cleanup failed%s", err)
else:
raise error.TestFail("Some parts of cleanup failed%s" %
err)
# Preparation
item = Cgroup('memory', self._client)
item.initialize(self.modules)
item.smoke_test()
pwd = item.mk_cgroup()
logging.debug("test_memory: Memory filling test")
meminfo = open('/proc/meminfo', 'r')
mem = meminfo.readline()
while not mem.startswith("MemFree"):
mem = meminfo.readline()
# Use only 1G or max of the free memory
mem = min(int(mem.split()[1]) / 1024, 1024)
mem = max(mem, 100) # at least 100M
try:
item.get_property("memory.memsw.limit_in_bytes")
except error.TestError:
# Doesn't support memsw limitation -> disabling
logging.info("System does not support 'memsw'")
utils.system("swapoff -a")
memsw = False
else:
# Supports memsw
memsw = True
# Clear swap
utils.system("swapoff -a")
utils.system("swapon -a")
meminfo.seek(0)
swap = meminfo.readline()
while not swap.startswith("SwapTotal"):
swap = meminfo.readline()
swap = int(swap.split()[1]) / 1024
if swap < mem / 2:
logging.error("Not enough swap memory to test 'memsw'")
memsw = False
meminfo.close()
outf = NamedTemporaryFile('w+', prefix="cgroup_client-", dir="/tmp")
logging.debug("test_memory: Initializition passed")
################################################
# Fill the memory without cgroup limitation
# Should pass
################################################
logging.debug("test_memory: Memfill WO cgroup")
ps = item.test("memfill %d %s" % (mem, outf.name))
ps.stdin.write('\n')
i = 0
while ps.poll() == None:
if i > 60:
break
i += 1
time.sleep(1)
if i > 60:
ps.terminate()
raise error.TestFail("Memory filling failed (WO cgroup)")
outf.seek(0)
outf.flush()
out = outf.readlines()
if (len(out) < 2) or (ps.poll() != 0):
raise error.TestFail("Process failed (WO cgroup); output:\n%s"
"\nReturn: %d" % (out, ps.poll()))
if not out[-1].startswith("PASS"):
raise error.TestFail("Unsuccessful memory filling " "(WO cgroup)")
logging.debug("test_memory: Memfill WO cgroup passed")
################################################
# Fill the memory with 1/2 memory limit
# memsw: should swap out part of the process and pass
# WO memsw: should fail (SIGKILL)
################################################
logging.debug("test_memory: Memfill mem only limit")
ps = item.test("memfill %d %s" % (mem, outf.name))
item.set_cgroup(ps.pid, pwd)
item.set_property_h("memory.limit_in_bytes", ("%dM" % (mem / 2)), pwd)
ps.stdin.write('\n')
i = 0
while ps.poll() == None:
if i > 120:
break
i += 1
time.sleep(1)
if i > 120:
ps.terminate()
raise error.TestFail("Memory filling failed (mem)")
outf.seek(0)
outf.flush()
out = outf.readlines()
if (len(out) < 2):
raise error.TestFail("Process failed (mem); output:\n%s"
"\nReturn: %d" % (out, ps.poll()))
if memsw:
if not out[-1].startswith("PASS"):
logging.error(
"test_memory: cgroup_client.py returned %d; "
"output:\n%s", ps.poll(), out)
raise error.TestFail("Unsuccessful memory filling (mem)")
else:
if out[-1].startswith("PASS"):
raise error.TestFail("Unexpected memory filling (mem)")
else:
filled = int(out[-2].split()[1][:-1])
if mem / 2 > 1.5 * filled:
logging.error(
"test_memory: Limit = %dM, Filled = %dM (+ "
"python overhead upto 1/3 (mem))", mem / 2, filled)
else:
logging.debug(
"test_memory: Limit = %dM, Filled = %dM (+ "
"python overhead upto 1/3 (mem))", mem / 2, filled)
logging.debug("test_memory: Memfill mem only cgroup passed")
################################################
# Fill the memory with 1/2 memory+swap limit
# Should fail
# (memory.limit_in_bytes have to be set prior to this test)
################################################
if memsw:
logging.debug("test_memory: Memfill mem + swap limit")
ps = item.test("memfill %d %s" % (mem, outf.name))
item.set_cgroup(ps.pid, pwd)
item.set_property_h("memory.memsw.limit_in_bytes",
"%dM" % (mem / 2), pwd)
ps.stdin.write('\n')
i = 0
while ps.poll() == None:
if i > 120:
break
i += 1
time.sleep(1)
if i > 120:
ps.terminate()
raise error.TestFail("Memory filling failed (mem)")
outf.seek(0)
outf.flush()
out = outf.readlines()
if (len(out) < 2):
raise error.TestFail("Process failed (memsw); output:\n%s"
"\nReturn: %d" % (out, ps.poll()))
if out[-1].startswith("PASS"):
raise error.TestFail("Unexpected memory filling (memsw)", mem)
else:
filled = int(out[-2].split()[1][:-1])
if mem / 2 > 1.5 * filled:
logging.error(
"test_memory: Limit = %dM, Filled = %dM (+ "
"python overhead upto 1/3 (memsw))", mem / 2, filled)
else:
logging.debug(
"test_memory: Limit = %dM, Filled = %dM (+ "
"python overhead upto 1/3 (memsw))", mem / 2, filled)
logging.debug("test_memory: Memfill mem + swap cgroup passed")
################################################
# CLEANUP
################################################
cleanup()
def align_to_genome(batch_reads_data,
genome_fn,
mapper_exe,
mapper_type,
genome_index,
output_format='sam'):
# prepare fasta text with batch reads
batch_reads_fasta = ''
for read_fn_sg, (_, _, basecalls, _, _, _) in \
batch_reads_data.iteritems():
batch_reads_fasta += ">" + read_fn_sg + '\n' + \
''.join(basecalls) + '\n'
read_fp = NamedTemporaryFile(suffix='.fasta')
read_fp.write(batch_reads_fasta)
read_fp.flush()
out_fp = NamedTemporaryFile()
# optionally suppress output from mapper with devnull sink
with open(os.devnull, 'w') as FNULL:
if mapper_type == 'graphmap':
mapper_options = prep_graphmap_options(genome_fn, read_fp.name,
out_fp.name, output_format)
stdout_sink = FNULL
elif mapper_type == 'bwa_mem':
mapper_options = prep_bwa_mem_options(genome_fn, read_fp.name)
stdout_sink = out_fp
else:
raise RuntimeError, 'Mapper not supported.'
try:
exitStatus = call([
mapper_exe,
] + mapper_options,
stdout=stdout_sink,
stderr=FNULL)
out_fp.seek(0)
align_output = out_fp.readlines()
# close files here so that they persist until
# after basecalling is finished
read_fp.close()
out_fp.close()
except:
# whole mapping call failed so all reads failed
return ([
('Problem running/parsing genome mapper. ' +
'Ensure you have a compatible version installed.' +
'Potentially failed to locate BWA index files.', read_fn_sg)
for read_fn_sg in batch_reads_data.keys()
], [])
if output_format == 'sam':
batch_parse_failed_reads, batch_align_data = parse_sam_output(
align_output, batch_reads_data, genome_index)
elif output_format == 'm5':
batch_parse_failed_reads, batch_align_data = parse_m5_output(
align_output, batch_reads_data)
else:
raise RuntimeError, 'Mapper output type not supported.'
clip_fix_align_data = fix_all_clipped_bases(batch_align_data,
batch_reads_data)
return batch_parse_failed_reads, clip_fix_align_data
def test_memory(self):
"""
Memory test
"""
def cleanup(supress=False):
""" cleanup """
logging.debug("test_memory: Cleanup")
err = ""
if item.rm_cgroup(pwd):
err += "\nCan't remove cgroup directory"
utils.system("swapon -a")
if err:
if supress:
logging.warn("Some parts of cleanup failed%s", err)
else:
raise error.TestFail("Some parts of cleanup failed%s" % err)
# Preparation
item = CG('memory', self._client)
item.initialize(self.modules)
item.smoke_test()
pwd = item.mk_cgroup()
logging.debug("test_memory: Memory filling test")
meminfo = open('/proc/meminfo','r')
mem = meminfo.readline()
while not mem.startswith("MemFree"):
mem = meminfo.readline()
# Use only 1G or max of the free memory
mem = min(int(mem.split()[1])/1024, 1024)
mem = max(mem, 100) # at least 100M
try:
memsw_limit_bytes = item.get_property("memory.memsw.limit_in_bytes")
except error.TestFail:
# Doesn't support memsw limitation -> disabling
logging.info("System does not support 'memsw'")
utils.system("swapoff -a")
memsw = False
else:
# Supports memsw
memsw = True
# Clear swap
utils.system("swapoff -a")
utils.system("swapon -a")
meminfo.seek(0)
swap = meminfo.readline()
while not swap.startswith("SwapTotal"):
swap = meminfo.readline()
swap = int(swap.split()[1])/1024
if swap < mem / 2:
logging.error("Not enough swap memory to test 'memsw'")
memsw = False
meminfo.close()
outf = NamedTemporaryFile('w+', prefix="cgroup_client-",
dir="/tmp")
logging.debug("test_memory: Initializition passed")
################################################
# Fill the memory without cgroup limitation
# Should pass
################################################
logging.debug("test_memory: Memfill WO cgroup")
ps = item.test("memfill %d %s" % (mem, outf.name))
ps.stdin.write('\n')
i = 0
while ps.poll() == None:
if i > 60:
break
i += 1
time.sleep(1)
if i > 60:
ps.terminate()
raise error.TestFail("Memory filling failed (WO cgroup)")
outf.seek(0)
outf.flush()
out = outf.readlines()
if (len(out) < 2) or (ps.poll() != 0):
raise error.TestFail("Process failed (WO cgroup); output:\n%s"
"\nReturn: %d" % (out, ps.poll()))
if not out[-1].startswith("PASS"):
raise error.TestFail("Unsuccessful memory filling "
"(WO cgroup)")
logging.debug("test_memory: Memfill WO cgroup passed")
################################################
# Fill the memory with 1/2 memory limit
# memsw: should swap out part of the process and pass
# WO memsw: should fail (SIGKILL)
################################################
logging.debug("test_memory: Memfill mem only limit")
ps = item.test("memfill %d %s" % (mem, outf.name))
item.set_cgroup(ps.pid, pwd)
item.set_property_h("memory.limit_in_bytes", ("%dM" % (mem/2)), pwd)
ps.stdin.write('\n')
i = 0
while ps.poll() == None:
if i > 120:
break
i += 1
time.sleep(1)
if i > 120:
ps.terminate()
raise error.TestFail("Memory filling failed (mem)")
outf.seek(0)
outf.flush()
out = outf.readlines()
if (len(out) < 2):
raise error.TestFail("Process failed (mem); output:\n%s"
"\nReturn: %d" % (out, ps.poll()))
if memsw:
if not out[-1].startswith("PASS"):
logging.error("test_memory: cgroup_client.py returned %d; "
"output:\n%s", ps.poll(), out)
raise error.TestFail("Unsuccessful memory filling (mem)")
else:
if out[-1].startswith("PASS"):
raise error.TestFail("Unexpected memory filling (mem)")
else:
filled = int(out[-2].split()[1][:-1])
if mem/2 > 1.5 * filled:
logging.error("test_memory: Limit = %dM, Filled = %dM (+ "
"python overhead upto 1/3 (mem))", mem/2,
filled)
else:
logging.debug("test_memory: Limit = %dM, Filled = %dM (+ "
"python overhead upto 1/3 (mem))", mem/2,
filled)
logging.debug("test_memory: Memfill mem only cgroup passed")
################################################
# Fill the memory with 1/2 memory+swap limit
# Should fail
# (memory.limit_in_bytes have to be set prior to this test)
################################################
if memsw:
logging.debug("test_memory: Memfill mem + swap limit")
ps = item.test("memfill %d %s" % (mem, outf.name))
item.set_cgroup(ps.pid, pwd)
item.set_property_h("memory.memsw.limit_in_bytes", "%dM"%(mem/2), pwd)
ps.stdin.write('\n')
i = 0
while ps.poll() == None:
if i > 120:
break
i += 1
time.sleep(1)
if i > 120:
ps.terminate()
raise error.TestFail("Memory filling failed (mem)")
outf.seek(0)
outf.flush()
out = outf.readlines()
if (len(out) < 2):
raise error.TestFail("Process failed (memsw); output:\n%s"
"\nReturn: %d" % (out, ps.poll()))
if out[-1].startswith("PASS"):
raise error.TestFail("Unexpected memory filling (memsw)",
mem)
else:
filled = int(out[-2].split()[1][:-1])
if mem / 2 > 1.5 * filled:
logging.error("test_memory: Limit = %dM, Filled = %dM (+ "
"python overhead upto 1/3 (memsw))", mem/2,
filled)
else:
logging.debug("test_memory: Limit = %dM, Filled = %dM (+ "
"python overhead upto 1/3 (memsw))", mem/2,
filled)
logging.debug("test_memory: Memfill mem + swap cgroup passed")
################################################
# CLEANUP
################################################
cleanup()
class Playbook(object):
ACTIVE = "ACTIVE"
PLAYBOOK_FAILED = "PLAYBOOK_FAILED"
def __init__(
self,
ip,
port,
playbooks_information,
osi_private_key,
public_key,
pool,
loaded_metadata_keys,
cloud_site,
):
self.loaded_metadata_keys = loaded_metadata_keys
self.cloud_site = cloud_site
self.redis = redis.Redis(connection_pool=pool) # redis connection
self.yaml_exec = ruamel.yaml.YAML() # yaml writer/reader
self.vars_files = [] # _vars_file.yml to read
self.tasks = [] # task list
self.always_tasks = []
self.process = (
None # init process, returncode, standard output, standard error output
)
self.returncode = -1
self.playbooks_information = playbooks_information
self.stdout = ""
self.stderr = ""
# init temporary directories and mandatory generic files
self.ancon_dir = "/code/VirtualMachineService/ancon" # path to this directory
self.playbooks_dir = self.ancon_dir + "/playbooks" # path to source playbooks
self.directory = TemporaryDirectory(dir=self.ancon_dir)
self.private_key = NamedTemporaryFile(mode="w+",
dir=self.directory.name,
delete=False,
prefix="key_")
self.private_key.write(osi_private_key)
self.private_key.close()
self.log_file_stdout = NamedTemporaryFile(mode="w+",
dir=self.directory.name,
delete=False,
prefix="log_stdout")
self.log_file_stderr = NamedTemporaryFile(mode="w+",
dir=self.directory.name,
delete=False,
prefix="log_err")
# create the custom playbook and save its name
self.playbook_exec_name = "generic_playbook.yml"
self.copy_playbooks_and_init(playbooks_information, public_key)
# create inventory
self.inventory = NamedTemporaryFile(mode="w+",
dir=self.directory.name,
delete=False,
prefix="inventory_")
inventory_string = (
f"[vm]\n"
f"{ip} ansible_port={port} ansible_user=ubuntu ansible_ssh_private_key_file={self.private_key.name} ansible_python_interpreter=/usr/bin/python3"
)
self.inventory.write(inventory_string)
self.inventory.close()
def copy_playbooks_and_init(self, playbooks_information, public_key):
# go through every wanted playbook
for k, v in playbooks_information.items():
self.copy_and_init(k, v)
# init yml to change public keys as last task
shutil.copy(self.playbooks_dir + "/change_key.yml",
self.directory.name)
shutil.copy(self.playbooks_dir + "/change_key_vars_file.yml",
self.directory.name)
with open(self.directory.name + "/change_key_vars_file.yml",
mode="r") as key_file:
data_ck = self.yaml_exec.load(key_file)
data_ck["change_key_vars"]["key"] = public_key.strip('"')
with open(self.directory.name + "/change_key_vars_file.yml",
mode="w") as key_file:
self.yaml_exec.dump(data_ck, key_file)
self.add_to_playbook_always_lists("change_key")
# write all vars_files and tasks in generic_playbook
shutil.copy(self.playbooks_dir + "/" + self.playbook_exec_name,
self.directory.name)
with open(self.directory.name + "/" + self.playbook_exec_name,
mode="r") as generic_playbook:
data_gp = self.yaml_exec.load(generic_playbook)
data_gp[0]["vars_files"] = self.vars_files
data_gp[0]["tasks"][0]["block"] = self.tasks
data_gp[0]["tasks"][0]["always"] = self.always_tasks
with open(self.directory.name + "/" + self.playbook_exec_name,
mode="w") as generic_playbook:
self.yaml_exec.dump(data_gp, generic_playbook)
def copy_and_init(self, playbook_name, playbook_vars):
def load_vars():
LOG.info(f" Playbook vars: {playbook_vars}")
if playbook_name == CONDA:
for k, v in playbook_vars.items():
if k == "packages":
p_array = []
p_dict = {}
for p in (v.strip('"')).split():
p_array.append(p.split("="))
for p in p_array:
p_dict.update({p[0]: {"version": p[1]}})
data[playbook_name + "_vars"][k] = p_dict
if playbook_name in self.loaded_metadata_keys:
for k, v in playbook_vars.items():
LOG.info(playbook_vars)
if k == "template_version":
data[playbook_name + "_vars"][k] = v
if k == "create_only_backend":
if playbook_vars[k] in ["false", "False"]:
data[playbook_name + "_vars"][k] = False
elif playbook_vars[k] in ["true", "True"]:
data[playbook_name + "_vars"][k] = True
if k == "base_url":
data[playbook_name + "_vars"][k] = v
if playbook_name == OPTIONAL:
for k, v in playbook_vars.items():
if k == MOSH:
data[playbook_name + "_defined"][k] = v
LOG.info(f"Playbook Data - {data}")
# copy whole directory
shutil.copytree(
f"{self.playbooks_dir}/{playbook_name}",
self.directory.name,
dirs_exist_ok=True,
)
site_specific_yml = f"/{playbook_name}{'-' + self.cloud_site}.yml"
playbook_name_local = playbook_name
if os.path.isfile(self.directory.name + site_specific_yml):
playbook_name_local = playbook_name + "-" + self.cloud_site
playbook_var_yml = f"/{playbook_name}_vars_file.yml"
try:
with open(self.directory.name + playbook_var_yml,
mode="r") as variables:
data = self.yaml_exec.load(variables)
load_vars()
with open(self.directory.name + playbook_var_yml,
mode="w") as variables:
self.yaml_exec.dump(data, variables)
self.add_to_playbook_lists(playbook_name_local, playbook_name)
except shutil.Error as e:
LOG.exception(e)
self.add_tasks_only(playbook_name_local)
except IOError as e:
LOG.exception(e)
self.add_tasks_only(playbook_name_local)
def add_to_playbook_lists(self, playbook_name_local, playbook_name):
self.vars_files.append(playbook_name + "_vars_file.yml")
self.tasks.append(
dict(
name=f"Running {playbook_name_local} tasks",
import_tasks=playbook_name_local + ".yml",
))
LOG.info("Added playbook: " + playbook_name_local + ".yml" +
", vars file: " + playbook_name + "_vars_file.yml")
def add_tasks_only(self, playbook_name):
self.tasks.append(
dict(
name=f"Running {playbook_name} tasks",
import_tasks=playbook_name + ".yml",
))
def add_to_playbook_always_lists(self, playbook_name):
self.vars_files.append(playbook_name + "_vars_file.yml")
self.always_tasks.append(
dict(
name=f"Running {playbook_name} tasks",
import_tasks=playbook_name + ".yml",
))
def add_always_tasks_only(self, playbook_name):
self.always_tasks.append(
dict(
name=f"Running {playbook_name} tasks",
import_tasks=playbook_name + ".yml",
))
def run_it(self):
command_string = "/usr/local/bin/ansible-playbook -v -i {0} {1}/{2}".format(
self.inventory.name, self.directory.name, self.playbook_exec_name)
LOG.info(f"Run Playbook with command {command_string}")
command_string = shlex.split(command_string)
self.process = subprocess.Popen(
command_string,
stdout=self.log_file_stdout,
stderr=self.log_file_stderr,
universal_newlines=True,
)
def check_status(self, openstack_id):
done = self.process.poll()
if done is None:
LOG.info(
f"Playbook for (openstack_id) {openstack_id} still in progress."
)
elif done != 0:
LOG.info(f"Playbook for (openstack_id) {openstack_id} has failed.")
self.redis.hset(openstack_id, "status", self.PLAYBOOK_FAILED)
self.returncode = self.process.returncode
self.process.wait()
else:
LOG.info(
f"Playbook for (openstack_id) {openstack_id} is successful.")
self.redis.hset(openstack_id, "status", self.ACTIVE)
self.returncode = self.process.returncode
self.process.wait()
return done
def get_logs(self):
self.log_file_stdout.seek(0, 0)
lines_stdout = self.log_file_stdout.readlines()
for line in lines_stdout:
self.stdout += line
self.log_file_stderr.seek(0, 0)
line_stderr = self.log_file_stderr.readlines()
for line in line_stderr:
self.stderr += line
return self.returncode, self.stdout, self.stderr
def cleanup(self, openstack_id):
# self.directory.cleanup()
self.redis.delete(openstack_id)
def stop(self, openstack_id):
self.process.terminate()
rc, stdout, stderr = self.get_logs()
logs_to_save = {"returncode": rc, "stdout": stdout, "stderr": stderr}
self.redis.hmset(f"pb_logs_{openstack_id}", logs_to_save)
self.cleanup(openstack_id)