def test_make_unstructured_grid(tmpdir):
""" Test the make_unstructured_grid routine."""
mesh = IO.GmshMesh()
mesh.read("/".join((DATA_DIR, 'Structured.msh')))
print("/".join((DATA_DIR, 'Structured.msh')))
num = len(mesh.nodes)
vel = numpy.zeros((num, 3))
pres = numpy.zeros((num, ))
time = 0
ugrid = IO.make_unstructured_grid(
mesh, vel, pres, time, outfile=tmpdir.join('Structured.vtu').strpath)
assert num > 0
assert ugrid.GetNumberOfPoints() == num
assert ugrid.GetNumberOfCells() == len(mesh.elements)
assert filecmp.cmpfiles(DATA_DIR, tmpdir.strpath, 'Structured.vtu')
ugrid = IO.make_unstructured_grid(
mesh,
lambda x: (0, 0, 0),
lambda x: 0,
time,
outfile=tmpdir.join('Structured.vtu').strpath)
assert ugrid.GetNumberOfPoints() == num
assert ugrid.GetNumberOfCells() == len(mesh.elements)
assert filecmp.cmpfiles(DATA_DIR, tmpdir.strpath, 'Structured.vtu')
def test_cmpfiles(self, comparator_tmpdir):
assert filecmp.cmpfiles(
comparator_tmpdir.dir,
comparator_tmpdir.dir,
["file"],
) == (["file"], [], []), "Comparing directory to itself fails"
assert filecmp.cmpfiles(
comparator_tmpdir.dir,
comparator_tmpdir.dir_same,
["file"],
) == (["file"], [], []), "Comparing directory to same fails"
# Try it with shallow=False
assert filecmp.cmpfiles(
comparator_tmpdir.dir,
comparator_tmpdir.dir,
["file"],
shallow=False,
) == (["file"], [], []), "Comparing directory to itself fails"
assert filecmp.cmpfiles(
comparator_tmpdir.dir,
comparator_tmpdir.dir_same,
["file"],
shallow=False,
), "Comparing directory to same fails"
# Add different file2
with open(os.path.join(comparator_tmpdir.dir, "file2"), 'w') as output:
output.write('Different contents.\n')
assert filecmp.cmpfiles(
comparator_tmpdir.dir,
comparator_tmpdir.dir_same,
["file", "file2"],
) != (["file"], ["file2"], []), "Comparing mismatched directories fails"
def test_cmpfiles(self):
self.failUnless(filecmp.cmpfiles(self.dir, self.dir, ['file']) ==
(['file'], [], []),
"Comparing directory to itself fails")
self.failUnless(filecmp.cmpfiles(self.dir, self.dir_same, ['file']) ==
(['file'], [], []),
"Comparing directory to same fails")
# Try it with shallow=False
self.failUnless(filecmp.cmpfiles(self.dir, self.dir, ['file'],
shallow=False) ==
(['file'], [], []),
"Comparing directory to itself fails")
self.failUnless(filecmp.cmpfiles(self.dir, self.dir_same, ['file'],
shallow=False),
"Comparing directory to same fails")
# Add different file2
output = open(os.path.join(self.dir, 'file2'), 'w')
output.write('Different contents.\n')
output.close()
self.failIf(filecmp.cmpfiles(self.dir, self.dir_same,
['file', 'file2']) ==
(['file'], ['file2'], []),
"Comparing mismatched directories fails")
def test_cmpfiles(self):
self.assertTrue(
filecmp.cmpfiles(self.dir, self.dir,
['file']) == (['file'], [], []),
"Comparing directory to itself fails")
self.assertTrue(
filecmp.cmpfiles(self.dir, self.dir_same,
['file']) == (['file'], [], []),
"Comparing directory to same fails")
# Try it with shallow=False
self.assertTrue(
filecmp.cmpfiles(self.dir, self.dir, ['file'],
shallow=False) == (['file'], [], []),
"Comparing directory to itself fails")
self.assertTrue(
filecmp.cmpfiles(self.dir, self.dir_same, ['file'], shallow=False),
"Comparing directory to same fails")
# Add different file2
with open(os.path.join(self.dir, 'file2'), 'w',
encoding="utf-8") as output:
output.write('Different contents.\n')
self.assertFalse(
filecmp.cmpfiles(self.dir, self.dir_same,
['file', 'file2']) == (['file'], ['file2'], []),
"Comparing mismatched directories fails")
def test_cmpfiles(self):
self.assertTrue(
filecmp.cmpfiles(self.dir, self.dir, ["file"]) == (["file"], [], []), "Comparing directory to itself fails"
)
self.assertTrue(
filecmp.cmpfiles(self.dir, self.dir_same, ["file"]) == (["file"], [], []),
"Comparing directory to same fails",
)
# Try it with shallow=False
self.assertTrue(
filecmp.cmpfiles(self.dir, self.dir, ["file"], shallow=False) == (["file"], [], []),
"Comparing directory to itself fails",
)
self.assertTrue(
filecmp.cmpfiles(self.dir, self.dir_same, ["file"], shallow=False), "Comparing directory to same fails"
)
# Add different file2
output = open(os.path.join(self.dir, "file2"), "w")
output.write("Different contents.\n")
output.close()
self.assertFalse(
filecmp.cmpfiles(self.dir, self.dir_same, ["file", "file2"]) == (["file"], ["file2"], []),
"Comparing mismatched directories fails",
)
def test_cmpfiles(self):
self.failUnless(
filecmp.cmpfiles(self.dir, self.dir,
['file']) == (['file'], [], []),
"Comparing directory to itself fails")
self.failUnless(
filecmp.cmpfiles(self.dir, self.dir_same,
['file']) == (['file'], [], []),
"Comparing directory to same fails")
# Try it with shallow=False
self.failUnless(
filecmp.cmpfiles(self.dir, self.dir, ['file'],
shallow=False) == (['file'], [], []),
"Comparing directory to itself fails")
self.failUnless(
filecmp.cmpfiles(self.dir, self.dir_same, ['file'], shallow=False),
"Comparing directory to same fails")
# Add different file2
output = open(os.path.join(self.dir, 'file2'), 'w')
output.write('Different contents.\n')
output.close()
self.failIf(
filecmp.cmpfiles(self.dir, self.dir_same,
['file', 'file2']) == (['file'], ['file2'], []),
"Comparing mismatched directories fails")
def test_make_unstructured_grid(tmpdir):
""" Test the make_unstructured_grid routine."""
mesh = IO.GmshMesh()
mesh.read("/".join((DATA_DIR, 'Structured.msh')))
print "/".join((DATA_DIR, 'Structured.msh'))
num = len(mesh.nodes)
vel = numpy.zeros((num, 3))
pres = numpy.zeros((num,))
time = 0
ugrid = IO.make_unstructured_grid(mesh, vel, pres, time,
outfile=tmpdir.join('Structured.vtu').strpath)
assert num > 0
assert ugrid.GetNumberOfPoints() == num
assert ugrid.GetNumberOfCells() == len(mesh.elements)
assert filecmp.cmpfiles(DATA_DIR, tmpdir.strpath, 'Structured.vtu')
ugrid = IO.make_unstructured_grid(mesh, lambda x: (0, 0, 0),
lambda x: 0, time,
outfile=tmpdir.join('Structured.vtu').strpath)
assert ugrid.GetNumberOfPoints() == num
assert ugrid.GetNumberOfCells() == len(mesh.elements)
assert filecmp.cmpfiles(DATA_DIR, tmpdir.strpath, 'Structured.vtu')
def test_run_container(validation_folders: Dict, host_folders: Dict, docker_client: docker.DockerClient, docker_image_key: str, container_variables: Dict):
# copy file to input folder
host_folders["input"].rmdir()
shutil.copytree(validation_folders["input"], host_folders["input"])
assert Path(host_folders["input"]).exists()
# run the container (this may take some time)
docker_client.containers.run(docker_image_key,
"run", detach=False, remove=True,
volumes = {
host_folders["input"] : {
'bind' : container_variables["INPUT_FOLDER"],
'mode': 'ro'
},
host_folders["output"] : {
'bind' : container_variables["OUTPUT_FOLDER"],
'mode': 'rw'
},
host_folders["log"] : {
'bind': container_variables["LOG_FOLDER"],
'mode': 'rw'
}
},
environment=container_variables)
for folder in ["input", "output", "log"]:
# test if the files that should be there are actually there and correct
list_of_files = [x.name for x in validation_folders[folder].iterdir()]
match, mismatch, errors = filecmp.cmpfiles(host_folders[folder], validation_folders[folder], list_of_files, shallow=False)
assert not mismatch, "wrong/incorrect files in {}".format(host_folders[folder])
assert not errors, "missing files in {}".format(host_folders[folder])
# test if the files that are there are matching the ones that should be
list_of_files = [x.name for x in host_folders[folder].iterdir()]
match, mismatch, errors = filecmp.cmpfiles(host_folders[folder], validation_folders[folder], list_of_files, shallow=False)
assert not mismatch, "wrong/incorrect generated files in {}".format(host_folders[folder])
assert not errors, "too many files in {}".format(host_folders[folder])
def test_single_reads(self):
sams = [
os.path.join('unit_test', 'data', 'single_reads_genome1.sam'),
os.path.join('unit_test', 'data', 'single_reads_genome2.sam')
]
out_dir = os.path.join('unit_test', 'output')
files_to_check = ['sorted1.sam',
'sorted2.sam'] #'supplementary_output.txt'
# no multiprocessing
sparta.sparta(sams,
num_processes=1,
output_dir=os.path.join(out_dir, 'no_mp'),
separated_samfiles=[
os.path.join(out_dir, 'no_mp', 'sorted1.sam'),
os.path.join(out_dir, 'no_mp', 'sorted2.sam')
])
# processes = cpu_count
sparta.sparta(sams,
output_dir=os.path.join(out_dir, 'mp'),
separated_samfiles=[
os.path.join(out_dir, 'mp', 'sorted1.sam'),
os.path.join(out_dir, 'mp', 'sorted2.sam')
])
# processes = 10
sparta.sparta(sams,
num_processes=10,
output_dir=os.path.join(out_dir, 'mp10'),
separated_samfiles=[
os.path.join(out_dir, 'mp10', 'sorted1.sam'),
os.path.join(out_dir, 'mp10', 'sorted2.sam')
])
match, mismatch, errors = filecmp.cmpfiles(os.path.join(
out_dir, 'no_mp'),
os.path.join(out_dir, 'mp'),
common=files_to_check)
match2, mismatch2, errors2 = filecmp.cmpfiles(
os.path.join(out_dir, 'no_mp'),
os.path.join(out_dir, 'mp10'),
common=files_to_check)
# check that output was created, and no files mismatched, and no errors occured
assert match != []
assert mismatch == []
assert errors == []
assert match2 != []
assert mismatch2 == []
assert errors2 == []
# clean up
shutil.rmtree(out_dir)
os.makedirs(out_dir)
def test_load_save_game(self):
print ""
print "TEST load_game functionality, first start a new game."
print "Make some changes, save the game, start another new game."
print "Then load game and check that the changes we made are as expected."
files.new_game()
rooms = name_lists.room_info().get_titles()
items = name_lists.item_info().get_titles()
new_rooms_dir = name_lists.save_info().get_temp_save_dir_rooms()
new_items_dir = name_lists.save_info().get_temp_save_dir_items()
save_rooms_dir = name_lists.save_info().get_save_dir_rooms()
save_items_dir = name_lists.save_info().get_save_dir_items()
print "Open each room and edit the visited attribute"
for title in rooms:
room = files.load_room(title)
room['visited'] = ""
files.store_room(room)
print "Open each item and edit the active attribute"
for title in items:
item = files.load_item(title)
if item['active']:
item['active'] = False
else:
item['active'] = True
files.store_item(item)
title = random.choice(rooms)
current_room = files.load_room(title)
files.save_game(self.player, current_room, 0)
files.new_game()
print "TEST compare the files in temp and save and confirm no match"
match, mismatch, errors = filecmp.cmpfiles(save_rooms_dir,
new_rooms_dir, rooms)
for file_ in match:
self.assertNotIn(file_, rooms)
self.assertEqual(len(match), 0)
self.assertEqual(len(errors), 0)
match, mismatch, errors = filecmp.cmpfiles(save_items_dir,
new_items_dir, items)
for file_ in match:
self.assertNotIn(file_, items)
self.assertEqual(len(match), 0)
self.assertEqual(len(errors), 0)
print "TEST that player file exists and that it is a JSON dict"
save_dir = name_lists.save_info().get_save_dir()
player_file = os.path.join(save_dir, 'player')
self.assertEqual(os.path.isfile(player_file), True)
try:
with open(player_file, 'r') as player_file:
player = json.load(player_file)
except Exception, e:
player = None
def mydir(arg, dirname, names):
file = [normpath(tfile) for tfile in names]
fullfilename = [abspath(join(dirname, tfile)) for tfile in names]
#copy it, $file and $fullfilename are one-one correspondence
for i in range(0, len(file)):
if isfile(fullfilename[i]): #only save files
#save WriteIMEI*.ini files
p = re.compile("WriteIMEI.*\.ini")
matchit = p.match(file[i])
if matchit:
#print matchit.group()
list_ini_files.append(fullfilename[i])
continue
if isdir(fullfilename[i]): #only save path
#save cal files's RID
p = re.compile("[0-9A-Fa-f]{32}")
matchit = p.match(fullfilename[i][-32:])
if matchit:
lallCalNumInCal.append(fullfilename[i][-32:]) # save RID
#here we need to compare with DEFAULT cal file under \Z\NVRAM\CALIBRAT\
#check DAUL DATA
dmatch, dmismatch, derrors = filecmp.cmpfiles(
str(fullfilename[i]) + "\\Z\\NVRAM\\CALIBRAT",
"REFCALIBRAT\\DUAL",
DUALCalFileList,
shallow=False)
qmatch, qmismatch, qerrors = filecmp.cmpfiles(
str(fullfilename[i]) + "\\Z\\NVRAM\\CALIBRAT",
"REFCALIBRAT\\QUAD",
QUADCalFileList,
shallow=False)
#print fullfilename[i][-32:]+"\n"
#print 'DMatch:', dmatch, '\n'
#print 'DMismatch:', dmismatch, '\n'
#print 'DErrors:', derrors, '\n'
#print 'QMatch:', qmatch, '\n'
#print 'QMismatch:', qmismatch, '\n'
#print 'QErrors:', qerrors, '\n'
if (len(dmatch) == 4):
lDUALDefaultCalByChipID.append(fullfilename[i][-32:])
if (len(qmatch) == 4):
lQUADDefaultCalByChipID.append(fullfilename[i][-32:])
if (len(derrors) == 0 and len(qerrors) == 0):
lIsQUADCalByChipID.append(fullfilename[i][-32:])
elif (len(derrors) == 0 and len(qerrors) == 4):
lIsDUALCalByChipID.append(fullfilename[i][-32:])
else:
lIsNULLBandCalByChipID.append(fullfilename[i][-32:])
continue
def get_diffs(gold_base, test_base, gold_root):
"""Diff test output with the golden directory.
Args:
gold_base: Golden (known-good) directory path.
test_base: Test output directory path.
gold_root: Returned paths are relative to this path.
Returns:
matches: Matching files.
mismatches: Different files.
missings: Files in golden root, missing in test_base.
extras: Files not in golden root, extra in test_base.
"""
# Walk the golden tree testing for differences with the test tree.
all_matches = []
all_mismatches = []
all_missings = []
for gold_dir, _, file_list in os.walk(gold_base):
if not file_list:
continue
test_dir = join_path(test_base, norm_relpath(gold_dir, gold_base))
(matches, mismatches, errors) = filecmp.cmpfiles(gold_dir,
test_dir,
file_list,
shallow=False)
rel_dir = norm_relpath(gold_dir, gold_root)
for name in matches:
all_matches.append(join_path(rel_dir, name))
for name in mismatches:
rel_file_path = join_path(rel_dir, name)
gold_path = join_path(gold_dir, name)
test_path = join_path(test_dir, name)
(_, ext) = os.path.splitext(name)
# TODO: USDC (and there by extension USDZ) output may not be
# deterministic, so we should use the usddiff tool in the USD library to
# perform deep compares when there's a binary mismatch.
if ext != '.usdz' or not compare_usdz_content(
gold_path, test_path):
all_mismatches.append(rel_file_path)
for name in errors:
all_missings.append(join_path(rel_dir, name))
# Walk the test tree to find new files not in the golden tree.
all_extras = []
for test_dir, _, file_list in os.walk(test_base):
if not file_list:
continue
gold_dir = join_path(gold_base, norm_relpath(test_dir, test_base))
(_, _, errors) = filecmp.cmpfiles(gold_dir, test_dir, file_list)
rel_dir = norm_relpath(gold_dir, gold_root)
for name in errors:
all_extras.append(join_path(rel_dir, name))
return (all_matches, all_mismatches, all_missings, all_extras)
def test_cpp_class(self):
generate_class(output_dir='.')
expected_cpp = ['class.cpp']
expected_h = ['class.h']
self.assertEqual(
filecmp.cmpfiles('.', 'test_assets', expected_cpp)[0],
expected_cpp)
self.assertEqual(
filecmp.cmpfiles('.', 'test_assets', expected_h)[0], expected_h)
os.remove(expected_cpp[0])
os.remove(expected_h[0])
def test_disease_gene_example_hetmat_archiving(tmpdir):
"""
Test archiving the hetmat corresponding to the hetnet in Figure 2C at https://doi.org/crz8.
"""
tmpdir = pathlib.Path(tmpdir)
graph = get_graph('disease-gene-example')
hetmat_0_dir = tmpdir.joinpath('disease-gene-example-0.hetmat')
hetmat = hetmatpy.hetmat.hetmat_from_graph(graph, hetmat_0_dir)
# Test creating archive
archive_path = hetmatpy.hetmat.archive.create_hetmat_archive(hetmat)
with zipfile.ZipFile(archive_path) as zip_file:
name_list = zip_file.namelist()
expected = [
'edges/DlT.sparse.npz',
'edges/GaD.sparse.npz',
'edges/GeT.sparse.npz',
'edges/GiG.sparse.npz',
'metagraph.json',
'nodes/Disease.tsv',
'nodes/Gene.tsv',
'nodes/Tissue.tsv',
]
assert name_list == expected
# Test round-tripped hetmat has same files
hetmat_1_dir = tmpdir.joinpath('disease-gene-example-1.hetmat')
hetmatpy.hetmat.archive.load_archive(archive_path, hetmat_1_dir)
match, mismatch, errors = filecmp.cmpfiles(hetmat_0_dir,
hetmat_1_dir,
common=expected,
shallow=False)
assert match == expected
assert not mismatch
assert not errors
# Test round-tripped hetmat has same files when specifying a
# subset of zip members in load_archive with source_paths.
hetmat_2_dir = tmpdir.joinpath('disease-gene-example-2.hetmat')
extract_only = [
'edges/GiG.sparse.npz',
'metagraph.json',
'nodes/Disease.tsv',
]
hetmatpy.hetmat.archive.load_archive(archive_path,
hetmat_2_dir,
source_paths=extract_only)
match, mismatch, errors = filecmp.cmpfiles(hetmat_0_dir,
hetmat_2_dir,
common=expected,
shallow=False)
assert match == extract_only
assert not mismatch
assert 'nodes/Tissue.tsv' in errors
def test_run_container(validation_folders: Dict, host_folders: Dict, docker_container: docker.models.containers.Container):
for folder in _FOLDER_NAMES:
# test if the files that should be there are actually there and correct
list_of_files = [
x.name for x in validation_folders[folder].iterdir() if not ".gitkeep" in x.name]
for file_name in list_of_files:
assert Path(host_folders[folder] / file_name).exists(
), f"{file_name} is missing from {host_folders[folder]}"
# we look for missing files only. contents is the responsibility of the service creator
_, _, errors = filecmp.cmpfiles(
host_folders[folder], validation_folders[folder], list_of_files, shallow=True)
assert not errors, f"{errors} are missing in {host_folders[folder]}"
if folder == "input":
continue
# test if the generated files are the ones expected
list_of_files = [
x.name for x in host_folders[folder].iterdir() if not ".gitkeep" in x.name]
for file_name in list_of_files:
assert Path(validation_folders[folder] / file_name).exists(
), "{} is not present in {}".format(file_name, validation_folders[folder])
_, _, errors = filecmp.cmpfiles(
host_folders[folder], validation_folders[folder], list_of_files, shallow=False)
# assert not mismatch, "wrong/incorrect generated files in {}".format(host_folders[folder])
assert not errors, f"{errors} should not be available in {host_folders[folder]}"
# check the output is correct based on container labels
output_cfg = {}
output_cfg_file = Path(host_folders["output"] / "outputs.json")
if output_cfg_file.exists():
with output_cfg_file.open() as fp:
output_cfg = json.load(fp)
container_labels = docker_container.labels
io_simcore_labels = _convert_to_simcore_labels(container_labels)
assert "outputs" in io_simcore_labels
for key, value in io_simcore_labels["outputs"].items():
assert "type" in value
# rationale: files are on their own and other types are in inputs.json
if not "data:" in value["type"]:
# check that keys are available
assert key in output_cfg
else:
# it's a file and it should be in the folder as well using key as the filename
filename_to_look_for = key
if "fileToKeyMap" in value:
# ...or there is a mapping
assert len(value["fileToKeyMap"]) > 0
for filename, mapped_value in value["fileToKeyMap"].items():
assert mapped_value == key
filename_to_look_for = filename
assert (host_folders["output"] / filename_to_look_for).exists()
def test_save_game(self):
print ""
print "TEST the save game of file_lib"
files.new_game()
rooms = name_lists.room_info().get_titles()
items = name_lists.item_info().get_titles()
title = random.choice(rooms)
current_room = files.load_room(title)
new_rooms_dir = name_lists.save_info().get_temp_save_dir_rooms()
new_items_dir = name_lists.save_info().get_temp_save_dir_items()
save_rooms_dir = name_lists.save_info().get_save_dir_rooms()
save_items_dir = name_lists.save_info().get_save_dir_items()
print "Open each room and edit the visited attribute"
for title in rooms:
room = files.load_room(title)
room['visited'] = True
files.store_room(room)
print "Open each item and edit the active attribute"
for title in items:
item = files.load_item(title)
if item['active']:
item['active'] = False
else:
item['active'] = True
files.store_item(item)
files.save_game(self.player, current_room, 0)
print "TEST compare the room files in both dirs to check that they match"
match, mismatch, errors = filecmp.cmpfiles(save_rooms_dir,
new_rooms_dir, rooms)
for file_ in match:
self.assertIn(file_, rooms)
self.assertEqual(len(mismatch), 0)
self.assertEqual(len(errors), 0)
print "TEST compare the item files in both dirs to check that they match"
match, mismatch, errors = filecmp.cmpfiles(save_items_dir,
new_items_dir, items)
for file_ in match:
self.assertIn(file_, items)
self.assertEqual(len(mismatch), 0)
self.assertEqual(len(errors), 0)
print "TEST that player file exists and that it is a JSON dict"
save_dir = name_lists.save_info().get_save_dir()
player_file = os.path.join(save_dir, 'player')
self.assertEqual(os.path.isfile(player_file), True)
try:
with open(player_file, 'r') as player_file:
player = json.load(player_file)
except Exception, e:
player = None
def test_new_game(self):
IGNORE = files.IGNORE
rooms_dir = os.path.abspath('data/rooms')
items_dir = os.path.abspath('data/items')
room_files = os.listdir(rooms_dir)
item_files = os.listdir(items_dir)
result = files.new_game()
new_rooms_dir = name_lists.save_info().get_temp_save_dir_rooms()
new_items_dir = name_lists.save_info().get_temp_save_dir_items()
new_room_files = os.listdir(new_rooms_dir)
new_item_files = os.listdir(new_items_dir)
print ""
print "TEST that files were transferred"
self.assertEqual(result, True)
for file_ in new_room_files:
print "File: " + file_ + " in temp save is in template"
self.assertIn(file_, room_files)
for file_ in room_files:
if file_ not in IGNORE:
print "File: " + file_ + " in temp save is in temp save"
self.assertIn(file_, new_room_files)
for file_ in new_item_files:
print "File: " + file_ + " in temp save is in template"
self.assertIn(file_, item_files)
for file_ in item_files:
if file_ not in IGNORE:
print "File: " + file_ + " in temp save is in temp save"
self.assertIn(file_, new_item_files)
print "TEST compare the room files in both dirs to check that they match"
match, mismatch, errors = filecmp.cmpfiles(rooms_dir, new_rooms_dir,
room_files)
for name in match:
self.assertIn(name, room_files)
self.assertIn(name, new_room_files)
for name in mismatch:
self.assertIn(name, IGNORE)
self.assertEqual(len(mismatch), 0)
print "TEST compare the item files in both dirs to check that they match"
match, mismatch, errors = filecmp.cmpfiles(items_dir, new_items_dir,
item_files)
for name in match:
self.assertIn(name, item_files)
self.assertIn(name, new_item_files)
for name in mismatch:
self.assertIn(name, IGNORE)
self.assertEqual(len(mismatch), 0)
def mydir(arg, dirname, names):
file = [normpath(tfile) for tfile in names]
fullfilename = [abspath(join(dirname,tfile)) for tfile in names]
#copy it, $file and $fullfilename are one-one correspondence
for i in range(0,len(file)):
if isfile(fullfilename[i]):#only save files
#save WriteIMEI*.ini files
p = re.compile("WriteIMEI.*\.ini")
matchit = p.match(file[i])
if matchit:
#print matchit.group()
list_ini_files.append(fullfilename[i])
continue
if isdir(fullfilename[i]):#only save path
#save cal files's RID
p = re.compile("[0-9A-Fa-f]{32}")
matchit = p.match(fullfilename[i][-32:])
if matchit:
lallCalNumInCal.append(fullfilename[i][-32:])# save RID
#here we need to compare with DEFAULT cal file under \Z\NVRAM\CALIBRAT\
#check DAUL DATA
dmatch, dmismatch, derrors = filecmp.cmpfiles(str(fullfilename[i])+"\\Z\\NVRAM\\CALIBRAT", "REFCALIBRAT\\DUAL", DUALCalFileList, shallow=False)
qmatch, qmismatch, qerrors = filecmp.cmpfiles(str(fullfilename[i])+"\\Z\\NVRAM\\CALIBRAT", "REFCALIBRAT\\QUAD", QUADCalFileList, shallow=False)
#print fullfilename[i][-32:]+"\n"
#print 'DMatch:', dmatch, '\n'
#print 'DMismatch:', dmismatch, '\n'
#print 'DErrors:', derrors, '\n'
#print 'QMatch:', qmatch, '\n'
#print 'QMismatch:', qmismatch, '\n'
#print 'QErrors:', qerrors, '\n'
if(len(dmatch) == 4):
lDUALDefaultCalByChipID.append(fullfilename[i][-32:])
if(len(qmatch) == 4):
lQUADDefaultCalByChipID.append(fullfilename[i][-32:])
if(len(derrors) == 0 and len(qerrors) == 0):
lIsQUADCalByChipID.append(fullfilename[i][-32:])
elif(len(derrors) == 0 and len(qerrors) == 4):
lIsDUALCalByChipID.append(fullfilename[i][-32:])
else:
lIsNULLBandCalByChipID.append(fullfilename[i][-32:])
continue
def _test_export_one_table(self, table_name):
output_temp_dir = tempfile.mkdtemp(dir=self.temp_dir)
optional_args = ['-c', os.path.join(self.temp_dir, str(self.year)), '-o', output_temp_dir, '-t', table_name]
ForkProcess().fork_new_process(self.export_from_cache_opus_path,
resources = None,
optional_args = optional_args)
files = [os.path.splitext(os.path.split(f)[1])[0] for f in glob(output_temp_dir + '/*')]
self.assertEqual( set(files), set([table_name]))
export_year = str(self.year + 100)
optional_args = ['-d', output_temp_dir, '-c', self.temp_dir, '-y', export_year, '-t', table_name]
ForkProcess().fork_new_process(self.export_to_cache_opus_path,
resources = None,
optional_args = optional_args)
exported_datasets = [os.path.split(f)[1] for f in glob(os.path.join(self.temp_dir, export_year) + '/*')]
self.assertEqual( set(exported_datasets), set([table_name]))
org_dir = os.path.join(self.temp_dir, str(self.year))
exp_dir = os.path.join(self.temp_dir, export_year)
flt_file_names = os.listdir(os.path.join(org_dir, table_name))
self.assertEqual( cmpfiles(os.path.join(org_dir, table_name),
os.path.join(exp_dir, table_name),
flt_file_names),
(flt_file_names, [], [] )
)
rmtree(output_temp_dir)
rmtree(exp_dir)
def _compareResultFolders(compare, tmpdir, pid, ignore_ftypes=('.log', '.txt', '.frcmod')):
import filecmp
def _cutfirstline(infile, outfile):
# Cut out the first line of prmtop which has a build date in it
with open(infile, 'r') as fin:
data = fin.read().splitlines(True)
with open(outfile, 'w') as fout:
fout.writelines(data[1:])
files = []
deletefiles = []
for f in glob(join(compare, '*')):
fname = os.path.basename(f)
if os.path.splitext(f)[1] in ignore_ftypes:
continue
if f.endswith('prmtop'):
_cutfirstline(f, join(compare, fname + '.mod'))
_cutfirstline(join(tmpdir, fname), os.path.join(tmpdir, fname + '.mod'))
files.append(os.path.basename(f) + '.mod')
deletefiles.append(join(compare, fname + '.mod'))
else:
files.append(os.path.basename(f))
match, mismatch, error = filecmp.cmpfiles(tmpdir, compare, files, shallow=False)
if len(mismatch) != 0 or len(error) != 0 or len(match) != len(files):
raise RuntimeError(
'Different results produced by amber.build for test {} between {} and {} in files {}.'.format(pid, compare, tmpdir, mismatch))
for f in deletefiles:
os.remove(f)
def cmpTwoLogDir(logPath1,logPath2,dataCardNames):
if not(os.path.isdir(logPath1)):
print color.red('Log directory ' + logPath1 + ' does not exist - EXIT!')
sys.exit(1)
if not(os.path.isdir(logPath2)):
print color.red('Log directory ' + logPath2 + ' does not exist - EXIT!')
sys.exit(1)
logFiles = []
for card in dataCardNames:
logFiles.append(card+'.log')
cmps = filecmp.cmpfiles(logPath1,logPath2,logFiles)
print color.green('=====================================================')
print color.green('Files which are the same:')
for file in cmps[0]:
print color.green(file)
print color.green('=====================================================')
print
print color.red('=====================================================')
print color.red('Files which are NOT the same:')
for file in cmps[1]:
print color.red(file)
print color.red('=====================================================')
print
print color.yellow('=====================================================')
print color.yellow('Files which script was not able to compare:')
for file in cmps[2]:
print color.yellow(file)
print color.yellow('=====================================================')
print
def test_cpp_variables(self):
'''
Test C++ variables generation
'''
cpp = CppFile('var.cpp')
variables = [CppVariable(name="var1",
type="char*",
is_class_member=False,
is_static=False,
is_const=True,
initialization_value='0'),
CppVariable(name="var2",
type="int",
is_class_member=False,
is_static=True,
is_const=False,
initialization_value='0'),
CppVariable(name="var3",
type="std::string",
is_class_member=False,
is_static=False,
is_const=False),
CppVariable(name="var3",
type="std::string",
is_class_member=False,
is_static=False,
is_const=False)]
for var in variables:
var.render_to_string(cpp)
self.assertTrue(filecmp.cmpfiles('.', 'tests', 'var.cpp'))
cpp.close()
def are_dir_trees_equal(dir1, dir2):
"""
Compare two directories recursively. Files in each directory are
assumed to be equal if their names and contents are equal.
@param dir1: First directory path
@param dir2: Second directory path
@return: True if the directory trees are the same and
there were no errors while accessing the directories or files,
False otherwise.
source: http://stackoverflow.com/questions/4187564/recursive-dircmp-compare-two-directories-to-ensure-they-have-the-same-files-and
"""
dirs_cmp = filecmp.dircmp(dir1, dir2)
if len(dirs_cmp.left_only)>0 or len(dirs_cmp.right_only)>0 or \
len(dirs_cmp.funny_files)>0:
return False
(_, mismatch, errors) = filecmp.cmpfiles(
dir1, dir2, dirs_cmp.common_files, shallow=False)
if len(mismatch)>0 or len(errors)>0:
return False
for common_dir in dirs_cmp.common_dirs:
new_dir1 = os.path.join(dir1, common_dir)
new_dir2 = os.path.join(dir2, common_dir)
if not are_dir_trees_equal(new_dir1, new_dir2):
return False
return True
def _sync_dir(temp_dir, file_names):
"""Sync the in ``temp_dir`` with the corresponding files under ``_rst_base_dir``. The list of
files should match ``file_names``."""
rel_dir = os.path.relpath(temp_dir, _rst_temp_dir)
source_dir = os.path.join(_rst_temp_dir, rel_dir)
target_dir = os.path.join(_rst_base_dir, rel_dir)
assert os.path.isdir(source_dir)
if os.path.isdir(target_dir):
source_files = _get_files_in_dir(source_dir)
target_files = _get_files_in_dir(target_dir)
for file in source_files:
source_file_path = os.path.join(source_dir, file)
target_file_path = os.path.join(target_dir, file)
if not os.path.isfile(target_file_path):
# Copy missing files.
shutil.copy2(source_file_path, target_file_path)
# Copy mismatched files.
elif not filecmp.cmp(source_file_path, target_file_path, shallow=False):
shutil.copy2(source_file_path, target_file_path)
for file in target_files:
source_file_path = os.path.join(source_dir, file)
target_file_path = os.path.join(target_dir, file)
if not os.path.isfile(source_file_path):
# Delete files in target that shouldn't be there.
os.remove(target_file_path)
else:
shutil.copytree(source_dir, target_dir)
# Make sure everything matches now.
match, mismatch, error = filecmp.cmpfiles(source_dir, target_dir, file_names, shallow=False)
assert len(match) == len(file_names) and len(mismatch) == 0 and len(error) == 0
def are_dir_trees_equal(dir1, dir2, ignore=None): """ Compare two directories recursively. Files in each directory are assumed to be equal if their names and contents are equal. @param dir1: First directory path @param dir2: Second directory path @param ignore: list of names to ignore (none are ignored by default) @return: True if the directory trees are the same and there were no errors while accessing the directories or files, False otherwise. """ dirs_cmp = filecmp.dircmp(dir1, dir2, ignore=ignore) if len(dirs_cmp.left_only)>0 or len(dirs_cmp.right_only)>0 or \ len(dirs_cmp.funny_files)>0: return False (_, mismatch, errors) = filecmp.cmpfiles( dir1, dir2, dirs_cmp.common_files, shallow=False) if len(mismatch)>0 or len(errors)>0: return False for common_dir in dirs_cmp.common_dirs: new_dir1 = os.path.join(dir1, common_dir) new_dir2 = os.path.join(dir2, common_dir) if not are_dir_trees_equal(new_dir1, new_dir2, ignore=ignore): return False return True
def compare_scenario_outputs(model_code, scenario):
# get the test files
test_scenario_output_folder = paths.scenario_output_folder(model_code, scenario)
test_scenario_output_files = os.listdir(test_scenario_output_folder)
# extract bz2 and gz archives
for output_file in test_scenario_output_files:
if output_file.endswith(".bz2"):
subprocess.run(["bzip2", "-d", "-f", test_scenario_output_folder + output_file])
if output_file.endswith(".gz"):
gz_decompression(test_scenario_output_folder + output_file)
# get the reference files
scenario_expected_outputs_folder = (
paths.scenario_folder(model_code, scenario) + REFERENCE_OUTPUTS_FOLDER_NAME + "/"
)
expected_output_files_list = os.listdir(scenario_expected_outputs_folder)
# compare the files
match, mismatch, errors = filecmp.cmpfiles(
test_scenario_output_folder, scenario_expected_outputs_folder, expected_output_files_list
)
# raise errors if mismatch or error
if len(mismatch) != 0:
raise ValueError("Mismatches in the output files {}".format(mismatch))
if len(errors) != 0:
raise ValueError("Errors in the comparison of files {}".format(errors))
def compare_trees(test_class, first, second):
diff = dircmp(first, second)
test_class.assertEquals([], diff.diff_files)
# Need special code to compare links
if len(diff.common_funny) > 0:
for filename in diff.common_funny:
first_file = path.join(first, filename)
second_file = path.join(second, filename)
if path.islink(first_file) and path.islink(second_file):
test_class.assertEquals(readlink(first_file),
readlink(second_file))
else:
test_class.fail('common_funny files was not empty!' \
'\n%s != %s' % (first_file, second_file))
test_class.assertEquals([], diff.left_only)
test_class.assertEquals([], diff.right_only)
files_to_compare = []
for root, directories, files in walk(first):
for cmp_file in files:
files_to_compare.append(path.join(root, cmp_file))
# Strip target file prefixes
files_to_compare = [name[len(first)+1:] for name in files_to_compare]
_, mismatch, error = cmpfiles(first, second, files_to_compare)
test_class.assertEquals([], mismatch)
def test_equilibration(self):
from htmd.home import home
import filecmp
from htmd.util import tempname
from glob import glob
from htmd.molecule.molecule import Molecule
tmpdir = tempname()
pdbid = '3PTB'
builddir = home(dataDir=os.path.join('test-acemd', pdbid, 'build'))
equil = Acemd('equilibration')
mol = Molecule(os.path.join(builddir, 'structure.pdb'))
celldim = mol.coords.max(axis=0) - mol.coords.min(axis=0)
equil.celldimension = ' '.join(['{:3.1f}'.format(val) for val in celldim.squeeze()])
equil.run = '2000'
equil.trajectoryfreq = 200
equil.temperature = 300
equil.write(builddir, tmpdir)
# Compare with reference
refdir = home(dataDir=os.path.join('test-acemd', pdbid, 'equil'))
files = [os.path.basename(f) for f in glob(os.path.join(refdir, '*'))]
match, mismatch, error = filecmp.cmpfiles(refdir, tmpdir, files, shallow=False)
if len(mismatch) != 0 or len(error) != 0 or len(match) != len(files):
raise RuntimeError('Different results produced by Acemd equilibration for '
'test {} between {} and {} in files {}.'.format(pdbid, refdir, tmpdir, mismatch))
def _compareDirectories(dir1, dir2):
"""
From: http://stackoverflow.com/a/6681395
Compare two directories recursively. Files in each directory are
assumed to be equal if their names and contents are equal.
@param dir1: First directory path
@param dir2: Second directory path
@return: True if the directory trees are the same and
there were no errors while accessing the directories or files,
False otherwise.
"""
dirs_cmp = filecmp.dircmp(dir1, dir2)
if len(dirs_cmp.left_only)>0 or len(dirs_cmp.right_only)>0 or \
len(dirs_cmp.funny_files)>0:
return False
(_, mismatch, errors) = filecmp.cmpfiles(
dir1, dir2, dirs_cmp.common_files, shallow=False)
if len(mismatch)>0 or len(errors)>0:
return False
for common_dir in dirs_cmp.common_dirs:
new_dir1 = os.path.join(dir1, common_dir)
new_dir2 = os.path.join(dir2, common_dir)
if not _compareDirectories(new_dir1, new_dir2):
return False
return True
def run_test(module, test_output_dir, benchmark_dir, controller_args):
test_name = module.name.split('.')[-1]
test_file = os.path.abspath(module.filename)
# Run the test.
test = TestCase(test_name, test_file, test_output_dir,
module.process_mininet_output, module.process_controller_output,
module.get_controller(), controller_args)
files = test.run()
files = [os.path.basename(f) for f in files]
# Compare the output.
match, mismatch, errors = filecmp.cmpfiles(benchmark_dir, test_output_dir, files)
if mismatch:
for fname in mismatch:
fname1 = os.path.join(benchmark_dir, fname)
fname2 = os.path.join(test_output_dir, fname)
diffname = os.path.join(test_output_dir, '%s.diff' % fname)
f1 = file(fname1, 'r')
f2 = file(fname2, 'r')
fdiff = file(diffname, 'w')
sys.stderr.write('--- Diff: %s vs. %s ---\n' %
(os.path.basename(fname1), os.path.basename(fname2)))
d = difflib.Differ()
diff = difflib.ndiff(f1.readlines(), f2.readlines())
for line in diff:
sys.stderr.write(line)
fdiff.write(line)
f1.close()
f2.close()
fdiff.close()
assert mismatch == []
assert errors == []
assert match == files
def are_dir_trees_equal(dir1, dir2):
"""
Compare two directories recursively. Files in each directory are
assumed to be equal if their names and contents are equal.
@param dir1: First directory path
@param dir2: Second directory path
@return: True if the directory trees are the same and
there were no errors while accessing the directories or files,
False otherwise.
"""
# compare file lists in both dirs. If found different lists
# or "funny" files (failed to compare) - return false
dirs_cmp = filecmp.dircmp(dir1, dir2)
if len(dirs_cmp.left_only) > 0 or len(dirs_cmp.right_only) > 0 or \
len(dirs_cmp.funny_files) > 0:
return False
# compare the common files between dir1 and dir2
(match, mismatch, errors) = filecmp.cmpfiles(
dir1, dir2, dirs_cmp.common_files, shallow=False)
if len(mismatch) > 0 or len(errors) > 0:
return False
# continue to compare sub-directories, recursively
for common_dir in dirs_cmp.common_dirs:
new_dir1 = os.path.join(dir1, common_dir)
new_dir2 = os.path.join(dir2, common_dir)
if not PluginInstallerTestCase.are_dir_trees_equal(
new_dir1, new_dir2):
return False
return True
def test_basic(self):
create_bundle(self.DATA['source'], self.DATA['filename'], '1234')
extract_bundle(self.DATA['filename'], self.DATA['output'], '1234')
common = ['main.sh', 'readme.txt']
result, _, _ = filecmp.cmpfiles(self.DATA['source'], self.DATA['output'], common)
self.assertEqual(result, common)
def directories_are_same(left_directory, right_directory):
"""Check recursively whether two directories contain the same files.
Based on https://stackoverflow.com/a/6681395
Keyword arguments:
left_directory -- one of the two directories to compare
right_directory -- the other directory to compare
"""
filecmp.clear_cache()
directory_comparison = filecmp.dircmp(a=left_directory, b=right_directory)
if (len(directory_comparison.left_only) > 0
or len(directory_comparison.right_only) > 0
or len(directory_comparison.funny_files) > 0):
return False
filecmp.clear_cache()
(_, mismatch,
errors) = filecmp.cmpfiles(a=left_directory,
b=right_directory,
common=directory_comparison.common_files,
shallow=False)
if len(mismatch) > 0 or len(errors) > 0:
return False
for common_dir in directory_comparison.common_dirs:
if not directories_are_same(
left_directory=left_directory.joinpath(common_dir),
right_directory=right_directory.joinpath(common_dir)):
return False
return True
def test_files_to_dir_copy_with_hard_links(self):
""" copy files in one folder to another, with hard-linking,
files' inodes should be the same."""
test_folder = prepare_test_folder("test files to dir copy with hard links")
originals_folder = safe_makedirs(os.path.join(test_folder, "originals"))
hard_links_folder = safe_makedirs(os.path.join(test_folder, "hard-links"))
originals_file_paths = create_random_files(originals_folder)
copy_command = self.ps_helper.copy_tool.copy_dir_files_to_dir(originals_folder, hard_links_folder, link_dest=True)
subprocess.check_output(copy_command, stdin=None, stderr=None, shell=True, universal_newlines=False)
hard_link_file_paths = []
for file_path in originals_file_paths:
folder, file_name = os.path.split(file_path)
hard_link_file_paths.append(os.path.join(hard_links_folder, file_name))
for hard_link_file_path in hard_link_file_paths:
self.assertTrue(os.path.isfile(hard_link_file_path))
comperer = filecmp.dircmp(originals_folder, hard_links_folder)
self.assertEqual(comperer.left_only, [], "Some files where not copied: {comperer.left_only}".format(**locals()))
self.assertEqual(comperer.right_only, [], "Extra files where copied: {comperer.right_only}".format(**locals()))
match, mismatch, errors = filecmp.cmpfiles(originals_folder, hard_links_folder, comperer.common, shallow=False)
self.assertEqual(errors, [], "some files are missing {errors}".format(**locals()))
self.assertEqual(mismatch, [], "some files are different {mismatch}".format(**locals()))
self.assertTrue(self.check_indoes_Equal(originals_folder, hard_links_folder, inspect.stack()[0][3]))
def are_dir_trees_equal(dir1, dir2):
"""
Compare two directories recursively. Files in each directory are
assumed to be equal if their names and contents are equal.
@param dir1: First directory path
@param dir2: Second directory path
@return: True if the directory trees are the same and
there were no errors while accessing the directories or files,
False otherwise.
"""
# compare file lists in both dirs. If found different lists
# or "funny" files (failed to compare) - return false
dirs_cmp = filecmp.dircmp(dir1, dir2)
if len(dirs_cmp.left_only) > 0 or len(dirs_cmp.right_only) > 0 or \
len(dirs_cmp.funny_files) > 0:
return False
# compare the common files between dir1 and dir2
(match, mismatch, errors) = filecmp.cmpfiles(dir1,
dir2,
dirs_cmp.common_files,
shallow=False)
if len(mismatch) > 0 or len(errors) > 0:
return False
# continue to compare sub-directories, recursively
for common_dir in dirs_cmp.common_dirs:
new_dir1 = os.path.join(dir1, common_dir)
new_dir2 = os.path.join(dir2, common_dir)
if not PluginInstallerTestCase.are_dir_trees_equal(
new_dir1, new_dir2):
return False
return True
def _poor_mans_rsync(self, source_directory, destination_directory):
# Do not delete the server directory while copying!
assert_readable_directory(source_directory)
source_files = [
entry for entry in os.listdir(source_directory)
if os.path.isfile(os.path.join(source_directory, entry))
]
assert_readable_directory(destination_directory)
destination_files = [
entry for entry in os.listdir(destination_directory)
if os.path.isfile(os.path.join(destination_directory, entry))
]
# First remove all destination files that are missing in the source.
for filename in destination_files:
if filename not in source_files:
LOG.info("Removing file '%s' from destination" % filename)
os.remove(os.path.join(destination_directory, filename))
# Compare files across source and destination.
(match, mismatch, error) = filecmp.cmpfiles(source_directory,
destination_directory,
source_files)
for filename in match:
LOG.info("Skipping file '%s' because it matches" % filename)
for filename in mismatch:
LOG.info("Copying file '%s' due to mismatch" % filename)
shutil.copy2(os.path.join(source_directory, filename),
destination_directory)
for filename in error:
LOG.info("Copying file '%s' because it is missing" % filename)
shutil.copy2(os.path.join(source_directory, filename),
destination_directory)
def phase3(self):
"""
Find out differences between common files.
Ensure we are using content comparison, not os.stat-only.
"""
comp = filecmp.cmpfiles(self.left, self.right, self.common_files, shallow=False)
self.same_files, self.diff_files, self.funny_files = comp
def assert_directories_equal(dir_a, dir_b):
"""
Check recursively directories have equal content.
:param dir_a: first directory to check
:param dir_b: second directory to check
"""
dirs_cmp = filecmp.dircmp(dir_a, dir_b)
assert (
len(dirs_cmp.left_only) == 0
), f"Found files: {dirs_cmp.left_only} in {dir_a}, but not {dir_b}."
assert (
len(dirs_cmp.right_only) == 0
), f"Found files: {dirs_cmp.right_only} in {dir_b}, but not {dir_a}."
assert (
len(dirs_cmp.funny_files) == 0
), f"Found files: {dirs_cmp.funny_files} in {dir_a}, {dir_b} which could not be compared."
(_, mismatch, errors) = filecmp.cmpfiles(dir_a, dir_b, dirs_cmp.common_files, shallow=False)
assert len(mismatch) == 0, f"Found mismatches: {mismatch} between {dir_a} {dir_b}."
assert len(errors) == 0, f"Found errors: {errors} between {dir_a} {dir_b}."
for common_dir in dirs_cmp.common_dirs:
inner_a = os.path.join(dir_a, common_dir)
inner_b = os.path.join(dir_b, common_dir)
assert_directories_equal(inner_a, inner_b)
def report_difference(casedirpath):
# get the directories to be compared
refpath = os.path.join(casedirpath, "ref")
outpath = os.path.join(casedirpath, "out")
if not os.path.isdir(refpath):
print "Test case has no reference directory"
return
if not os.path.isdir(refpath):
print "Test case has no output directory"
return
# check for recursive subdirectories
if len(filter(lambda fn: os.path.isdir(fn), os.listdir(refpath))) > 0:
print "Reference directory contains a subdirectory"
return
if len(filter(lambda fn: os.path.isdir(fn), os.listdir(outpath))) > 0:
print "Output directory contains a sub directory"
return
# verify list of filenames
dircomp = filecmp.dircmp(outpath, refpath, ignore=['.DS_Store'])
if (len(dircomp.left_only) > 0):
print "Output contains " + str(len(dircomp.left_only)) + " extra file(s)"
if (len(dircomp.right_only) > 0):
print "Output misses " + str(len(dircomp.right_only)) + " file(s)"
# compare common files
matches, mismatches, errors = filecmp.cmpfiles(outpath, refpath, dircomp.common, shallow=False)
for filename in matches:
print "Output file matches: " + filename
for filename in mismatches + errors:
if equalfiles(os.path.join(outpath, filename), os.path.join(refpath, filename)):
print "Output file matches: " + filename
else:
print "Output file differs: " + filename + " <-------"
def run_test(module, test_output_dir, benchmark_dir, controller_args):
test_name = module.name.split('.')[-1]
test_file = os.path.abspath(module.filename)
# Run the test.
test = TestCase(test_name, test_file, test_output_dir,
module.process_mininet_output,
module.process_controller_output, module.get_controller(),
controller_args)
files = test.run()
files = [os.path.basename(f) for f in files]
# Compare the output.
match, mismatch, errors = filecmp.cmpfiles(benchmark_dir, test_output_dir,
files)
if mismatch:
for fname in mismatch:
fname1 = os.path.join(benchmark_dir, fname)
fname2 = os.path.join(test_output_dir, fname)
diffname = os.path.join(test_output_dir, '%s.diff' % fname)
f1 = file(fname1, 'r')
f2 = file(fname2, 'r')
fdiff = file(diffname, 'w')
sys.stderr.write(
'--- Diff: %s vs. %s ---\n' %
(os.path.basename(fname1), os.path.basename(fname2)))
d = difflib.Differ()
diff = difflib.ndiff(f1.readlines(), f2.readlines())
for line in diff:
sys.stderr.write(line)
fdiff.write(line)
f1.close()
f2.close()
fdiff.close()
assert mismatch == []
assert errors == []
assert match == files
def are_dir_trees_equal(dir1, dir2):
"""
Compare two directories recursively. Files in each directory are
assumed to be equal if their names and contents are equal.
@param dir1: First directory path
@param dir2: Second directory path
@return: True if the directory trees are the same and
there were no errors while accessing the directories or files,
False otherwise.
"""
dirs_cmp = filecmp.dircmp(dir1, dir2)
if (len(dirs_cmp.left_only) > 0 or len(dirs_cmp.right_only) > 0 or
len(dirs_cmp.funny_files) > 0):
return False
(_, mismatch, errors) = filecmp.cmpfiles(
dir1, dir2, dirs_cmp.common_files, shallow=False)
if len(mismatch) > 0 or len(errors) > 0:
return False
for common_dir in dirs_cmp.common_dirs:
new_dir1 = os.path.join(dir1, common_dir)
new_dir2 = os.path.join(dir2, common_dir)
if not are_dir_trees_equal(new_dir1, new_dir2):
return False
return True
def test(self):
self.update_json("slackware", "12.0")
self.update_json("slackware", "12.1")
self.update_json("slackware", "13.0")
self.update_json("slackware64", "13.0")
self.update_json("slackware", "13.1")
self.update_json("slackware64", "13.1")
self.update_json("slackware", "13.37")
self.update_json("slackware64", "13.37")
self.update_json("slackware", "14.0")
self.update_json("slackware64", "14.0")
self.update_json("slackware", "14.1")
self.update_json("slackware64", "14.1")
self.update_json("slackware", "14.2")
self.update_json("slackware64", "14.2")
self.update_json("slackware", "current")
self.update_json("slackware64", "current")
base_rss = os.listdir(self.baseline)
match, mismatch, error = filecmp.cmpfiles(self.baseline, self.output, base_rss, False)
self.assertEqual(len(base_rss), len(match))
self.assertEqual(0, len(mismatch))
self.assertEqual(0, len(error))
def test_01(self):
CommandMerge.main(fastqinfo=self.fastqinfo, fastqdir=self.fastqdir, fastainfo=self.fastainfo,
fastadir=self.fastadir)
self.assertTrue(filecmp.cmp(self.fastainfo, self.fastainfo_bak, shallow=True))
self.assertTrue(filecmp.cmpfiles(self.fastadir, self.fastadir_bak, common=[
'MFZR_14Ben01_1_fw_48.fasta.gz', 'MFZR_14Ben01_2_fw_48.fasta'], shallow=True))
def get_current_status_vs_destdir(bookroot, bookfiles, destdir):
# convert posix paths to os specific paths
book_home = pathof(bookroot).replace("/", os.sep)
dest_path = pathof(destdir).replace("/", os.sep)
# convert from bookpaths to os relative file paths
filepaths = []
for bkpath in bookfiles:
afile = pathof(bkpath)
afile = afile.replace("/", os.sep)
filepaths.append(afile)
if "mimetype" in filepaths:
filepaths.remove("mimetype")
repolist = walk_folder(dest_path)
# determine what has been deleted
deleted = []
for fpath in repolist:
if fpath not in filepaths:
deleted.append(fpath)
if "mimetype" in deleted:
deleted.remove("mimetype")
# now use pythons built in filecmp to determine added and modified
(unchanged, modified, added) = filecmp.cmpfiles(dest_path,
book_home,
filepaths,
shallow=False)
# convert everything back to posix style bookpaths
for i in range(len(deleted)):
deleted[i] = deleted[i].replace(os.sep, "/")
for i in range(len(added)):
added[i] = added[i].replace(os.sep, "/")
for i in range(len(modified)):
modified[i] = modified[i].replace(os.sep, "/")
return (deleted, added, modified)
def _doVersionComparison(self):
# get directories to compare
for scanDir in self.scanDirs:
pathRepoScanDir = self.pathWPRepoVersion + '/' + scanDir
self.listSubDirs.append(pathRepoScanDir)
for path, subdirs, files in os.walk(pathRepoScanDir):
for subDir in subdirs:
subDirFix = path + '/' + subDir
self.listSubDirs.append(subDirFix.replace('\\', '/'))
# do directory comparison
for subdir in self.listSubDirs:
pathCmpRepo = subdir
pathCmpSite = self.pathWPWorkingVersion + pathCmpRepo.replace(self.pathWPRepoVersion, '')
cmpResults = filecmp.dircmp(pathCmpRepo, pathCmpSite)
for right_only in cmpResults.right_only:
self.compareResults['siteOnly'].append(subdir.replace(self.pathWPRepoVersion, '') + '/' + right_only)
for left_only in cmpResults.left_only:
self.compareResults['repoOnly'].append(subdir.replace(self.pathWPRepoVersion, '') + '/' + left_only)
for diff_file in cmpResults.diff_files:
self.compareResults['altered'].append(subdir.replace(self.pathWPRepoVersion, '') + '/' + diff_file)
# do file comparisons
fileCompareResult = filecmp.cmpfiles(self.pathWPRepoVersion, self.pathWPWorkingVersion, self.scanFiles, False)
for fileRepoOnly in fileCompareResult[2]:
self.compareResults['repoOnly'].append('/' + fileRepoOnly)
for fileAltered in fileCompareResult[1]:
self.compareResults['altered'].append('/' + fileAltered)
def test_writeDict2File(self):
tmpDir = utils.makeTempDir(dir=self.testDataDir)
tmpDict = {'key1':set([1,2,3,4]), 'key2':set([-1,-2,-3,-4])}
trainByJaccard.writeDict2File(tmpDir, tmpDict)
rtn = filecmp.cmpfiles(tmpDir, os.path.join(self.testDir, 'data'), ['key1', 'key2'])
self.assertTrue(rtn)
def fileCompare(emailaddr):
# Determine the items that exist in both directories
d1_contents = set(
os.listdir(
'//mnt/c/Users/Matthew\ Houy/Documents/Python/FedRAMPParser/FedRAMPCompare'
))
d2_contents = set(
os.listdir(
'//mnt/c/Users/Matthew\ Houy/Documents/Python/FedRAMPParser/FedRAMPAuthoritative'
))
common = list(d1_contents & d2_contents)
common_files = [
f for f in common if os.path.isfile(
os.path.join(
'//mnt/c/Users/Matthew\ Houy/Documents/Python/FedRAMPParser/FedRAMPCompare',
f))
]
# Determine the item that exist in one directory, but not the other
uncommon = list(d1_contents ^ d2_contents)
# Compare the directories
match, mismatch, errors = filecmp.cmpfiles(
'//mnt/c/Users/Matthew\ Houy/Documents/Python/FedRAMPParser/FedRAMPCompare',
'//mnt/c/Users/Matthew\ Houy/Documents/Python/FedRAMPParseFedRAMPAuthoritative',
common_files,
shallow=False)
# Create strings for email and printing
matchstr = "\n\t".join(match)
mismatchstr = "\n\t".join(mismatch)
uncommonstr = "\n\t".join(uncommon)
errorstr = "\n\t".join(errors)
# Print comparison output
print 'Match:', matchstr
print 'Mismatch:', mismatchstr
print 'Uncommon:', uncommonstr
print 'Errors:', errorstr
if mismatch or uncommon or errors:
fromaddr = '*****@*****.**'
toaddrs = emailaddr
msg = "\r\n".join([
"From: [email protected]", "",
"Subject: FedRAMP GUIDANCE ALERT", "",
"The following files have been altered:", "", mismatchstr, "",
"The following files have been added or removed as guidance:", "",
uncommonstr, "", "The following files have caused an error:", "",
errorstr, ""
])
username = '******'
password = '******'
server = smtplib.SMTP('smtp.gmail.com:587')
server.ehlo()
server.starttls()
server.login(username, password)
server.sendmail(fromaddr, toaddrs, msg)
server.quit()
def are_dir_trees_equal(dir1, dir2):
"""
Compare two directories recursively. Files in each directory are
assumed to be equal if their names and contents are equal.
@param dir1: First directory path
@param dir2: Second directory path
@return: True if the directory trees are the same and
there were no errors while accessing the directories or files,
False otherwise.
# http://stackoverflow.com/questions/4187564/recursive-dircmp-compare-two-directories-to-ensure-they-have-the-same-files-and/6681395#6681395
"""
dirs_cmp = filecmp.dircmp(dir1, dir2)
if len(dirs_cmp.left_only) > 0 or len(dirs_cmp.right_only) > 0 or \
len(dirs_cmp.funny_files) > 0:
print('LEFT', dirs_cmp.left_only)
print('RIGHT', dirs_cmp.right_only)
print('FUNNY', dirs_cmp.funny_files)
return False
(_, mismatch, errors) = filecmp.cmpfiles(
dir1, dir2, dirs_cmp.common_files, shallow=False)
if len(mismatch) > 0 or len(errors) > 0:
print(mismatch)
print(errors)
return False
for common_dir in dirs_cmp.common_dirs:
new_dir1 = os.path.join(dir1, common_dir)
new_dir2 = os.path.join(dir2, common_dir)
if not are_dir_trees_equal(new_dir1, new_dir2):
return False
return True
def test_dtw_synth(self):
"""Simple characterization test for dtw_synth."""
uttIds = readUttIds(join('test_data', 'corpus.lst'))
with TempDir() as tempDir:
synthOutDir = tempDir.location
p = subprocess.Popen([
join('bin', 'dtw_synth'),
'--exts', 'mgc,lf0,bap',
'--param_orders', '40,1,5',
join('test_data', 'ref-examples'),
join('test_data', 'synth-examples'),
synthOutDir,
] + uttIds, stdout=PIPE, stderr=PIPE)
stdout, stderr = p.communicate()
stdoutGood = (
'processing cmu_us_arctic_slt_a0003\n'
'MCD = 6.175330 (641 frames)\n'
'warping 683 frames -> 641 frames (29 repeated, 71 dropped)\n'
'\n'
'processing cmu_us_arctic_slt_a0044\n'
'MCD = 5.577534 (613 frames)\n'
'warping 653 frames -> 613 frames (35 repeated, 75 dropped)\n'
'\n'
'overall MCD = 5.883106 (1254 frames)\n'
)
self.assertEqual(stderr, '')
self.assertEqual(stdout, stdoutGood)
synthOutDirGood = join('test_data', 'out-dtw_synth')
filenames = [ '%s.mgc' % uttId for uttId in uttIds ]
match, mismatch, errors = cmpfiles(synthOutDir, synthOutDirGood,
filenames, shallow = False)
self.assertEqual(match, filenames)
self.assertFalse(mismatch)
self.assertFalse(errors)
def phase3(self):
""" Find out differences between common files, with shallow=False """
x = filecmp.cmpfiles(self.left,
self.right,
self.common_files,
shallow=False)
self.same_files, self.diff_files, self.funny_files = x
def compare_dirs(dir1: Text, dir2: Text):
"""Recursively compares contents of the two directories.
Args:
dir1: path to a directory.
dir2: path to another directory.
Returns:
a boolean whether the specified directories have the same file contents.
"""
dir_cmp = filecmp.dircmp(dir1, dir2)
if not all(not v for v in (dir_cmp.left_only, dir_cmp.right_only,
dir_cmp.funny_files)):
return False
_, mismatch, errors = filecmp.cmpfiles(dir1,
dir2,
dir_cmp.common_files,
shallow=False)
if mismatch or errors:
return False
for common_dir in dir_cmp.common_dirs:
new_dir1 = os.path.join(dir1, common_dir)
new_dir2 = os.path.join(dir2, common_dir)
if not compare_dirs(new_dir1, new_dir2):
return False
return True
def _compareResultFolders(compare, tmpdir, pid):
ignore_ftypes = ('.log', '.txt')
files = []
deletefiles = []
for f in glob(join(compare, '*')):
fname = os.path.basename(f)
if os.path.splitext(f)[1] in ignore_ftypes:
continue
if f.endswith('prmtop'):
_cutfirstline(f, join(compare, fname + '.mod'))
_cutfirstline(join(tmpdir, fname),
os.path.join(tmpdir, fname + '.mod'))
files.append(os.path.basename(f) + '.mod')
deletefiles.append(join(compare, fname + '.mod'))
else:
files.append(os.path.basename(f))
match, mismatch, error = filecmp.cmpfiles(tmpdir,
compare,
files,
shallow=False)
if len(mismatch) != 0 or len(error) != 0 or len(match) != len(files):
raise RuntimeError(
'Different results produced by amber.build for test {} between {} and {} in files {}.'
.format(pid, compare, tmpdir, mismatch))
for f in deletefiles:
os.remove(f)
def test_backup_verbose(self):
"""Test of method Engine.backup()."""
archive_dir = self.engine.backup(source=self.source,
backup_dir=self.backup,
verbose=True,
suffix=u'-тест')
self.assertEqual(os.path.abspath(self.backup),
os.path.abspath(os.path.join(archive_dir,
os.path.pardir)))
archive = os.path.join(archive_dir, 'source.tar.bz2')
self.assertTrue(os.path.exists(archive))
self.assertEqual(u'Backup started.\n'
u'----------------------\n'
u'Making archive directory in '
u'{backup}/{date}-тест\n\n'
u'Performing backups\n'
u'----------------------\n'
u'Backup of {source}\n'
u'----------------------\n'
u'All backups complete!\n'.format(backup=self.backup,
date=date.today(),
source=self.source),
self.out.data)
self.assertEqual('', self.err.data)
args = ['/bin/tar', '-xjf', archive, '-C', self.restore]
with open(os.devnull, 'w') as out:
self.assertEqual(0, subprocess.call(args, stdout=out))
rest_dir = os.path.join(self.restore, os.path.basename(self.source))
self.assertTrue(filecmp.cmpfiles(self.source, rest_dir,
self.filelist, shallow=False))
def _walk(self, oldpath, newpath):
oldfiles = []
newfiles = []
includes = eval(Config.get("actions", "include"))
excludes = eval(Config.get("actions", "exclude"))
for root, dirs, files in os.walk(oldpath):
relpath = path.relpath(root, oldpath)
if relpath == ".":
relpath = ""
for filename in files:
filepath = path.join(relpath, filename)
if (any(map(lambda glob: fnmatch(filepath, glob), excludes))):
continue
if (any(map(lambda glob: fnmatch(filepath, glob), includes))):
oldfiles.append(filepath)
for root, dirs, files in os.walk(newpath):
relpath = path.relpath(root, newpath)
if relpath == ".":
relpath = ""
for filename in files:
filepath = path.join(relpath, filename)
if (any(map(lambda glob: fnmatch(filepath, glob), excludes))):
continue
if (any(map(lambda glob: fnmatch(filepath, glob), includes))):
newfiles.append(filepath)
unmodified, modified, removed = filecmp.cmpfiles(oldpath, newpath,
oldfiles, shallow=False)
unmodified, modified, added = filecmp.cmpfiles(oldpath, newpath,
newfiles, shallow=False)
unmodified = slashes(unmodified)
modified = slashes(modified)
added = slashes(added)
removed = slashes(removed)
return unmodified, modified, added, removed
def test_integration(msgsteiner):
parser = OptionParser()
parser.add_option('--msgpath',dest='msgsteiner',type='string',help='Path to msgsteiner9 code, be sure to include!')
phos_weights = os.path.join(os.path.dirname(__file__), '..', 'example', 'a549', 'Tgfb_phos.txt')
#provided config file
garnet_conf = os.path.join(os.path.dirname(__file__), '..', 'example', 'a549', 'tgfb_garnet.cfg')
#forest requires more inputs
forest_conf = os.path.join(os.path.dirname(__file__), '..', 'example', 'a549', 'tgfb_forest.cfg') #provided config file
edge_file = os.path.join(os.path.dirname(__file__), '..', 'data', 'iref_mitab_miscore_2013_08_12_interactome.txt') #interactom
#WE NEED MSGSTEINER9 INSTALLED!!!
msgsteinerpath = msgsteiner
if msgsteinerpath == None:
print 'Please provide path to msgsteiner using --msgpath option'
assert 0
# Create a tmp directory for output
forest_out = tempfile.mkdtemp()
#Arbitrary value
seed = 2
try:
script_dir = os.path.dirname(__file__)
forest_path = os.path.join(script_dir, '..', 'scripts', 'forest.py')
fcmd='python %s --prize=%s --edge=%s --conf=%s --outpath=%s --msgpath=%s --seed=%s'%(
forest_path, phos_weights, edge_file, forest_conf, forest_out, msgsteinerpath, seed)
subprocess.call(shlex.split(fcmd), shell=False)
curr_dir = os.path.dirname(__file__)
match, mismatch, errors = filecmp.cmpfiles(forest_out, os.path.join(curr_dir, 'integration_test_standard'),
['result_augmentedForest.sif', 'result_dummyForest.sif', 'result_edgeattributes.tsv',
'result_info.txt', 'result_nodeattributes.tsv', 'result_optimalForest.sif'],
shallow=False)
if len(match) != 6:
print 'Mismatching files: ', mismatch
print 'Errors: ', errors
assert 0, 'Not all Forest output files match'
else:
assert 1
except IOError as e:
print 'IO error'
finally:
shutil.rmtree(forest_out)
def compare(self, p=""):
d1 = os.path.join(self.dir1, p)
d2 = os.path.join(self.dir2, p)
dcmp = filecmp.dircmp(d1, d2, ignore=[])
self.left_only.extend(os.path.join(p, n) for n in dcmp.left_only)
self.right_only.extend(os.path.join(p, n) for n in dcmp.right_only)
self.common_funny.extend(os.path.join(p, n) for n in dcmp.common_funny)
self.funny_files.extend(os.path.join(p, n) for n in dcmp.funny_files)
(match, mismatch, errors) = filecmp.cmpfiles(d1, d2, dcmp.common_files, shallow=False)
self.diff_files.extend(os.path.join(p, n) for n in mismatch)
self.funny_files.extend(os.path.join(p, n) for n in errors)
for d in dcmp.common_dirs:
self.compare(os.path.join(p, d))
def test_cpp_function(self):
'''
Test C++ functions generation
'''
cpp = CppFile('func.cpp')
hpp = CppFile('func.h')
def function_body(_, cpp1):
cpp1('return 42;')
functions = [CppFunction(name='GetParam', ret_type='int'),
CppFunction(name='Calculate', ret_type='void'),
CppFunction(name='GetAnswer', ret_type='int', implementation_handle=function_body)]
for func in functions:
func.render_to_string(hpp)
for func in functions:
func.render_to_string_declaration(hpp)
for func in functions:
func.render_to_string_implementation(cpp)
self.assertTrue(filecmp.cmpfiles('.', 'tests', 'func.cpp'))
self.assertTrue(filecmp.cmpfiles('.', 'tests', 'func.h'))
cpp.close()
hpp.close()
def diff(d1, d2):
# Determine the items that exist in both directories
d1_contents = set(os.listdir(d1))
d2_contents = set(os.listdir(d2))
common = list(d1_contents & d2_contents)
common_files = [f for f in common if os.path.isfile(os.path.join(d1, f))]
print "Common files:", common_files
# Compare the directories
match, mismatch, errors = filecmp.cmpfiles(d1, d2, common_files)
print "Match:", match
print "Mismatch:", mismatch
print "Errors:", errors
def equal_dirs(self, dir1, dir2, ignore=[]):
"""Checks equality between dir1 and dir2, ignores files in 'ignore' list."""
dirs_cmp = filecmp.dircmp(dir1, dir2, ignore)
if (len(dirs_cmp.left_only) > 0 or len(dirs_cmp.right_only) > 0 or
len(dirs_cmp.funny_files) > 0):
return False
(_, mismatch, errors) = filecmp.cmpfiles(dir1, dir2, dirs_cmp.common_files, shallow=False)
if len(mismatch) > 0 or len(errors) > 0:
return False
for common_dir in dirs_cmp.common_dirs:
new_dir1 = os.path.join(dir1, common_dir)
new_dir2 = os.path.join(dir2, common_dir)
if not self.equal_dirs(new_dir1, new_dir2, ignore):
return False
return True
def test_patch(self):
retval = self.cherry.patch(
base_sha='7bee46915e2a29ac789ac18b43dcdb438377bfbd',
target_sha='b23e9fd3a3518faf703d9cc90c3cafb079e4488d',
target_branch='test_branch')
self.assertTrue(retval)
(match, mismatch, errors) = filecmp.cmpfiles(
self.cherry.files_base,
self.patch_dir,
['README.md'])
self.assertEqual(len(match), 1)
self.assertEqual(len(mismatch), 0)
self.assertEqual(len(errors), 0)
