def test_make_unstructured_grid(tmpdir): """ Test the make_unstructured_grid routine.""" mesh = IO.GmshMesh() mesh.read("/".join((DATA_DIR, 'Structured.msh'))) print("/".join((DATA_DIR, 'Structured.msh'))) num = len(mesh.nodes) vel = numpy.zeros((num, 3)) pres = numpy.zeros((num, )) time = 0 ugrid = IO.make_unstructured_grid( mesh, vel, pres, time, outfile=tmpdir.join('Structured.vtu').strpath) assert num > 0 assert ugrid.GetNumberOfPoints() == num assert ugrid.GetNumberOfCells() == len(mesh.elements) assert filecmp.cmpfiles(DATA_DIR, tmpdir.strpath, 'Structured.vtu') ugrid = IO.make_unstructured_grid( mesh, lambda x: (0, 0, 0), lambda x: 0, time, outfile=tmpdir.join('Structured.vtu').strpath) assert ugrid.GetNumberOfPoints() == num assert ugrid.GetNumberOfCells() == len(mesh.elements) assert filecmp.cmpfiles(DATA_DIR, tmpdir.strpath, 'Structured.vtu')
def test_cmpfiles(self, comparator_tmpdir): assert filecmp.cmpfiles( comparator_tmpdir.dir, comparator_tmpdir.dir, ["file"], ) == (["file"], [], []), "Comparing directory to itself fails" assert filecmp.cmpfiles( comparator_tmpdir.dir, comparator_tmpdir.dir_same, ["file"], ) == (["file"], [], []), "Comparing directory to same fails" # Try it with shallow=False assert filecmp.cmpfiles( comparator_tmpdir.dir, comparator_tmpdir.dir, ["file"], shallow=False, ) == (["file"], [], []), "Comparing directory to itself fails" assert filecmp.cmpfiles( comparator_tmpdir.dir, comparator_tmpdir.dir_same, ["file"], shallow=False, ), "Comparing directory to same fails" # Add different file2 with open(os.path.join(comparator_tmpdir.dir, "file2"), 'w') as output: output.write('Different contents.\n') assert filecmp.cmpfiles( comparator_tmpdir.dir, comparator_tmpdir.dir_same, ["file", "file2"], ) != (["file"], ["file2"], []), "Comparing mismatched directories fails"
def test_cmpfiles(self): self.failUnless(filecmp.cmpfiles(self.dir, self.dir, ['file']) == (['file'], [], []), "Comparing directory to itself fails") self.failUnless(filecmp.cmpfiles(self.dir, self.dir_same, ['file']) == (['file'], [], []), "Comparing directory to same fails") # Try it with shallow=False self.failUnless(filecmp.cmpfiles(self.dir, self.dir, ['file'], shallow=False) == (['file'], [], []), "Comparing directory to itself fails") self.failUnless(filecmp.cmpfiles(self.dir, self.dir_same, ['file'], shallow=False), "Comparing directory to same fails") # Add different file2 output = open(os.path.join(self.dir, 'file2'), 'w') output.write('Different contents.\n') output.close() self.failIf(filecmp.cmpfiles(self.dir, self.dir_same, ['file', 'file2']) == (['file'], ['file2'], []), "Comparing mismatched directories fails")
def test_cmpfiles(self): self.assertTrue( filecmp.cmpfiles(self.dir, self.dir, ['file']) == (['file'], [], []), "Comparing directory to itself fails") self.assertTrue( filecmp.cmpfiles(self.dir, self.dir_same, ['file']) == (['file'], [], []), "Comparing directory to same fails") # Try it with shallow=False self.assertTrue( filecmp.cmpfiles(self.dir, self.dir, ['file'], shallow=False) == (['file'], [], []), "Comparing directory to itself fails") self.assertTrue( filecmp.cmpfiles(self.dir, self.dir_same, ['file'], shallow=False), "Comparing directory to same fails") # Add different file2 with open(os.path.join(self.dir, 'file2'), 'w', encoding="utf-8") as output: output.write('Different contents.\n') self.assertFalse( filecmp.cmpfiles(self.dir, self.dir_same, ['file', 'file2']) == (['file'], ['file2'], []), "Comparing mismatched directories fails")
def test_cmpfiles(self): self.assertTrue( filecmp.cmpfiles(self.dir, self.dir, ["file"]) == (["file"], [], []), "Comparing directory to itself fails" ) self.assertTrue( filecmp.cmpfiles(self.dir, self.dir_same, ["file"]) == (["file"], [], []), "Comparing directory to same fails", ) # Try it with shallow=False self.assertTrue( filecmp.cmpfiles(self.dir, self.dir, ["file"], shallow=False) == (["file"], [], []), "Comparing directory to itself fails", ) self.assertTrue( filecmp.cmpfiles(self.dir, self.dir_same, ["file"], shallow=False), "Comparing directory to same fails" ) # Add different file2 output = open(os.path.join(self.dir, "file2"), "w") output.write("Different contents.\n") output.close() self.assertFalse( filecmp.cmpfiles(self.dir, self.dir_same, ["file", "file2"]) == (["file"], ["file2"], []), "Comparing mismatched directories fails", )
def test_cmpfiles(self): self.failUnless( filecmp.cmpfiles(self.dir, self.dir, ['file']) == (['file'], [], []), "Comparing directory to itself fails") self.failUnless( filecmp.cmpfiles(self.dir, self.dir_same, ['file']) == (['file'], [], []), "Comparing directory to same fails") # Try it with shallow=False self.failUnless( filecmp.cmpfiles(self.dir, self.dir, ['file'], shallow=False) == (['file'], [], []), "Comparing directory to itself fails") self.failUnless( filecmp.cmpfiles(self.dir, self.dir_same, ['file'], shallow=False), "Comparing directory to same fails") # Add different file2 output = open(os.path.join(self.dir, 'file2'), 'w') output.write('Different contents.\n') output.close() self.failIf( filecmp.cmpfiles(self.dir, self.dir_same, ['file', 'file2']) == (['file'], ['file2'], []), "Comparing mismatched directories fails")
def test_make_unstructured_grid(tmpdir): """ Test the make_unstructured_grid routine.""" mesh = IO.GmshMesh() mesh.read("/".join((DATA_DIR, 'Structured.msh'))) print "/".join((DATA_DIR, 'Structured.msh')) num = len(mesh.nodes) vel = numpy.zeros((num, 3)) pres = numpy.zeros((num,)) time = 0 ugrid = IO.make_unstructured_grid(mesh, vel, pres, time, outfile=tmpdir.join('Structured.vtu').strpath) assert num > 0 assert ugrid.GetNumberOfPoints() == num assert ugrid.GetNumberOfCells() == len(mesh.elements) assert filecmp.cmpfiles(DATA_DIR, tmpdir.strpath, 'Structured.vtu') ugrid = IO.make_unstructured_grid(mesh, lambda x: (0, 0, 0), lambda x: 0, time, outfile=tmpdir.join('Structured.vtu').strpath) assert ugrid.GetNumberOfPoints() == num assert ugrid.GetNumberOfCells() == len(mesh.elements) assert filecmp.cmpfiles(DATA_DIR, tmpdir.strpath, 'Structured.vtu')
def test_run_container(validation_folders: Dict, host_folders: Dict, docker_client: docker.DockerClient, docker_image_key: str, container_variables: Dict): # copy file to input folder host_folders["input"].rmdir() shutil.copytree(validation_folders["input"], host_folders["input"]) assert Path(host_folders["input"]).exists() # run the container (this may take some time) docker_client.containers.run(docker_image_key, "run", detach=False, remove=True, volumes = { host_folders["input"] : { 'bind' : container_variables["INPUT_FOLDER"], 'mode': 'ro' }, host_folders["output"] : { 'bind' : container_variables["OUTPUT_FOLDER"], 'mode': 'rw' }, host_folders["log"] : { 'bind': container_variables["LOG_FOLDER"], 'mode': 'rw' } }, environment=container_variables) for folder in ["input", "output", "log"]: # test if the files that should be there are actually there and correct list_of_files = [x.name for x in validation_folders[folder].iterdir()] match, mismatch, errors = filecmp.cmpfiles(host_folders[folder], validation_folders[folder], list_of_files, shallow=False) assert not mismatch, "wrong/incorrect files in {}".format(host_folders[folder]) assert not errors, "missing files in {}".format(host_folders[folder]) # test if the files that are there are matching the ones that should be list_of_files = [x.name for x in host_folders[folder].iterdir()] match, mismatch, errors = filecmp.cmpfiles(host_folders[folder], validation_folders[folder], list_of_files, shallow=False) assert not mismatch, "wrong/incorrect generated files in {}".format(host_folders[folder]) assert not errors, "too many files in {}".format(host_folders[folder])
def test_single_reads(self): sams = [ os.path.join('unit_test', 'data', 'single_reads_genome1.sam'), os.path.join('unit_test', 'data', 'single_reads_genome2.sam') ] out_dir = os.path.join('unit_test', 'output') files_to_check = ['sorted1.sam', 'sorted2.sam'] #'supplementary_output.txt' # no multiprocessing sparta.sparta(sams, num_processes=1, output_dir=os.path.join(out_dir, 'no_mp'), separated_samfiles=[ os.path.join(out_dir, 'no_mp', 'sorted1.sam'), os.path.join(out_dir, 'no_mp', 'sorted2.sam') ]) # processes = cpu_count sparta.sparta(sams, output_dir=os.path.join(out_dir, 'mp'), separated_samfiles=[ os.path.join(out_dir, 'mp', 'sorted1.sam'), os.path.join(out_dir, 'mp', 'sorted2.sam') ]) # processes = 10 sparta.sparta(sams, num_processes=10, output_dir=os.path.join(out_dir, 'mp10'), separated_samfiles=[ os.path.join(out_dir, 'mp10', 'sorted1.sam'), os.path.join(out_dir, 'mp10', 'sorted2.sam') ]) match, mismatch, errors = filecmp.cmpfiles(os.path.join( out_dir, 'no_mp'), os.path.join(out_dir, 'mp'), common=files_to_check) match2, mismatch2, errors2 = filecmp.cmpfiles( os.path.join(out_dir, 'no_mp'), os.path.join(out_dir, 'mp10'), common=files_to_check) # check that output was created, and no files mismatched, and no errors occured assert match != [] assert mismatch == [] assert errors == [] assert match2 != [] assert mismatch2 == [] assert errors2 == [] # clean up shutil.rmtree(out_dir) os.makedirs(out_dir)
def test_load_save_game(self): print "" print "TEST load_game functionality, first start a new game." print "Make some changes, save the game, start another new game." print "Then load game and check that the changes we made are as expected." files.new_game() rooms = name_lists.room_info().get_titles() items = name_lists.item_info().get_titles() new_rooms_dir = name_lists.save_info().get_temp_save_dir_rooms() new_items_dir = name_lists.save_info().get_temp_save_dir_items() save_rooms_dir = name_lists.save_info().get_save_dir_rooms() save_items_dir = name_lists.save_info().get_save_dir_items() print "Open each room and edit the visited attribute" for title in rooms: room = files.load_room(title) room['visited'] = "" files.store_room(room) print "Open each item and edit the active attribute" for title in items: item = files.load_item(title) if item['active']: item['active'] = False else: item['active'] = True files.store_item(item) title = random.choice(rooms) current_room = files.load_room(title) files.save_game(self.player, current_room, 0) files.new_game() print "TEST compare the files in temp and save and confirm no match" match, mismatch, errors = filecmp.cmpfiles(save_rooms_dir, new_rooms_dir, rooms) for file_ in match: self.assertNotIn(file_, rooms) self.assertEqual(len(match), 0) self.assertEqual(len(errors), 0) match, mismatch, errors = filecmp.cmpfiles(save_items_dir, new_items_dir, items) for file_ in match: self.assertNotIn(file_, items) self.assertEqual(len(match), 0) self.assertEqual(len(errors), 0) print "TEST that player file exists and that it is a JSON dict" save_dir = name_lists.save_info().get_save_dir() player_file = os.path.join(save_dir, 'player') self.assertEqual(os.path.isfile(player_file), True) try: with open(player_file, 'r') as player_file: player = json.load(player_file) except Exception, e: player = None
def mydir(arg, dirname, names): file = [normpath(tfile) for tfile in names] fullfilename = [abspath(join(dirname, tfile)) for tfile in names] #copy it, $file and $fullfilename are one-one correspondence for i in range(0, len(file)): if isfile(fullfilename[i]): #only save files #save WriteIMEI*.ini files p = re.compile("WriteIMEI.*\.ini") matchit = p.match(file[i]) if matchit: #print matchit.group() list_ini_files.append(fullfilename[i]) continue if isdir(fullfilename[i]): #only save path #save cal files's RID p = re.compile("[0-9A-Fa-f]{32}") matchit = p.match(fullfilename[i][-32:]) if matchit: lallCalNumInCal.append(fullfilename[i][-32:]) # save RID #here we need to compare with DEFAULT cal file under \Z\NVRAM\CALIBRAT\ #check DAUL DATA dmatch, dmismatch, derrors = filecmp.cmpfiles( str(fullfilename[i]) + "\\Z\\NVRAM\\CALIBRAT", "REFCALIBRAT\\DUAL", DUALCalFileList, shallow=False) qmatch, qmismatch, qerrors = filecmp.cmpfiles( str(fullfilename[i]) + "\\Z\\NVRAM\\CALIBRAT", "REFCALIBRAT\\QUAD", QUADCalFileList, shallow=False) #print fullfilename[i][-32:]+"\n" #print 'DMatch:', dmatch, '\n' #print 'DMismatch:', dmismatch, '\n' #print 'DErrors:', derrors, '\n' #print 'QMatch:', qmatch, '\n' #print 'QMismatch:', qmismatch, '\n' #print 'QErrors:', qerrors, '\n' if (len(dmatch) == 4): lDUALDefaultCalByChipID.append(fullfilename[i][-32:]) if (len(qmatch) == 4): lQUADDefaultCalByChipID.append(fullfilename[i][-32:]) if (len(derrors) == 0 and len(qerrors) == 0): lIsQUADCalByChipID.append(fullfilename[i][-32:]) elif (len(derrors) == 0 and len(qerrors) == 4): lIsDUALCalByChipID.append(fullfilename[i][-32:]) else: lIsNULLBandCalByChipID.append(fullfilename[i][-32:]) continue
def get_diffs(gold_base, test_base, gold_root): """Diff test output with the golden directory. Args: gold_base: Golden (known-good) directory path. test_base: Test output directory path. gold_root: Returned paths are relative to this path. Returns: matches: Matching files. mismatches: Different files. missings: Files in golden root, missing in test_base. extras: Files not in golden root, extra in test_base. """ # Walk the golden tree testing for differences with the test tree. all_matches = [] all_mismatches = [] all_missings = [] for gold_dir, _, file_list in os.walk(gold_base): if not file_list: continue test_dir = join_path(test_base, norm_relpath(gold_dir, gold_base)) (matches, mismatches, errors) = filecmp.cmpfiles(gold_dir, test_dir, file_list, shallow=False) rel_dir = norm_relpath(gold_dir, gold_root) for name in matches: all_matches.append(join_path(rel_dir, name)) for name in mismatches: rel_file_path = join_path(rel_dir, name) gold_path = join_path(gold_dir, name) test_path = join_path(test_dir, name) (_, ext) = os.path.splitext(name) # TODO: USDC (and there by extension USDZ) output may not be # deterministic, so we should use the usddiff tool in the USD library to # perform deep compares when there's a binary mismatch. if ext != '.usdz' or not compare_usdz_content( gold_path, test_path): all_mismatches.append(rel_file_path) for name in errors: all_missings.append(join_path(rel_dir, name)) # Walk the test tree to find new files not in the golden tree. all_extras = [] for test_dir, _, file_list in os.walk(test_base): if not file_list: continue gold_dir = join_path(gold_base, norm_relpath(test_dir, test_base)) (_, _, errors) = filecmp.cmpfiles(gold_dir, test_dir, file_list) rel_dir = norm_relpath(gold_dir, gold_root) for name in errors: all_extras.append(join_path(rel_dir, name)) return (all_matches, all_mismatches, all_missings, all_extras)
def test_cpp_class(self): generate_class(output_dir='.') expected_cpp = ['class.cpp'] expected_h = ['class.h'] self.assertEqual( filecmp.cmpfiles('.', 'test_assets', expected_cpp)[0], expected_cpp) self.assertEqual( filecmp.cmpfiles('.', 'test_assets', expected_h)[0], expected_h) os.remove(expected_cpp[0]) os.remove(expected_h[0])
def test_disease_gene_example_hetmat_archiving(tmpdir): """ Test archiving the hetmat corresponding to the hetnet in Figure 2C at https://doi.org/crz8. """ tmpdir = pathlib.Path(tmpdir) graph = get_graph('disease-gene-example') hetmat_0_dir = tmpdir.joinpath('disease-gene-example-0.hetmat') hetmat = hetmatpy.hetmat.hetmat_from_graph(graph, hetmat_0_dir) # Test creating archive archive_path = hetmatpy.hetmat.archive.create_hetmat_archive(hetmat) with zipfile.ZipFile(archive_path) as zip_file: name_list = zip_file.namelist() expected = [ 'edges/DlT.sparse.npz', 'edges/GaD.sparse.npz', 'edges/GeT.sparse.npz', 'edges/GiG.sparse.npz', 'metagraph.json', 'nodes/Disease.tsv', 'nodes/Gene.tsv', 'nodes/Tissue.tsv', ] assert name_list == expected # Test round-tripped hetmat has same files hetmat_1_dir = tmpdir.joinpath('disease-gene-example-1.hetmat') hetmatpy.hetmat.archive.load_archive(archive_path, hetmat_1_dir) match, mismatch, errors = filecmp.cmpfiles(hetmat_0_dir, hetmat_1_dir, common=expected, shallow=False) assert match == expected assert not mismatch assert not errors # Test round-tripped hetmat has same files when specifying a # subset of zip members in load_archive with source_paths. hetmat_2_dir = tmpdir.joinpath('disease-gene-example-2.hetmat') extract_only = [ 'edges/GiG.sparse.npz', 'metagraph.json', 'nodes/Disease.tsv', ] hetmatpy.hetmat.archive.load_archive(archive_path, hetmat_2_dir, source_paths=extract_only) match, mismatch, errors = filecmp.cmpfiles(hetmat_0_dir, hetmat_2_dir, common=expected, shallow=False) assert match == extract_only assert not mismatch assert 'nodes/Tissue.tsv' in errors
def test_run_container(validation_folders: Dict, host_folders: Dict, docker_container: docker.models.containers.Container): for folder in _FOLDER_NAMES: # test if the files that should be there are actually there and correct list_of_files = [ x.name for x in validation_folders[folder].iterdir() if not ".gitkeep" in x.name] for file_name in list_of_files: assert Path(host_folders[folder] / file_name).exists( ), f"{file_name} is missing from {host_folders[folder]}" # we look for missing files only. contents is the responsibility of the service creator _, _, errors = filecmp.cmpfiles( host_folders[folder], validation_folders[folder], list_of_files, shallow=True) assert not errors, f"{errors} are missing in {host_folders[folder]}" if folder == "input": continue # test if the generated files are the ones expected list_of_files = [ x.name for x in host_folders[folder].iterdir() if not ".gitkeep" in x.name] for file_name in list_of_files: assert Path(validation_folders[folder] / file_name).exists( ), "{} is not present in {}".format(file_name, validation_folders[folder]) _, _, errors = filecmp.cmpfiles( host_folders[folder], validation_folders[folder], list_of_files, shallow=False) # assert not mismatch, "wrong/incorrect generated files in {}".format(host_folders[folder]) assert not errors, f"{errors} should not be available in {host_folders[folder]}" # check the output is correct based on container labels output_cfg = {} output_cfg_file = Path(host_folders["output"] / "outputs.json") if output_cfg_file.exists(): with output_cfg_file.open() as fp: output_cfg = json.load(fp) container_labels = docker_container.labels io_simcore_labels = _convert_to_simcore_labels(container_labels) assert "outputs" in io_simcore_labels for key, value in io_simcore_labels["outputs"].items(): assert "type" in value # rationale: files are on their own and other types are in inputs.json if not "data:" in value["type"]: # check that keys are available assert key in output_cfg else: # it's a file and it should be in the folder as well using key as the filename filename_to_look_for = key if "fileToKeyMap" in value: # ...or there is a mapping assert len(value["fileToKeyMap"]) > 0 for filename, mapped_value in value["fileToKeyMap"].items(): assert mapped_value == key filename_to_look_for = filename assert (host_folders["output"] / filename_to_look_for).exists()
def test_save_game(self): print "" print "TEST the save game of file_lib" files.new_game() rooms = name_lists.room_info().get_titles() items = name_lists.item_info().get_titles() title = random.choice(rooms) current_room = files.load_room(title) new_rooms_dir = name_lists.save_info().get_temp_save_dir_rooms() new_items_dir = name_lists.save_info().get_temp_save_dir_items() save_rooms_dir = name_lists.save_info().get_save_dir_rooms() save_items_dir = name_lists.save_info().get_save_dir_items() print "Open each room and edit the visited attribute" for title in rooms: room = files.load_room(title) room['visited'] = True files.store_room(room) print "Open each item and edit the active attribute" for title in items: item = files.load_item(title) if item['active']: item['active'] = False else: item['active'] = True files.store_item(item) files.save_game(self.player, current_room, 0) print "TEST compare the room files in both dirs to check that they match" match, mismatch, errors = filecmp.cmpfiles(save_rooms_dir, new_rooms_dir, rooms) for file_ in match: self.assertIn(file_, rooms) self.assertEqual(len(mismatch), 0) self.assertEqual(len(errors), 0) print "TEST compare the item files in both dirs to check that they match" match, mismatch, errors = filecmp.cmpfiles(save_items_dir, new_items_dir, items) for file_ in match: self.assertIn(file_, items) self.assertEqual(len(mismatch), 0) self.assertEqual(len(errors), 0) print "TEST that player file exists and that it is a JSON dict" save_dir = name_lists.save_info().get_save_dir() player_file = os.path.join(save_dir, 'player') self.assertEqual(os.path.isfile(player_file), True) try: with open(player_file, 'r') as player_file: player = json.load(player_file) except Exception, e: player = None
def test_new_game(self): IGNORE = files.IGNORE rooms_dir = os.path.abspath('data/rooms') items_dir = os.path.abspath('data/items') room_files = os.listdir(rooms_dir) item_files = os.listdir(items_dir) result = files.new_game() new_rooms_dir = name_lists.save_info().get_temp_save_dir_rooms() new_items_dir = name_lists.save_info().get_temp_save_dir_items() new_room_files = os.listdir(new_rooms_dir) new_item_files = os.listdir(new_items_dir) print "" print "TEST that files were transferred" self.assertEqual(result, True) for file_ in new_room_files: print "File: " + file_ + " in temp save is in template" self.assertIn(file_, room_files) for file_ in room_files: if file_ not in IGNORE: print "File: " + file_ + " in temp save is in temp save" self.assertIn(file_, new_room_files) for file_ in new_item_files: print "File: " + file_ + " in temp save is in template" self.assertIn(file_, item_files) for file_ in item_files: if file_ not in IGNORE: print "File: " + file_ + " in temp save is in temp save" self.assertIn(file_, new_item_files) print "TEST compare the room files in both dirs to check that they match" match, mismatch, errors = filecmp.cmpfiles(rooms_dir, new_rooms_dir, room_files) for name in match: self.assertIn(name, room_files) self.assertIn(name, new_room_files) for name in mismatch: self.assertIn(name, IGNORE) self.assertEqual(len(mismatch), 0) print "TEST compare the item files in both dirs to check that they match" match, mismatch, errors = filecmp.cmpfiles(items_dir, new_items_dir, item_files) for name in match: self.assertIn(name, item_files) self.assertIn(name, new_item_files) for name in mismatch: self.assertIn(name, IGNORE) self.assertEqual(len(mismatch), 0)
def mydir(arg, dirname, names): file = [normpath(tfile) for tfile in names] fullfilename = [abspath(join(dirname,tfile)) for tfile in names] #copy it, $file and $fullfilename are one-one correspondence for i in range(0,len(file)): if isfile(fullfilename[i]):#only save files #save WriteIMEI*.ini files p = re.compile("WriteIMEI.*\.ini") matchit = p.match(file[i]) if matchit: #print matchit.group() list_ini_files.append(fullfilename[i]) continue if isdir(fullfilename[i]):#only save path #save cal files's RID p = re.compile("[0-9A-Fa-f]{32}") matchit = p.match(fullfilename[i][-32:]) if matchit: lallCalNumInCal.append(fullfilename[i][-32:])# save RID #here we need to compare with DEFAULT cal file under \Z\NVRAM\CALIBRAT\ #check DAUL DATA dmatch, dmismatch, derrors = filecmp.cmpfiles(str(fullfilename[i])+"\\Z\\NVRAM\\CALIBRAT", "REFCALIBRAT\\DUAL", DUALCalFileList, shallow=False) qmatch, qmismatch, qerrors = filecmp.cmpfiles(str(fullfilename[i])+"\\Z\\NVRAM\\CALIBRAT", "REFCALIBRAT\\QUAD", QUADCalFileList, shallow=False) #print fullfilename[i][-32:]+"\n" #print 'DMatch:', dmatch, '\n' #print 'DMismatch:', dmismatch, '\n' #print 'DErrors:', derrors, '\n' #print 'QMatch:', qmatch, '\n' #print 'QMismatch:', qmismatch, '\n' #print 'QErrors:', qerrors, '\n' if(len(dmatch) == 4): lDUALDefaultCalByChipID.append(fullfilename[i][-32:]) if(len(qmatch) == 4): lQUADDefaultCalByChipID.append(fullfilename[i][-32:]) if(len(derrors) == 0 and len(qerrors) == 0): lIsQUADCalByChipID.append(fullfilename[i][-32:]) elif(len(derrors) == 0 and len(qerrors) == 4): lIsDUALCalByChipID.append(fullfilename[i][-32:]) else: lIsNULLBandCalByChipID.append(fullfilename[i][-32:]) continue
def _test_export_one_table(self, table_name): output_temp_dir = tempfile.mkdtemp(dir=self.temp_dir) optional_args = ['-c', os.path.join(self.temp_dir, str(self.year)), '-o', output_temp_dir, '-t', table_name] ForkProcess().fork_new_process(self.export_from_cache_opus_path, resources = None, optional_args = optional_args) files = [os.path.splitext(os.path.split(f)[1])[0] for f in glob(output_temp_dir + '/*')] self.assertEqual( set(files), set([table_name])) export_year = str(self.year + 100) optional_args = ['-d', output_temp_dir, '-c', self.temp_dir, '-y', export_year, '-t', table_name] ForkProcess().fork_new_process(self.export_to_cache_opus_path, resources = None, optional_args = optional_args) exported_datasets = [os.path.split(f)[1] for f in glob(os.path.join(self.temp_dir, export_year) + '/*')] self.assertEqual( set(exported_datasets), set([table_name])) org_dir = os.path.join(self.temp_dir, str(self.year)) exp_dir = os.path.join(self.temp_dir, export_year) flt_file_names = os.listdir(os.path.join(org_dir, table_name)) self.assertEqual( cmpfiles(os.path.join(org_dir, table_name), os.path.join(exp_dir, table_name), flt_file_names), (flt_file_names, [], [] ) ) rmtree(output_temp_dir) rmtree(exp_dir)
def _compareResultFolders(compare, tmpdir, pid, ignore_ftypes=('.log', '.txt', '.frcmod')): import filecmp def _cutfirstline(infile, outfile): # Cut out the first line of prmtop which has a build date in it with open(infile, 'r') as fin: data = fin.read().splitlines(True) with open(outfile, 'w') as fout: fout.writelines(data[1:]) files = [] deletefiles = [] for f in glob(join(compare, '*')): fname = os.path.basename(f) if os.path.splitext(f)[1] in ignore_ftypes: continue if f.endswith('prmtop'): _cutfirstline(f, join(compare, fname + '.mod')) _cutfirstline(join(tmpdir, fname), os.path.join(tmpdir, fname + '.mod')) files.append(os.path.basename(f) + '.mod') deletefiles.append(join(compare, fname + '.mod')) else: files.append(os.path.basename(f)) match, mismatch, error = filecmp.cmpfiles(tmpdir, compare, files, shallow=False) if len(mismatch) != 0 or len(error) != 0 or len(match) != len(files): raise RuntimeError( 'Different results produced by amber.build for test {} between {} and {} in files {}.'.format(pid, compare, tmpdir, mismatch)) for f in deletefiles: os.remove(f)
def cmpTwoLogDir(logPath1,logPath2,dataCardNames): if not(os.path.isdir(logPath1)): print color.red('Log directory ' + logPath1 + ' does not exist - EXIT!') sys.exit(1) if not(os.path.isdir(logPath2)): print color.red('Log directory ' + logPath2 + ' does not exist - EXIT!') sys.exit(1) logFiles = [] for card in dataCardNames: logFiles.append(card+'.log') cmps = filecmp.cmpfiles(logPath1,logPath2,logFiles) print color.green('=====================================================') print color.green('Files which are the same:') for file in cmps[0]: print color.green(file) print color.green('=====================================================') print print color.red('=====================================================') print color.red('Files which are NOT the same:') for file in cmps[1]: print color.red(file) print color.red('=====================================================') print print color.yellow('=====================================================') print color.yellow('Files which script was not able to compare:') for file in cmps[2]: print color.yellow(file) print color.yellow('=====================================================') print
def test_cpp_variables(self): ''' Test C++ variables generation ''' cpp = CppFile('var.cpp') variables = [CppVariable(name="var1", type="char*", is_class_member=False, is_static=False, is_const=True, initialization_value='0'), CppVariable(name="var2", type="int", is_class_member=False, is_static=True, is_const=False, initialization_value='0'), CppVariable(name="var3", type="std::string", is_class_member=False, is_static=False, is_const=False), CppVariable(name="var3", type="std::string", is_class_member=False, is_static=False, is_const=False)] for var in variables: var.render_to_string(cpp) self.assertTrue(filecmp.cmpfiles('.', 'tests', 'var.cpp')) cpp.close()
def are_dir_trees_equal(dir1, dir2): """ Compare two directories recursively. Files in each directory are assumed to be equal if their names and contents are equal. @param dir1: First directory path @param dir2: Second directory path @return: True if the directory trees are the same and there were no errors while accessing the directories or files, False otherwise. source: http://stackoverflow.com/questions/4187564/recursive-dircmp-compare-two-directories-to-ensure-they-have-the-same-files-and """ dirs_cmp = filecmp.dircmp(dir1, dir2) if len(dirs_cmp.left_only)>0 or len(dirs_cmp.right_only)>0 or \ len(dirs_cmp.funny_files)>0: return False (_, mismatch, errors) = filecmp.cmpfiles( dir1, dir2, dirs_cmp.common_files, shallow=False) if len(mismatch)>0 or len(errors)>0: return False for common_dir in dirs_cmp.common_dirs: new_dir1 = os.path.join(dir1, common_dir) new_dir2 = os.path.join(dir2, common_dir) if not are_dir_trees_equal(new_dir1, new_dir2): return False return True
def _sync_dir(temp_dir, file_names): """Sync the in ``temp_dir`` with the corresponding files under ``_rst_base_dir``. The list of files should match ``file_names``.""" rel_dir = os.path.relpath(temp_dir, _rst_temp_dir) source_dir = os.path.join(_rst_temp_dir, rel_dir) target_dir = os.path.join(_rst_base_dir, rel_dir) assert os.path.isdir(source_dir) if os.path.isdir(target_dir): source_files = _get_files_in_dir(source_dir) target_files = _get_files_in_dir(target_dir) for file in source_files: source_file_path = os.path.join(source_dir, file) target_file_path = os.path.join(target_dir, file) if not os.path.isfile(target_file_path): # Copy missing files. shutil.copy2(source_file_path, target_file_path) # Copy mismatched files. elif not filecmp.cmp(source_file_path, target_file_path, shallow=False): shutil.copy2(source_file_path, target_file_path) for file in target_files: source_file_path = os.path.join(source_dir, file) target_file_path = os.path.join(target_dir, file) if not os.path.isfile(source_file_path): # Delete files in target that shouldn't be there. os.remove(target_file_path) else: shutil.copytree(source_dir, target_dir) # Make sure everything matches now. match, mismatch, error = filecmp.cmpfiles(source_dir, target_dir, file_names, shallow=False) assert len(match) == len(file_names) and len(mismatch) == 0 and len(error) == 0
def are_dir_trees_equal(dir1, dir2, ignore=None): """ Compare two directories recursively. Files in each directory are assumed to be equal if their names and contents are equal. @param dir1: First directory path @param dir2: Second directory path @param ignore: list of names to ignore (none are ignored by default) @return: True if the directory trees are the same and there were no errors while accessing the directories or files, False otherwise. """ dirs_cmp = filecmp.dircmp(dir1, dir2, ignore=ignore) if len(dirs_cmp.left_only)>0 or len(dirs_cmp.right_only)>0 or \ len(dirs_cmp.funny_files)>0: return False (_, mismatch, errors) = filecmp.cmpfiles( dir1, dir2, dirs_cmp.common_files, shallow=False) if len(mismatch)>0 or len(errors)>0: return False for common_dir in dirs_cmp.common_dirs: new_dir1 = os.path.join(dir1, common_dir) new_dir2 = os.path.join(dir2, common_dir) if not are_dir_trees_equal(new_dir1, new_dir2, ignore=ignore): return False return True
def compare_scenario_outputs(model_code, scenario): # get the test files test_scenario_output_folder = paths.scenario_output_folder(model_code, scenario) test_scenario_output_files = os.listdir(test_scenario_output_folder) # extract bz2 and gz archives for output_file in test_scenario_output_files: if output_file.endswith(".bz2"): subprocess.run(["bzip2", "-d", "-f", test_scenario_output_folder + output_file]) if output_file.endswith(".gz"): gz_decompression(test_scenario_output_folder + output_file) # get the reference files scenario_expected_outputs_folder = ( paths.scenario_folder(model_code, scenario) + REFERENCE_OUTPUTS_FOLDER_NAME + "/" ) expected_output_files_list = os.listdir(scenario_expected_outputs_folder) # compare the files match, mismatch, errors = filecmp.cmpfiles( test_scenario_output_folder, scenario_expected_outputs_folder, expected_output_files_list ) # raise errors if mismatch or error if len(mismatch) != 0: raise ValueError("Mismatches in the output files {}".format(mismatch)) if len(errors) != 0: raise ValueError("Errors in the comparison of files {}".format(errors))
def compare_trees(test_class, first, second): diff = dircmp(first, second) test_class.assertEquals([], diff.diff_files) # Need special code to compare links if len(diff.common_funny) > 0: for filename in diff.common_funny: first_file = path.join(first, filename) second_file = path.join(second, filename) if path.islink(first_file) and path.islink(second_file): test_class.assertEquals(readlink(first_file), readlink(second_file)) else: test_class.fail('common_funny files was not empty!' \ '\n%s != %s' % (first_file, second_file)) test_class.assertEquals([], diff.left_only) test_class.assertEquals([], diff.right_only) files_to_compare = [] for root, directories, files in walk(first): for cmp_file in files: files_to_compare.append(path.join(root, cmp_file)) # Strip target file prefixes files_to_compare = [name[len(first)+1:] for name in files_to_compare] _, mismatch, error = cmpfiles(first, second, files_to_compare) test_class.assertEquals([], mismatch)
def test_equilibration(self): from htmd.home import home import filecmp from htmd.util import tempname from glob import glob from htmd.molecule.molecule import Molecule tmpdir = tempname() pdbid = '3PTB' builddir = home(dataDir=os.path.join('test-acemd', pdbid, 'build')) equil = Acemd('equilibration') mol = Molecule(os.path.join(builddir, 'structure.pdb')) celldim = mol.coords.max(axis=0) - mol.coords.min(axis=0) equil.celldimension = ' '.join(['{:3.1f}'.format(val) for val in celldim.squeeze()]) equil.run = '2000' equil.trajectoryfreq = 200 equil.temperature = 300 equil.write(builddir, tmpdir) # Compare with reference refdir = home(dataDir=os.path.join('test-acemd', pdbid, 'equil')) files = [os.path.basename(f) for f in glob(os.path.join(refdir, '*'))] match, mismatch, error = filecmp.cmpfiles(refdir, tmpdir, files, shallow=False) if len(mismatch) != 0 or len(error) != 0 or len(match) != len(files): raise RuntimeError('Different results produced by Acemd equilibration for ' 'test {} between {} and {} in files {}.'.format(pdbid, refdir, tmpdir, mismatch))
def _compareDirectories(dir1, dir2): """ From: http://stackoverflow.com/a/6681395 Compare two directories recursively. Files in each directory are assumed to be equal if their names and contents are equal. @param dir1: First directory path @param dir2: Second directory path @return: True if the directory trees are the same and there were no errors while accessing the directories or files, False otherwise. """ dirs_cmp = filecmp.dircmp(dir1, dir2) if len(dirs_cmp.left_only)>0 or len(dirs_cmp.right_only)>0 or \ len(dirs_cmp.funny_files)>0: return False (_, mismatch, errors) = filecmp.cmpfiles( dir1, dir2, dirs_cmp.common_files, shallow=False) if len(mismatch)>0 or len(errors)>0: return False for common_dir in dirs_cmp.common_dirs: new_dir1 = os.path.join(dir1, common_dir) new_dir2 = os.path.join(dir2, common_dir) if not _compareDirectories(new_dir1, new_dir2): return False return True
def run_test(module, test_output_dir, benchmark_dir, controller_args): test_name = module.name.split('.')[-1] test_file = os.path.abspath(module.filename) # Run the test. test = TestCase(test_name, test_file, test_output_dir, module.process_mininet_output, module.process_controller_output, module.get_controller(), controller_args) files = test.run() files = [os.path.basename(f) for f in files] # Compare the output. match, mismatch, errors = filecmp.cmpfiles(benchmark_dir, test_output_dir, files) if mismatch: for fname in mismatch: fname1 = os.path.join(benchmark_dir, fname) fname2 = os.path.join(test_output_dir, fname) diffname = os.path.join(test_output_dir, '%s.diff' % fname) f1 = file(fname1, 'r') f2 = file(fname2, 'r') fdiff = file(diffname, 'w') sys.stderr.write('--- Diff: %s vs. %s ---\n' % (os.path.basename(fname1), os.path.basename(fname2))) d = difflib.Differ() diff = difflib.ndiff(f1.readlines(), f2.readlines()) for line in diff: sys.stderr.write(line) fdiff.write(line) f1.close() f2.close() fdiff.close() assert mismatch == [] assert errors == [] assert match == files
def are_dir_trees_equal(dir1, dir2): """ Compare two directories recursively. Files in each directory are assumed to be equal if their names and contents are equal. @param dir1: First directory path @param dir2: Second directory path @return: True if the directory trees are the same and there were no errors while accessing the directories or files, False otherwise. """ # compare file lists in both dirs. If found different lists # or "funny" files (failed to compare) - return false dirs_cmp = filecmp.dircmp(dir1, dir2) if len(dirs_cmp.left_only) > 0 or len(dirs_cmp.right_only) > 0 or \ len(dirs_cmp.funny_files) > 0: return False # compare the common files between dir1 and dir2 (match, mismatch, errors) = filecmp.cmpfiles( dir1, dir2, dirs_cmp.common_files, shallow=False) if len(mismatch) > 0 or len(errors) > 0: return False # continue to compare sub-directories, recursively for common_dir in dirs_cmp.common_dirs: new_dir1 = os.path.join(dir1, common_dir) new_dir2 = os.path.join(dir2, common_dir) if not PluginInstallerTestCase.are_dir_trees_equal( new_dir1, new_dir2): return False return True
def test_basic(self): create_bundle(self.DATA['source'], self.DATA['filename'], '1234') extract_bundle(self.DATA['filename'], self.DATA['output'], '1234') common = ['main.sh', 'readme.txt'] result, _, _ = filecmp.cmpfiles(self.DATA['source'], self.DATA['output'], common) self.assertEqual(result, common)
def directories_are_same(left_directory, right_directory): """Check recursively whether two directories contain the same files. Based on https://stackoverflow.com/a/6681395 Keyword arguments: left_directory -- one of the two directories to compare right_directory -- the other directory to compare """ filecmp.clear_cache() directory_comparison = filecmp.dircmp(a=left_directory, b=right_directory) if (len(directory_comparison.left_only) > 0 or len(directory_comparison.right_only) > 0 or len(directory_comparison.funny_files) > 0): return False filecmp.clear_cache() (_, mismatch, errors) = filecmp.cmpfiles(a=left_directory, b=right_directory, common=directory_comparison.common_files, shallow=False) if len(mismatch) > 0 or len(errors) > 0: return False for common_dir in directory_comparison.common_dirs: if not directories_are_same( left_directory=left_directory.joinpath(common_dir), right_directory=right_directory.joinpath(common_dir)): return False return True
def test_files_to_dir_copy_with_hard_links(self): """ copy files in one folder to another, with hard-linking, files' inodes should be the same.""" test_folder = prepare_test_folder("test files to dir copy with hard links") originals_folder = safe_makedirs(os.path.join(test_folder, "originals")) hard_links_folder = safe_makedirs(os.path.join(test_folder, "hard-links")) originals_file_paths = create_random_files(originals_folder) copy_command = self.ps_helper.copy_tool.copy_dir_files_to_dir(originals_folder, hard_links_folder, link_dest=True) subprocess.check_output(copy_command, stdin=None, stderr=None, shell=True, universal_newlines=False) hard_link_file_paths = [] for file_path in originals_file_paths: folder, file_name = os.path.split(file_path) hard_link_file_paths.append(os.path.join(hard_links_folder, file_name)) for hard_link_file_path in hard_link_file_paths: self.assertTrue(os.path.isfile(hard_link_file_path)) comperer = filecmp.dircmp(originals_folder, hard_links_folder) self.assertEqual(comperer.left_only, [], "Some files where not copied: {comperer.left_only}".format(**locals())) self.assertEqual(comperer.right_only, [], "Extra files where copied: {comperer.right_only}".format(**locals())) match, mismatch, errors = filecmp.cmpfiles(originals_folder, hard_links_folder, comperer.common, shallow=False) self.assertEqual(errors, [], "some files are missing {errors}".format(**locals())) self.assertEqual(mismatch, [], "some files are different {mismatch}".format(**locals())) self.assertTrue(self.check_indoes_Equal(originals_folder, hard_links_folder, inspect.stack()[0][3]))
def are_dir_trees_equal(dir1, dir2): """ Compare two directories recursively. Files in each directory are assumed to be equal if their names and contents are equal. @param dir1: First directory path @param dir2: Second directory path @return: True if the directory trees are the same and there were no errors while accessing the directories or files, False otherwise. """ # compare file lists in both dirs. If found different lists # or "funny" files (failed to compare) - return false dirs_cmp = filecmp.dircmp(dir1, dir2) if len(dirs_cmp.left_only) > 0 or len(dirs_cmp.right_only) > 0 or \ len(dirs_cmp.funny_files) > 0: return False # compare the common files between dir1 and dir2 (match, mismatch, errors) = filecmp.cmpfiles(dir1, dir2, dirs_cmp.common_files, shallow=False) if len(mismatch) > 0 or len(errors) > 0: return False # continue to compare sub-directories, recursively for common_dir in dirs_cmp.common_dirs: new_dir1 = os.path.join(dir1, common_dir) new_dir2 = os.path.join(dir2, common_dir) if not PluginInstallerTestCase.are_dir_trees_equal( new_dir1, new_dir2): return False return True
def _poor_mans_rsync(self, source_directory, destination_directory): # Do not delete the server directory while copying! assert_readable_directory(source_directory) source_files = [ entry for entry in os.listdir(source_directory) if os.path.isfile(os.path.join(source_directory, entry)) ] assert_readable_directory(destination_directory) destination_files = [ entry for entry in os.listdir(destination_directory) if os.path.isfile(os.path.join(destination_directory, entry)) ] # First remove all destination files that are missing in the source. for filename in destination_files: if filename not in source_files: LOG.info("Removing file '%s' from destination" % filename) os.remove(os.path.join(destination_directory, filename)) # Compare files across source and destination. (match, mismatch, error) = filecmp.cmpfiles(source_directory, destination_directory, source_files) for filename in match: LOG.info("Skipping file '%s' because it matches" % filename) for filename in mismatch: LOG.info("Copying file '%s' due to mismatch" % filename) shutil.copy2(os.path.join(source_directory, filename), destination_directory) for filename in error: LOG.info("Copying file '%s' because it is missing" % filename) shutil.copy2(os.path.join(source_directory, filename), destination_directory)
def phase3(self): """ Find out differences between common files. Ensure we are using content comparison, not os.stat-only. """ comp = filecmp.cmpfiles(self.left, self.right, self.common_files, shallow=False) self.same_files, self.diff_files, self.funny_files = comp
def assert_directories_equal(dir_a, dir_b): """ Check recursively directories have equal content. :param dir_a: first directory to check :param dir_b: second directory to check """ dirs_cmp = filecmp.dircmp(dir_a, dir_b) assert ( len(dirs_cmp.left_only) == 0 ), f"Found files: {dirs_cmp.left_only} in {dir_a}, but not {dir_b}." assert ( len(dirs_cmp.right_only) == 0 ), f"Found files: {dirs_cmp.right_only} in {dir_b}, but not {dir_a}." assert ( len(dirs_cmp.funny_files) == 0 ), f"Found files: {dirs_cmp.funny_files} in {dir_a}, {dir_b} which could not be compared." (_, mismatch, errors) = filecmp.cmpfiles(dir_a, dir_b, dirs_cmp.common_files, shallow=False) assert len(mismatch) == 0, f"Found mismatches: {mismatch} between {dir_a} {dir_b}." assert len(errors) == 0, f"Found errors: {errors} between {dir_a} {dir_b}." for common_dir in dirs_cmp.common_dirs: inner_a = os.path.join(dir_a, common_dir) inner_b = os.path.join(dir_b, common_dir) assert_directories_equal(inner_a, inner_b)
def report_difference(casedirpath): # get the directories to be compared refpath = os.path.join(casedirpath, "ref") outpath = os.path.join(casedirpath, "out") if not os.path.isdir(refpath): print "Test case has no reference directory" return if not os.path.isdir(refpath): print "Test case has no output directory" return # check for recursive subdirectories if len(filter(lambda fn: os.path.isdir(fn), os.listdir(refpath))) > 0: print "Reference directory contains a subdirectory" return if len(filter(lambda fn: os.path.isdir(fn), os.listdir(outpath))) > 0: print "Output directory contains a sub directory" return # verify list of filenames dircomp = filecmp.dircmp(outpath, refpath, ignore=['.DS_Store']) if (len(dircomp.left_only) > 0): print "Output contains " + str(len(dircomp.left_only)) + " extra file(s)" if (len(dircomp.right_only) > 0): print "Output misses " + str(len(dircomp.right_only)) + " file(s)" # compare common files matches, mismatches, errors = filecmp.cmpfiles(outpath, refpath, dircomp.common, shallow=False) for filename in matches: print "Output file matches: " + filename for filename in mismatches + errors: if equalfiles(os.path.join(outpath, filename), os.path.join(refpath, filename)): print "Output file matches: " + filename else: print "Output file differs: " + filename + " <-------"
def run_test(module, test_output_dir, benchmark_dir, controller_args): test_name = module.name.split('.')[-1] test_file = os.path.abspath(module.filename) # Run the test. test = TestCase(test_name, test_file, test_output_dir, module.process_mininet_output, module.process_controller_output, module.get_controller(), controller_args) files = test.run() files = [os.path.basename(f) for f in files] # Compare the output. match, mismatch, errors = filecmp.cmpfiles(benchmark_dir, test_output_dir, files) if mismatch: for fname in mismatch: fname1 = os.path.join(benchmark_dir, fname) fname2 = os.path.join(test_output_dir, fname) diffname = os.path.join(test_output_dir, '%s.diff' % fname) f1 = file(fname1, 'r') f2 = file(fname2, 'r') fdiff = file(diffname, 'w') sys.stderr.write( '--- Diff: %s vs. %s ---\n' % (os.path.basename(fname1), os.path.basename(fname2))) d = difflib.Differ() diff = difflib.ndiff(f1.readlines(), f2.readlines()) for line in diff: sys.stderr.write(line) fdiff.write(line) f1.close() f2.close() fdiff.close() assert mismatch == [] assert errors == [] assert match == files
def are_dir_trees_equal(dir1, dir2): """ Compare two directories recursively. Files in each directory are assumed to be equal if their names and contents are equal. @param dir1: First directory path @param dir2: Second directory path @return: True if the directory trees are the same and there were no errors while accessing the directories or files, False otherwise. """ dirs_cmp = filecmp.dircmp(dir1, dir2) if (len(dirs_cmp.left_only) > 0 or len(dirs_cmp.right_only) > 0 or len(dirs_cmp.funny_files) > 0): return False (_, mismatch, errors) = filecmp.cmpfiles( dir1, dir2, dirs_cmp.common_files, shallow=False) if len(mismatch) > 0 or len(errors) > 0: return False for common_dir in dirs_cmp.common_dirs: new_dir1 = os.path.join(dir1, common_dir) new_dir2 = os.path.join(dir2, common_dir) if not are_dir_trees_equal(new_dir1, new_dir2): return False return True
def test(self): self.update_json("slackware", "12.0") self.update_json("slackware", "12.1") self.update_json("slackware", "13.0") self.update_json("slackware64", "13.0") self.update_json("slackware", "13.1") self.update_json("slackware64", "13.1") self.update_json("slackware", "13.37") self.update_json("slackware64", "13.37") self.update_json("slackware", "14.0") self.update_json("slackware64", "14.0") self.update_json("slackware", "14.1") self.update_json("slackware64", "14.1") self.update_json("slackware", "14.2") self.update_json("slackware64", "14.2") self.update_json("slackware", "current") self.update_json("slackware64", "current") base_rss = os.listdir(self.baseline) match, mismatch, error = filecmp.cmpfiles(self.baseline, self.output, base_rss, False) self.assertEqual(len(base_rss), len(match)) self.assertEqual(0, len(mismatch)) self.assertEqual(0, len(error))
def test_01(self): CommandMerge.main(fastqinfo=self.fastqinfo, fastqdir=self.fastqdir, fastainfo=self.fastainfo, fastadir=self.fastadir) self.assertTrue(filecmp.cmp(self.fastainfo, self.fastainfo_bak, shallow=True)) self.assertTrue(filecmp.cmpfiles(self.fastadir, self.fastadir_bak, common=[ 'MFZR_14Ben01_1_fw_48.fasta.gz', 'MFZR_14Ben01_2_fw_48.fasta'], shallow=True))
def get_current_status_vs_destdir(bookroot, bookfiles, destdir): # convert posix paths to os specific paths book_home = pathof(bookroot).replace("/", os.sep) dest_path = pathof(destdir).replace("/", os.sep) # convert from bookpaths to os relative file paths filepaths = [] for bkpath in bookfiles: afile = pathof(bkpath) afile = afile.replace("/", os.sep) filepaths.append(afile) if "mimetype" in filepaths: filepaths.remove("mimetype") repolist = walk_folder(dest_path) # determine what has been deleted deleted = [] for fpath in repolist: if fpath not in filepaths: deleted.append(fpath) if "mimetype" in deleted: deleted.remove("mimetype") # now use pythons built in filecmp to determine added and modified (unchanged, modified, added) = filecmp.cmpfiles(dest_path, book_home, filepaths, shallow=False) # convert everything back to posix style bookpaths for i in range(len(deleted)): deleted[i] = deleted[i].replace(os.sep, "/") for i in range(len(added)): added[i] = added[i].replace(os.sep, "/") for i in range(len(modified)): modified[i] = modified[i].replace(os.sep, "/") return (deleted, added, modified)
def _doVersionComparison(self): # get directories to compare for scanDir in self.scanDirs: pathRepoScanDir = self.pathWPRepoVersion + '/' + scanDir self.listSubDirs.append(pathRepoScanDir) for path, subdirs, files in os.walk(pathRepoScanDir): for subDir in subdirs: subDirFix = path + '/' + subDir self.listSubDirs.append(subDirFix.replace('\\', '/')) # do directory comparison for subdir in self.listSubDirs: pathCmpRepo = subdir pathCmpSite = self.pathWPWorkingVersion + pathCmpRepo.replace(self.pathWPRepoVersion, '') cmpResults = filecmp.dircmp(pathCmpRepo, pathCmpSite) for right_only in cmpResults.right_only: self.compareResults['siteOnly'].append(subdir.replace(self.pathWPRepoVersion, '') + '/' + right_only) for left_only in cmpResults.left_only: self.compareResults['repoOnly'].append(subdir.replace(self.pathWPRepoVersion, '') + '/' + left_only) for diff_file in cmpResults.diff_files: self.compareResults['altered'].append(subdir.replace(self.pathWPRepoVersion, '') + '/' + diff_file) # do file comparisons fileCompareResult = filecmp.cmpfiles(self.pathWPRepoVersion, self.pathWPWorkingVersion, self.scanFiles, False) for fileRepoOnly in fileCompareResult[2]: self.compareResults['repoOnly'].append('/' + fileRepoOnly) for fileAltered in fileCompareResult[1]: self.compareResults['altered'].append('/' + fileAltered)
def test_writeDict2File(self): tmpDir = utils.makeTempDir(dir=self.testDataDir) tmpDict = {'key1':set([1,2,3,4]), 'key2':set([-1,-2,-3,-4])} trainByJaccard.writeDict2File(tmpDir, tmpDict) rtn = filecmp.cmpfiles(tmpDir, os.path.join(self.testDir, 'data'), ['key1', 'key2']) self.assertTrue(rtn)
def fileCompare(emailaddr): # Determine the items that exist in both directories d1_contents = set( os.listdir( '//mnt/c/Users/Matthew\ Houy/Documents/Python/FedRAMPParser/FedRAMPCompare' )) d2_contents = set( os.listdir( '//mnt/c/Users/Matthew\ Houy/Documents/Python/FedRAMPParser/FedRAMPAuthoritative' )) common = list(d1_contents & d2_contents) common_files = [ f for f in common if os.path.isfile( os.path.join( '//mnt/c/Users/Matthew\ Houy/Documents/Python/FedRAMPParser/FedRAMPCompare', f)) ] # Determine the item that exist in one directory, but not the other uncommon = list(d1_contents ^ d2_contents) # Compare the directories match, mismatch, errors = filecmp.cmpfiles( '//mnt/c/Users/Matthew\ Houy/Documents/Python/FedRAMPParser/FedRAMPCompare', '//mnt/c/Users/Matthew\ Houy/Documents/Python/FedRAMPParseFedRAMPAuthoritative', common_files, shallow=False) # Create strings for email and printing matchstr = "\n\t".join(match) mismatchstr = "\n\t".join(mismatch) uncommonstr = "\n\t".join(uncommon) errorstr = "\n\t".join(errors) # Print comparison output print 'Match:', matchstr print 'Mismatch:', mismatchstr print 'Uncommon:', uncommonstr print 'Errors:', errorstr if mismatch or uncommon or errors: fromaddr = '*****@*****.**' toaddrs = emailaddr msg = "\r\n".join([ "From: [email protected]", "", "Subject: FedRAMP GUIDANCE ALERT", "", "The following files have been altered:", "", mismatchstr, "", "The following files have been added or removed as guidance:", "", uncommonstr, "", "The following files have caused an error:", "", errorstr, "" ]) username = '******' password = '******' server = smtplib.SMTP('smtp.gmail.com:587') server.ehlo() server.starttls() server.login(username, password) server.sendmail(fromaddr, toaddrs, msg) server.quit()
def are_dir_trees_equal(dir1, dir2): """ Compare two directories recursively. Files in each directory are assumed to be equal if their names and contents are equal. @param dir1: First directory path @param dir2: Second directory path @return: True if the directory trees are the same and there were no errors while accessing the directories or files, False otherwise. # http://stackoverflow.com/questions/4187564/recursive-dircmp-compare-two-directories-to-ensure-they-have-the-same-files-and/6681395#6681395 """ dirs_cmp = filecmp.dircmp(dir1, dir2) if len(dirs_cmp.left_only) > 0 or len(dirs_cmp.right_only) > 0 or \ len(dirs_cmp.funny_files) > 0: print('LEFT', dirs_cmp.left_only) print('RIGHT', dirs_cmp.right_only) print('FUNNY', dirs_cmp.funny_files) return False (_, mismatch, errors) = filecmp.cmpfiles( dir1, dir2, dirs_cmp.common_files, shallow=False) if len(mismatch) > 0 or len(errors) > 0: print(mismatch) print(errors) return False for common_dir in dirs_cmp.common_dirs: new_dir1 = os.path.join(dir1, common_dir) new_dir2 = os.path.join(dir2, common_dir) if not are_dir_trees_equal(new_dir1, new_dir2): return False return True
def test_dtw_synth(self): """Simple characterization test for dtw_synth.""" uttIds = readUttIds(join('test_data', 'corpus.lst')) with TempDir() as tempDir: synthOutDir = tempDir.location p = subprocess.Popen([ join('bin', 'dtw_synth'), '--exts', 'mgc,lf0,bap', '--param_orders', '40,1,5', join('test_data', 'ref-examples'), join('test_data', 'synth-examples'), synthOutDir, ] + uttIds, stdout=PIPE, stderr=PIPE) stdout, stderr = p.communicate() stdoutGood = ( 'processing cmu_us_arctic_slt_a0003\n' 'MCD = 6.175330 (641 frames)\n' 'warping 683 frames -> 641 frames (29 repeated, 71 dropped)\n' '\n' 'processing cmu_us_arctic_slt_a0044\n' 'MCD = 5.577534 (613 frames)\n' 'warping 653 frames -> 613 frames (35 repeated, 75 dropped)\n' '\n' 'overall MCD = 5.883106 (1254 frames)\n' ) self.assertEqual(stderr, '') self.assertEqual(stdout, stdoutGood) synthOutDirGood = join('test_data', 'out-dtw_synth') filenames = [ '%s.mgc' % uttId for uttId in uttIds ] match, mismatch, errors = cmpfiles(synthOutDir, synthOutDirGood, filenames, shallow = False) self.assertEqual(match, filenames) self.assertFalse(mismatch) self.assertFalse(errors)
def phase3(self): """ Find out differences between common files, with shallow=False """ x = filecmp.cmpfiles(self.left, self.right, self.common_files, shallow=False) self.same_files, self.diff_files, self.funny_files = x
def compare_dirs(dir1: Text, dir2: Text): """Recursively compares contents of the two directories. Args: dir1: path to a directory. dir2: path to another directory. Returns: a boolean whether the specified directories have the same file contents. """ dir_cmp = filecmp.dircmp(dir1, dir2) if not all(not v for v in (dir_cmp.left_only, dir_cmp.right_only, dir_cmp.funny_files)): return False _, mismatch, errors = filecmp.cmpfiles(dir1, dir2, dir_cmp.common_files, shallow=False) if mismatch or errors: return False for common_dir in dir_cmp.common_dirs: new_dir1 = os.path.join(dir1, common_dir) new_dir2 = os.path.join(dir2, common_dir) if not compare_dirs(new_dir1, new_dir2): return False return True
def _compareResultFolders(compare, tmpdir, pid): ignore_ftypes = ('.log', '.txt') files = [] deletefiles = [] for f in glob(join(compare, '*')): fname = os.path.basename(f) if os.path.splitext(f)[1] in ignore_ftypes: continue if f.endswith('prmtop'): _cutfirstline(f, join(compare, fname + '.mod')) _cutfirstline(join(tmpdir, fname), os.path.join(tmpdir, fname + '.mod')) files.append(os.path.basename(f) + '.mod') deletefiles.append(join(compare, fname + '.mod')) else: files.append(os.path.basename(f)) match, mismatch, error = filecmp.cmpfiles(tmpdir, compare, files, shallow=False) if len(mismatch) != 0 or len(error) != 0 or len(match) != len(files): raise RuntimeError( 'Different results produced by amber.build for test {} between {} and {} in files {}.' .format(pid, compare, tmpdir, mismatch)) for f in deletefiles: os.remove(f)
def test_backup_verbose(self): """Test of method Engine.backup().""" archive_dir = self.engine.backup(source=self.source, backup_dir=self.backup, verbose=True, suffix=u'-тест') self.assertEqual(os.path.abspath(self.backup), os.path.abspath(os.path.join(archive_dir, os.path.pardir))) archive = os.path.join(archive_dir, 'source.tar.bz2') self.assertTrue(os.path.exists(archive)) self.assertEqual(u'Backup started.\n' u'----------------------\n' u'Making archive directory in ' u'{backup}/{date}-тест\n\n' u'Performing backups\n' u'----------------------\n' u'Backup of {source}\n' u'----------------------\n' u'All backups complete!\n'.format(backup=self.backup, date=date.today(), source=self.source), self.out.data) self.assertEqual('', self.err.data) args = ['/bin/tar', '-xjf', archive, '-C', self.restore] with open(os.devnull, 'w') as out: self.assertEqual(0, subprocess.call(args, stdout=out)) rest_dir = os.path.join(self.restore, os.path.basename(self.source)) self.assertTrue(filecmp.cmpfiles(self.source, rest_dir, self.filelist, shallow=False))
def _walk(self, oldpath, newpath): oldfiles = [] newfiles = [] includes = eval(Config.get("actions", "include")) excludes = eval(Config.get("actions", "exclude")) for root, dirs, files in os.walk(oldpath): relpath = path.relpath(root, oldpath) if relpath == ".": relpath = "" for filename in files: filepath = path.join(relpath, filename) if (any(map(lambda glob: fnmatch(filepath, glob), excludes))): continue if (any(map(lambda glob: fnmatch(filepath, glob), includes))): oldfiles.append(filepath) for root, dirs, files in os.walk(newpath): relpath = path.relpath(root, newpath) if relpath == ".": relpath = "" for filename in files: filepath = path.join(relpath, filename) if (any(map(lambda glob: fnmatch(filepath, glob), excludes))): continue if (any(map(lambda glob: fnmatch(filepath, glob), includes))): newfiles.append(filepath) unmodified, modified, removed = filecmp.cmpfiles(oldpath, newpath, oldfiles, shallow=False) unmodified, modified, added = filecmp.cmpfiles(oldpath, newpath, newfiles, shallow=False) unmodified = slashes(unmodified) modified = slashes(modified) added = slashes(added) removed = slashes(removed) return unmodified, modified, added, removed
def test_integration(msgsteiner): parser = OptionParser() parser.add_option('--msgpath',dest='msgsteiner',type='string',help='Path to msgsteiner9 code, be sure to include!') phos_weights = os.path.join(os.path.dirname(__file__), '..', 'example', 'a549', 'Tgfb_phos.txt') #provided config file garnet_conf = os.path.join(os.path.dirname(__file__), '..', 'example', 'a549', 'tgfb_garnet.cfg') #forest requires more inputs forest_conf = os.path.join(os.path.dirname(__file__), '..', 'example', 'a549', 'tgfb_forest.cfg') #provided config file edge_file = os.path.join(os.path.dirname(__file__), '..', 'data', 'iref_mitab_miscore_2013_08_12_interactome.txt') #interactom #WE NEED MSGSTEINER9 INSTALLED!!! msgsteinerpath = msgsteiner if msgsteinerpath == None: print 'Please provide path to msgsteiner using --msgpath option' assert 0 # Create a tmp directory for output forest_out = tempfile.mkdtemp() #Arbitrary value seed = 2 try: script_dir = os.path.dirname(__file__) forest_path = os.path.join(script_dir, '..', 'scripts', 'forest.py') fcmd='python %s --prize=%s --edge=%s --conf=%s --outpath=%s --msgpath=%s --seed=%s'%( forest_path, phos_weights, edge_file, forest_conf, forest_out, msgsteinerpath, seed) subprocess.call(shlex.split(fcmd), shell=False) curr_dir = os.path.dirname(__file__) match, mismatch, errors = filecmp.cmpfiles(forest_out, os.path.join(curr_dir, 'integration_test_standard'), ['result_augmentedForest.sif', 'result_dummyForest.sif', 'result_edgeattributes.tsv', 'result_info.txt', 'result_nodeattributes.tsv', 'result_optimalForest.sif'], shallow=False) if len(match) != 6: print 'Mismatching files: ', mismatch print 'Errors: ', errors assert 0, 'Not all Forest output files match' else: assert 1 except IOError as e: print 'IO error' finally: shutil.rmtree(forest_out)
def compare(self, p=""): d1 = os.path.join(self.dir1, p) d2 = os.path.join(self.dir2, p) dcmp = filecmp.dircmp(d1, d2, ignore=[]) self.left_only.extend(os.path.join(p, n) for n in dcmp.left_only) self.right_only.extend(os.path.join(p, n) for n in dcmp.right_only) self.common_funny.extend(os.path.join(p, n) for n in dcmp.common_funny) self.funny_files.extend(os.path.join(p, n) for n in dcmp.funny_files) (match, mismatch, errors) = filecmp.cmpfiles(d1, d2, dcmp.common_files, shallow=False) self.diff_files.extend(os.path.join(p, n) for n in mismatch) self.funny_files.extend(os.path.join(p, n) for n in errors) for d in dcmp.common_dirs: self.compare(os.path.join(p, d))
def test_cpp_function(self): ''' Test C++ functions generation ''' cpp = CppFile('func.cpp') hpp = CppFile('func.h') def function_body(_, cpp1): cpp1('return 42;') functions = [CppFunction(name='GetParam', ret_type='int'), CppFunction(name='Calculate', ret_type='void'), CppFunction(name='GetAnswer', ret_type='int', implementation_handle=function_body)] for func in functions: func.render_to_string(hpp) for func in functions: func.render_to_string_declaration(hpp) for func in functions: func.render_to_string_implementation(cpp) self.assertTrue(filecmp.cmpfiles('.', 'tests', 'func.cpp')) self.assertTrue(filecmp.cmpfiles('.', 'tests', 'func.h')) cpp.close() hpp.close()
def diff(d1, d2): # Determine the items that exist in both directories d1_contents = set(os.listdir(d1)) d2_contents = set(os.listdir(d2)) common = list(d1_contents & d2_contents) common_files = [f for f in common if os.path.isfile(os.path.join(d1, f))] print "Common files:", common_files # Compare the directories match, mismatch, errors = filecmp.cmpfiles(d1, d2, common_files) print "Match:", match print "Mismatch:", mismatch print "Errors:", errors
def equal_dirs(self, dir1, dir2, ignore=[]): """Checks equality between dir1 and dir2, ignores files in 'ignore' list.""" dirs_cmp = filecmp.dircmp(dir1, dir2, ignore) if (len(dirs_cmp.left_only) > 0 or len(dirs_cmp.right_only) > 0 or len(dirs_cmp.funny_files) > 0): return False (_, mismatch, errors) = filecmp.cmpfiles(dir1, dir2, dirs_cmp.common_files, shallow=False) if len(mismatch) > 0 or len(errors) > 0: return False for common_dir in dirs_cmp.common_dirs: new_dir1 = os.path.join(dir1, common_dir) new_dir2 = os.path.join(dir2, common_dir) if not self.equal_dirs(new_dir1, new_dir2, ignore): return False return True
def test_patch(self): retval = self.cherry.patch( base_sha='7bee46915e2a29ac789ac18b43dcdb438377bfbd', target_sha='b23e9fd3a3518faf703d9cc90c3cafb079e4488d', target_branch='test_branch') self.assertTrue(retval) (match, mismatch, errors) = filecmp.cmpfiles( self.cherry.files_base, self.patch_dir, ['README.md']) self.assertEqual(len(match), 1) self.assertEqual(len(mismatch), 0) self.assertEqual(len(errors), 0)