def test_one_replacement(self): """Test a target with a single replacement editor Before the test execution we copy the target file from a sample each time. Target will be removed on successful test execution. On failure it will be kept for debugging and overwritten on next test execution (no checks) """ sample = os.path.join(self.test_files_dir, "one_replacement.sample") targetfile = os.path.join(self.test_files_dir, "one_replacement.conf") expected = os.path.join(self.test_files_dir, "one_replacement.expected") thisincr = 10 backupfile = "{0}.backup.{1}".format(targetfile, thisincr) shutil.copy(sample, targetfile) tgt = target.Target() tgt.targetfile = targetfile tgt.edlist.append(editor.Editor("replace", 10, ("email", "*****@*****.**"))) thisincr = mock.Mock() thisincr.is_current.return_value = True thisincr.__int__ = mock.Mock() thisincr.__int__.return_value = 10 tgt.run_editors(thisincr) result = filecmp.cmp(targetfile, expected) self.assertTrue(result) if not result: print >> sys.stderr, "Failed result in {}".format(targetfile) else: os.unlink(targetfile) self.assertTrue(filecmp.cmp(backupfile, sample)) os.unlink(backupfile)
def test_SetUpInDirAndFsConfig_WithRootFsConfig(self): root_dir = common.MakeTempDir() with open(os.path.join(root_dir, 'init'), 'w') as init_fp: init_fp.write('init') origin_in = common.MakeTempDir() with open(os.path.join(origin_in, 'file'), 'w') as in_fp: in_fp.write('system-file') os.symlink('../etc', os.path.join(origin_in, 'symlink')) fs_config_system = self._gen_fs_config('system') fs_config_root = self._gen_fs_config('root') prop_dict = { 'fs_config': fs_config_system, 'mount_point': 'system', 'root_dir': root_dir, 'root_fs_config': fs_config_root, } in_dir, fs_config = SetUpInDirAndFsConfig(origin_in, prop_dict) self.assertTrue(filecmp.cmp( os.path.join(in_dir, 'init'), os.path.join(root_dir, 'init'))) self.assertTrue(filecmp.cmp( os.path.join(in_dir, 'system', 'file'), os.path.join(origin_in, 'file'))) self.assertTrue(os.path.islink(os.path.join(in_dir, 'system', 'symlink'))) with open(fs_config) as fs_config_fp: fs_config_data = fs_config_fp.readlines() self.assertIn('fs-config-system\n', fs_config_data) self.assertIn('fs-config-root\n', fs_config_data) self.assertEqual('/', prop_dict['mount_point'])
def test_compressor_nominify(self): comm.clear_compressor() compre = "" comm.compressor(compre, self) self.assertTrue(filecmp.cmp(comm.compDir + "script.js", comm.oriDir + "script.js")) self.assertTrue(filecmp.cmp(comm.compDir + "style.css", comm.oriDir + "style.css")) comm.clear_compressor()
def test_two_replacements(self): """Test a target with a two replacement editors Basically copied from test_one_replacement, did not bother to factor out common code. """ sample = os.path.join(self.test_files_dir, "two_replacements.sample") targetfile = os.path.join(self.test_files_dir, "two_replacements.conf") expected = os.path.join(self.test_files_dir, "two_replacements.expected") thisincr = mock.Mock() thisincr.is_current.return_value = True thisincr.__int__ = mock.Mock() thisincr.__int__.return_value = 22 backupfile = "{0}.backup.{1}".format(targetfile, int(thisincr)) shutil.copy(sample, targetfile) tgt = target.Target() tgt.targetfile = targetfile tgt.edlist.append(editor.Editor("replace", 22, ("email", "*****@*****.**"))) tgt.edlist.append(editor.Editor("replace", 22, ("myip", "1.2.3.4"))) tgt.run_editors(thisincr) result = filecmp.cmp(targetfile, expected) self.assertTrue(result) if not result: print >> sys.stderr, "Failed result in {}".format(targetfile) else: os.unlink(targetfile) self.assertTrue(filecmp.cmp(backupfile, sample)) os.unlink(backupfile)
def test_minimap_reads_to_all_ref_seqs(self): '''test test_minimap_reads_to_all_ref_seqs''' clusters_tsv = os.path.join(data_dir, 'clusters_minimap_reads_to_all_refs.clstrs.tsv') ref_fasta = os.path.join(data_dir, 'clusters_minimap_reads_to_all_refs.ref.fa') reads_1 = os.path.join(data_dir, 'clusters_minimap_reads_to_all_refs.reads_1.fq') reads_2 = os.path.join(data_dir, 'clusters_minimap_reads_to_all_refs.reads_2.fq') tmp_outprefix = 'tmp.clusters_test_minimap_reads_to_all_ref_seqs' clusters.Clusters._minimap_reads_to_all_ref_seqs(clusters_tsv, ref_fasta, reads_1, reads_2, tmp_outprefix) expected_cluster2rep = os.path.join(data_dir, 'clusters_minimap_reads_to_all_refs.out.clstr2rep') expected_cluster_counts = os.path.join(data_dir, 'clusters_minimap_reads_to_all_refs.out.clstr_count') expected_proper_pairs = os.path.join(data_dir, 'clusters_minimap_reads_to_all_refs.out.pairs') expected_insert_hist = os.path.join(data_dir, 'clusters_minimap_reads_to_all_refs.out.hist') # not sure that the reads order is preserved, so just check read store file exists self.assertTrue(os.path.exists(os.path.join(tmp_outprefix + '.reads'))) self.assertTrue(filecmp.cmp(expected_cluster2rep, tmp_outprefix + '.cluster2representative', shallow=False)) self.assertTrue(filecmp.cmp(expected_cluster_counts, tmp_outprefix + '.clusterCounts', shallow=False)) self.assertTrue(filecmp.cmp(expected_proper_pairs, tmp_outprefix + '.properPairs', shallow=False)) self.assertTrue(filecmp.cmp(expected_insert_hist, tmp_outprefix + '.insertHistogram', shallow=False)) os.unlink(tmp_outprefix + '.cluster2representative') os.unlink(tmp_outprefix + '.clusterCounts') os.unlink(tmp_outprefix + '.properPairs') os.unlink(tmp_outprefix + '.insertHistogram') os.unlink(tmp_outprefix + '.reads')
def test_uploadFileEntity(): # Create a FileEntity # Dictionaries default to FileEntity as a type fname = utils.make_bogus_data_file() schedule_for_cleanup(fname) entity = {'name' : 'fooUploadFileEntity', \ 'description' : 'A test file entity', \ 'parentId' : project['id']} entity = syn.uploadFile(entity, fname) # Download and verify entity = syn.downloadEntity(entity) assert entity['files'][0] == os.path.basename(fname) assert filecmp.cmp(fname, entity['path']) # Check if we upload the wrong type of file handle fh = syn.restGET('/entity/%s/filehandles' % entity.id)['list'][0] assert fh['concreteType'] == 'org.sagebionetworks.repo.model.file.S3FileHandle' # Create a different temporary file fname = utils.make_bogus_data_file() schedule_for_cleanup(fname) # Update existing FileEntity entity = syn.uploadFile(entity, fname) # Download and verify that it is the same file entity = syn.downloadEntity(entity) assert entity['files'][0] == os.path.basename(fname) assert filecmp.cmp(fname, entity['path'])
def tst_link(mnt_dir): name1 = pjoin(mnt_dir, name_generator()) name2 = pjoin(mnt_dir, name_generator()) shutil.copyfile(TEST_FILE, name1) assert filecmp.cmp(name1, TEST_FILE, False) fstat1 = os.lstat(name1) assert fstat1.st_nlink == 1 os.link(name1, name2) fstat1 = os.lstat(name1) fstat2 = os.lstat(name2) for attr in ('st_mode', 'st_dev', 'st_uid', 'st_gid', 'st_size', 'st_atime', 'st_mtime', 'st_ctime'): assert getattr(fstat1, attr) == getattr(fstat2, attr) assert os.path.basename(name2) in os.listdir(mnt_dir) assert filecmp.cmp(name1, name2, False) os.unlink(name2) assert os.path.basename(name2) not in os.listdir(mnt_dir) with pytest.raises(FileNotFoundError): os.lstat(name2) os.unlink(name1)
def test_MaterialPackageWrangler_make_package_09(): r'''Makes handmade package. Copies canned material definition py. Makes output py. Corrupts output py. Makes sure score manager starts when output py is corrupt. Removes package. ''' with systemtools.FilesystemState(remove=[path]): input_ = 'mm new testnotes y q' ide._run(input_=input_) assert os.path.exists(path) assert os.path.exists(definition_py_path) assert not os.path.exists(output_py_path) shutil.copyfile( boilerplate_definition_py_path, definition_py_path, ) assert filecmp.cmp( definition_py_path, boilerplate_definition_py_path, ) input_ = 'mm testnotes dp y q' ide._run(input_=input_) assert os.path.exists(output_py_path) assert not filecmp.cmp(output_py_path, exception_file_path) shutil.copyfile(exception_file_path, output_py_path) assert filecmp.cmp(output_py_path, exception_file_path) input_ = 'mm rm testnotes remove q' ide._run(input_=input_) assert not os.path.exists(path)
def make_makefile(directory): """ Make makefile into a selected directory. Parameters ---------- directory : str relative path to tested directory """ print("Testing " + directory) make_ok = False old_pwd = os.getcwd() os.chdir(os.path.dirname(os.path.abspath(__file__)) + '/' + directory) run_fobis(fake_args=['build', '-f', 'fobos', '-m', 'makefile_check']) make_ok = filecmp.cmp('makefile_check', 'makefile_ok') if not make_ok: if os.path.exists('makefile_ok2'): make_ok = filecmp.cmp('makefile_check', 'makefile_ok2') if not make_ok: print('makefile generated') with open('makefile_check', 'r') as mk_check: print(mk_check.read()) run_fobis(fake_args=['clean', '-f', 'fobos']) os.chdir(old_pwd) return make_ok
def testOutsideChrootDifferentTokenFile(self): mocked_outsidechroot = self._MockOutsideChroot('foo') self.mox.StubOutWithMock(cgt, '_ChrootPathToExternalPath') self.mox.StubOutWithMock(os.path, 'exists') self.mox.StubOutWithMock(shutil, 'copy2') self.mox.StubOutWithMock(filecmp, 'cmp') self.mox.StubOutWithMock(build_lib, 'RunCommand') cmd = ['check_gdata_token', 'foo'] run_result = cros_test_lib.EasyAttr(returncode=0) # Create replay script. build_lib.RunCommand(cmd, enter_chroot=True, print_cmd=False, error_code_ok=True).AndReturn(run_result) cgt._ChrootPathToExternalPath(cgt.TOKEN_FILE).AndReturn('chr-tok') os.path.exists('chr-tok').AndReturn(True) os.path.exists(cgt.TOKEN_FILE).AndReturn(True) filecmp.cmp(cgt.TOKEN_FILE, 'chr-tok').AndReturn(False) shutil.copy2('chr-tok', cgt.TOKEN_FILE) self.mox.ReplayAll() # Run test verification. with self.OutputCapturer(): cgt.OutsideChroot.Run(mocked_outsidechroot) self.mox.VerifyAll()
def test_variations(self): """Adds image and checks filesystem as well as width and height.""" instance = ResizeModel.objects.create( image=self.fixtures['600x400.jpg'] ) source_file = os.path.join(FIXTURE_DIR, '600x400.jpg') self.assertTrue(os.path.exists(os.path.join(IMG_DIR, 'image.jpg'))) self.assertEqual(instance.image.width, 600) self.assertEqual(instance.image.height, 400) path = os.path.join(IMG_DIR, 'image.jpg') assert filecmp.cmp(source_file, path) path = os.path.join(IMG_DIR, 'image.medium.jpg') assert os.path.exists(path) self.assertEqual(instance.image.medium.width, 400) self.assertLessEqual(instance.image.medium.height, 400) self.assertFalse(filecmp.cmp( source_file, os.path.join(IMG_DIR, 'image.medium.jpg'))) self.assertTrue(os.path.exists( os.path.join(IMG_DIR, 'image.thumbnail.jpg')) ) self.assertEqual(instance.image.thumbnail.width, 100) self.assertLessEqual(instance.image.thumbnail.height, 75) self.assertFalse(filecmp.cmp( source_file, os.path.join(IMG_DIR, 'image.thumbnail.jpg')) )
def compare_files(tshark_bin, tmpdir, tshark_cmp, num_procs, max_files, cap_files): pool = multiprocessing.Pool(num_procs) results_bin = [pool.apply_async(dissect_file_process, [tshark_bin, tmpdir, file]) for file in cap_files] results_cmp = [pool.apply_async(dissect_file_process, [tshark_cmp, tmpdir, file]) for file in cap_files] try: for (cur_item_idx,(result_async_bin, result_async_cmp)) in enumerate(zip(results_bin, results_cmp)): file_result_bin = result_async_bin.get() file_result_cmp = result_async_cmp.get() if file_result_cmp[1] is False or file_result_bin[1] is False: action = "FAILED (exitcode)" if not filecmp.cmp(file_result_bin[2], file_result_cmp[2]): action = "FAILED (stdout)" if not filecmp.cmp(file_result_bin[3], file_result_cmp[3]): action = "FAILED (stderr)" else: action = "PASSED" os.remove(file_result_bin[2]) os.remove(file_result_cmp[2]) os.remove(file_result_bin[3]) os.remove(file_result_cmp[3]) print "%s [%u/%u] %s %u bytes" % (action, cur_item_idx+1, max_files, file_result_bin[0], os.path.getsize(file_result_bin[0])) print "%s [%u/%u] %s %u bytes" % (action, cur_item_idx+1, max_files, file_result_cmp[0], os.path.getsize(file_result_cmp[0])) except KeyboardInterrupt: print "%s was interrupted by user" % (sys.argv[0]) pool.terminate() exit(1)
def test_Branch(self): """ Test Branch and Branch.run. Note that fuzzy junctions are not merged. """ test_name = "test_branch" good_gff_fn = op.join(_OUT_DIR_, test_name + ".good.gff.unfuzzy") bad_gff_fn = op.join(_OUT_DIR_, test_name + ".bad.gff.unfuzzy") group_fn = op.join(_OUT_DIR_, test_name + ".group.txt.unfuzzy") rmpath(good_gff_fn) rmpath(bad_gff_fn) rmpath(group_fn) b = Branch(isoform_filename=READS_DS, sam_filename=SORTED_GMAP_SAM, cov_threshold=2, min_aln_coverage=0.99, min_aln_identity=0.95) b.run(allow_extra_5exon=True, skip_5_exon_alt=False, ignored_ids_fn=None, good_gff_fn=good_gff_fn, bad_gff_fn=bad_gff_fn, group_fn=group_fn) self.assertTrue(op.exists(good_gff_fn)) self.assertTrue(op.exists(bad_gff_fn)) self.assertTrue(op.exists(group_fn)) std_good_gff_fn = op.join(SIV_STD_DIR, "test_branch", test_name + ".good.gff.unfuzzy") std_bad_gff_fn = op.join(SIV_STD_DIR, "test_branch", test_name + ".bad.gff.unfuzzy") std_group_fn = op.join(SIV_STD_DIR, "test_branch", test_name + ".group.txt.unfuzzy") print "Comparing %s and %s" % (good_gff_fn, std_good_gff_fn) self.assertTrue(filecmp.cmp(good_gff_fn, std_good_gff_fn)) self.assertTrue(filecmp.cmp(bad_gff_fn, std_bad_gff_fn)) self.assertTrue(filecmp.cmp(group_fn, std_group_fn))
def main(): cdcVisLinks = getVisLinks() global modifiedPage for link in cdcVisLinks: visPageUrl = 'http://www.cdc.gov/vaccines/hcp/vis/vis-statements/' + link visPageReq = urllib.request.Request(visPageUrl) visPageResp = urllib.request.urlopen(visPageReq) visPageRespData = visPageResp.read() downloadPage = re.findall(r'href="/vaccines/hcp/vis/vis-statements/(.*?.pdf)"',str(visPageRespData)) downloadLink = 'http://www.cdc.gov/vaccines/hcp/vis/vis-statements/' + downloadPage[0] urllib.request.urlretrieve(downloadLink,downloadPage[0]) # Download VIS PDF pdfFileName = downloadPage[0] if os.path.isfile(currentFilePath+'/lib/'+downloadPage[0]) == True: #print downloadPage[0],'File exists in ./lib/ folder' if filecmp.cmp(downloadPage[0],currentFilePath+'/lib/'+downloadPage[0]) == True: subprocess.call(["rm",downloadPage[0]]) elif filecmp.cmp(downloadPage[0],currentFilePath+'/lib/'+downloadPage[0]) == False: subprocess.call(["mv",downloadPage[0],currentFilePath+'/lib/']) modifiedPage.append(pdfFileName) writeLogEntry(logFileName, [pdfFileName,datetime.datetime.now().strftime("%m/%d/%Y %H:%M:%S")]) else: #print downloadPage[0],'File does not exist in ./lib/ folder' subprocess.call(["mv",downloadPage[0],'./lib/']) modifiedPage.append(pdfFileName) writeLogEntry(logFileName, [pdfFileName,datetime.datetime.now().strftime("%m/%d/%Y %H:%M:%S")])
def _copy_file(self, filename, destination): filepath = os.path.join(THIS_FOLDER, filename) def do_it(): shutil.copyfile(filepath, destination) if self.creating: do_it() else: # first check if the file is already present if not os.path.exists(destination): do_it() else: # file exists. Let's check if what the instance has is the # vainilla old version or not old_version_filepath = os.path.join( self.old_version_path, 'iepy', 'instantiation', filename) if os.path.exists(old_version_filepath) and filecmp.cmp(old_version_filepath, destination): # vainilla old version. Let's simply upgrade it do_it() else: # customized file. # Check if it's exactly what the new version provides if filecmp.cmp(filepath, destination): # you have local changes that are 100% new version. Sounds like # you made your instance with a nightly built.. dunno, but we'll # do nothing, since is not needed. return # We'll back it up, and later upgrade self.preserve_old_file_version_as_copy(destination) do_it()
def test_saveResponse(self): """ Fetches and stores response information as SEED RESP file. """ client = Client() start = UTCDateTime("2005-001T00:00:00") end = UTCDateTime("2008-001T00:00:00") # RESP, single channel origfile = os.path.join(self.path, "data", "RESP.ANMO.IU.00.BHZ") tempfile = NamedTemporaryFile().name client.saveResponse(tempfile, "IU", "ANMO", "00", "BHZ", start, end) self.assertTrue(filecmp.cmp(origfile, tempfile)) os.remove(tempfile) # RESP, multiple channels origfile = os.path.join(self.path, "data", "RESP.ANMO.IU._.BH_") tempfile = NamedTemporaryFile().name client.saveResponse(tempfile, "IU", "ANMO", "*", "BH?", start, end) self.assertTrue(filecmp.cmp(origfile, tempfile)) os.remove(tempfile) # StationXML, single channel tempfile = NamedTemporaryFile().name client.saveResponse(tempfile, "IU", "ANMO", "00", "BHZ", start, end, format="StationXML") data = open(tempfile).read() self.assertTrue('<Station net_code="IU" sta_code="ANMO">' in data) os.remove(tempfile) # SACPZ, single channel tempfile = NamedTemporaryFile().name client.saveResponse(tempfile, "IU", "ANMO", "00", "BHZ", start, end, format="SACPZ") data = open(tempfile).read() self.assertTrue("NETWORK (KNETWK): IU" in data) self.assertTrue("STATION (KSTNM): ANMO" in data) os.remove(tempfile)
def test_folder_decompression(self): zip_file = "tests/file_manager_tests/data/dir.zip" out_dir = "tests/file_manager_tests/data/dirout" if os.path.exists(zip_file): os.remove(zip_file) if os.path.exists(out_dir): shutil.rmtree(out_dir) self.assertFalse(os.path.exists(out_dir) and os.path.exists(zip_file), "File(s) exists prior to test") FileUtils.compress_file("tests/file_manager_tests/data/dir", zip_file) FileUtils.decompress_file(zip_file, out_dir) self.assertTrue(os.path.exists(out_dir) and os.path.isdir(out_dir), "Dir does not exist after test") self.assertTrue(os.path.exists(out_dir + "/tests/file_manager_tests/data/dir/b.txt"), "File b does not exist after test") self.assertTrue(os.path.exists(out_dir + "/tests/file_manager_tests/data/dir/c.txt"), "File c does not exist after test") self.assertTrue(filecmp.cmp("tests/file_manager_tests/data/dir/b.txt", out_dir + "/tests/file_manager_tests/data/dir/b.txt"), "File b is not the same") self.assertTrue(filecmp.cmp("tests/file_manager_tests/data/dir/c.txt", out_dir + "/tests/file_manager_tests/data/dir/c.txt"), "File c is not the same") os.remove(zip_file) shutil.rmtree(out_dir) return
def test_gbif_to_file(self, mock_urlretrieve=None, mock_urlopen=None): mock_urlretrieve.side_effect = self._urlretrieve mock_urlopen.side_effect = self._urlopen # mock urllib.urlretrieve .... # return zip file with data.csv and citation.csv # mock urllib.urlretriev ... # return gbif_metadata.json file_dest = { 'url': 'file://{}'.format(self.tmpdir) } move(self.gbif_source, file_dest) # Check files are created self.assertTrue(os.path.exists( os.path.join(self.tmpdir, 'gbif_dataset.json'))) self.assertTrue(os.path.exists( os.path.join(self.tmpdir, 'gbif_occurrence.zip'))) self.assertTrue(os.path.exists( os.path.join(self.tmpdir, 'gbif_metadata.json'))) # Check file contents zf = zipfile.ZipFile(os.path.join(self.tmpdir, 'gbif_occurrence.zip')) zf.extractall(self.tmpdir) self.assertTrue(filecmp.cmp(os.path.join(self.tmpdir, 'gbif_metadata.json'), pkg_resources.resource_filename(__name__, 'data/gbif_metadata.json'))) self.assertTrue(filecmp.cmp(os.path.join(self.tmpdir, 'data', 'gbif_occurrence.csv'), pkg_resources.resource_filename(__name__, 'data/gbif_occurrence.csv'))) self.assertTrue(filecmp.cmp(os.path.join(self.tmpdir, 'data', 'gbif_citation.txt'), pkg_resources.resource_filename(__name__, 'data/gbif_citation.txt')))
def test_file_overwrite(self): """ Make sure FileWrapper overwrites an existing file """ # read the content to be written from the expected result expected_file = os.path.join("data", "filewrap.txt") with open(expected_file, "r") as fh_src: org_data = fh_src.read() # make the target file dummy_file = os.path.join("data", "fileappend.txt") temp_file = os.path.join("data", "deleteme.txt") shutil.copyfile(dummy_file, temp_file) # make sure the current content is different from the expected result self.assertFalse(filecmp.cmp(expected_file, temp_file)) # FileWrapper overwrites the target file w = target.FileWrapper(temp_file) try: with w.open() as fh: fh.write(org_data) self.assertTrue(filecmp.cmp(expected_file, temp_file)) finally: os.remove(temp_file)
def test_processPrimers(self): """Test function _processPrimers().""" inPFN = op.join(self.dataDir, "test_primers_in.fasta") obj = Classifier() # Test on an artificial example. outPFN = op.join(self.outDir, "test_primers_out.fasta") stdoutPFN = op.join(self.stdDir, "test_primers_out.fasta") obj._processPrimers(primer_fn=inPFN, window_size=50, primer_out_fn=outPFN, revcmp_primers=False) self.assertTrue(filecmp.cmp(outPFN, stdoutPFN)) # Test on real PacBio primers.fasta pbPFN = op.join(self.dataDir, "primers.fasta") # outPFN2 = primers.fasta for primer detection. outPFN2 = op.join(self.outDir, "test_primers_out_2.fasta") stdoutPFN2 = op.join(self.stdDir, "test_primers_out_2.fasta") obj._processPrimers(primer_fn=pbPFN, window_size=50, primer_out_fn=outPFN2, revcmp_primers=False) self.assertTrue(filecmp.cmp(outPFN2, stdoutPFN2)) # outPFN3 = primers.fasta for chimera detection. outPFN2 = op.join(self.outDir, "test_primers_out_3.fasta") stdoutPFN2 = op.join(self.stdDir, "test_primers_out_3.fasta") obj._processPrimers(primer_fn=pbPFN, window_size=50, primer_out_fn=outPFN2, revcmp_primers=True) self.assertTrue(filecmp.cmp(outPFN2, stdoutPFN2))
def test_getBestFrontBackRecord(self): """Test function _parseBestFrontBackRecord().""" obj = Classifier() domFN = op.join(self.dataDir, "test_parseHmmDom.dom") front, back = obj._getBestFrontBackRecord(domFN) # In the following, verify the front and back are equivalent # to stdout/test_parseHmmDom_dFront/Back.txt def prettystr(d): """Return Pretty print string for front & back.""" return "\n".join( [key + ":\n" + "\n".join( [k + ":" + str(v) for k, v in val.iteritems()]) for key, val in d.iteritems()]) frontFN = op.join(self.outDir, "test_parseHmmDom_dFront.txt") backFN = op.join(self.outDir, "test_parseHmmDom_dBack.txt") f = open(frontFN, 'w') f.write(prettystr(front)) f.close() f = open(backFN, 'w') f.write(prettystr(back)) f.close() stdoutFrontFN = op.join(self.stdDir, "test_parseHmmDom_dFront.txt") stdoutBackFN = op.join(self.stdDir, "test_parseHmmDom_dBack.txt") self.assertTrue(filecmp.cmp(frontFN, stdoutFrontFN)) self.assertTrue(filecmp.cmp(backFN, stdoutBackFN))
def test01(self): """ Test basic usage """ datadir = path.join(self.datadir) this_test = sys._getframe().f_code.co_name sseqids = path.join(datadir, 'sseqid_basic') outdir = self.mkoutdir() fasta_out = path.join(outdir, 'out1.fasta') info_out = path.join(outdir, 'out1.csv') fasta_ref = path.join(datadir, 'test01', 'ref.fasta') info_ref = path.join(datadir, 'test01', 'ref.csv') args = ['--outfasta', fasta_out, '--seqinfo', info_out, '--email', '*****@*****.**', sseqids] log.info(self.log_info.format(' '.join(map(str, args)))) self.main(args) self.assertTrue(filecmp.cmp(fasta_ref, fasta_out)) self.assertTrue(filecmp.cmp(info_ref, info_out))
def test_zip(self): file_uri = 'file://'+resources_path+'heb.nt.zip' ae = ArchiveExtractor(file_uri) self.assertTrue(filecmp.cmp(ae.get_extracted_file_path_list()[0], original_head), "file 1 differs") self.assertTrue(filecmp.cmp(ae.get_extracted_file_path_list()[1], original_tail), "file 2 differs")
def test_split_by_base_count(self): '''Check that fasta/q files get split by base count correctly''' infile = os.path.join(data_dir, 'sequences_test_split_test.fa') outprefix = 'tmp.sequences_test_split_test.fa.test' length2files = {2: ['1','2','3','4'], 3: ['1','2','3'], 4: ['1', '2', '3'], 6: ['1', '2']} for l in length2files: tasks.split_by_base_count(infile, outprefix, l) for x in range(len(length2files[l])): file_index = str(length2files[l][x]) fname = outprefix + '.' + file_index self.assertTrue(filecmp.cmp(fname, infile + '.' + str(l) + '.' + file_index)) os.unlink(fname) # check that limiting the number of files works tasks.split_by_base_count(infile, outprefix, 6, 2) for i in range(1,4): test_file = outprefix + '.' + str(i) self.assertTrue(filecmp.cmp(test_file, os.path.join(data_dir, 'sequences_test_split_test.fa.6.limit2.') + str(i))) os.unlink(test_file) # check big sequence not broken tasks.split_by_base_count(os.path.join(data_dir, 'sequences_test_split_test.long.fa'), outprefix, 2) self.assertTrue(filecmp.cmp(outprefix + '.1', os.path.join(data_dir, 'sequences_test_split_test.long.fa.2.1'))) self.assertTrue(filecmp.cmp(outprefix + '.2', os.path.join(data_dir, 'sequences_test_split_test.long.fa.2.2'))) os.unlink(outprefix + '.1') os.unlink(outprefix + '.2')
def test_to_fasta(self): '''Test to_fasta''' tmpfile = 'tmp.to_fasta' infiles = [ 'sequences_test_good_file.fq', 'sequences_test_gffv3.gff', 'sequences_test_gffv3.no_FASTA_line.gff', 'sequences_test.embl', 'sequences_test.gbk', 'sequences_test_phylip.interleaved', 'sequences_test_phylip.interleaved2', 'sequences_test_phylip.sequential' ] infiles = [os.path.join(data_dir, x) for x in infiles] expected_outfiles = [x + '.to_fasta' for x in infiles] for i in range(len(infiles)): tasks.to_fasta(infiles[i], tmpfile) self.assertTrue(filecmp.cmp(expected_outfiles[i], tmpfile)) tasks.to_fasta(os.path.join(data_dir, 'sequences_test.fa'), tmpfile, line_length=3) self.assertTrue(filecmp.cmp(os.path.join(data_dir, 'sequences_test.line_length3.fa'), tmpfile)) tasks.to_fasta(os.path.join(data_dir, 'sequences_test_strip_after_whitespace.fa'), tmpfile, strip_after_first_whitespace=True) self.assertTrue(filecmp.cmp(os.path.join(data_dir, 'sequences_test_strip_after_whitespace.fa.to_fasta'), tmpfile)) os.unlink(tmpfile)
def test_bam_extract_01(self): TEST_DIR, T_TEST_DIR = self.__get_temp_dirs() input_file = TEST_DIR + "test_terg_02.bam" output_file = T_TEST_DIR + "test_terg_02.filtered.bam" output_file_s = T_TEST_DIR + "test_terg_02.filtered.sam" test_file = TEST_DIR + "test_terg_02.filtered.sam" # c = BAMExtract(input_file) # c.extract("chr21:39000000-40000000", "chr5:1-2", output_file) command = ["bin/dr-disco", "bam-extract", "chr21:39000000-40000000", "chr5:1-2", output_file, input_file] self.assertEqual(subprocess.call(command), 0) # Bam2Sam fhq = open(output_file_s, "w") fhq.write(pysam.view(output_file)) fhq.close() if not filecmp.cmp(output_file_s, test_file): print 'diff \'' + output_file_s + '\' \'' + test_file + '\'' self.assertTrue(filecmp.cmp(output_file_s, test_file))
def test_real_grammar_and_keyword_file(self): self._copy_file_without_generated_keywords(KEYWORD_FILE, TEST_PY_FILE) self.addCleanup(support.unlink, TEST_PY_FILE) self.assertFalse(filecmp.cmp(KEYWORD_FILE, TEST_PY_FILE)) self.assertEqual((0, b''), self._generate_keywords(GRAMMAR_FILE, TEST_PY_FILE)) self.assertTrue(filecmp.cmp(KEYWORD_FILE, TEST_PY_FILE))
def test_writeSeqs(self): fdb = FastaDB.FastaDB() fdb.read_seqs('three.fasta') with tempfile.NamedTemporaryFile() as temp: fdb.write_seqs(temp.name) # self.assertTrue(hashlib.sha256(open(temp.name, 'rb').read()).digest() == hashlib.sha256(open("three.fasta", 'rb').read()).digest()) filecmp.cmp(temp.name, 'three.fasta')
def test_backup(self): self._init_test_data_file() title1 = 'title 1' title2 = 'title 2' title3 = 'title 3' title4 = 'title 4' tag1 = 'tag 1' tag2 = 'tag 2' tag3 = 'tag 3' tag4 = 'tag 4' tags1 = [tag1, tag2] tags2 = [tag3, tag4] item1 = Item(title1, {Elem.TAGS : tags1}) item2 = Item(title2, {Elem.TAGS : tags2}) item3 = Item(title3, {Elem.TAGS : tags1}) item4 = Item(title4, {Elem.TAGS : tags2}) orgm = Organizem(TEST_DATA_FILE, IS_UNIT_TESTING) orgm.add_item(item1) orgm.add_item(item2) orgm.add_item(item3) orgm.add_item(item4) bak_data_file = 'orgm_test.dat_bak' orgm.backup(bak_data_file) import filecmp filecmp.cmp(TEST_DATA_FILE, bak_data_file)
def compare_dl_files(fileExtension): # Need to handle gbk in a unique manner as the date will always modify the file slightly so a diff will return false if fileExtension == 'gbk': newestFile = min(glob.iglob('/Users/jmatsumura/Downloads/VAC1_test2.annotation.*.'+fileExtension), key=os.path.getctime) my_cmd = ['diff', '/Users/jmatsumura/mana_dumps/VAC1_test2.annotation.20160329.gbk'] + [newestFile] with open('/Users/jmatsumura/mana_dumps/gbk_diff.txt', "w") as outfile: subprocess.call(my_cmd, stdout=outfile) result = "OK" if os.stat("/Users/jmatsumura/mana_dumps/gbk_diff.txt").st_size < 300 else "FAILED" # Similar to the previous, handle by file size differences. elif fileExtension == 'GO_annotation.txt': newestFile = min(glob.iglob('/Users/jmatsumura/Downloads/VAC1_test2_'+fileExtension), key=os.path.getctime) my_cmd = ['diff', '/Users/jmatsumura/mana_dumps/VAC1_test2_GO_annotation.txt'] + [newestFile] with open('/Users/jmatsumura/mana_dumps/GO_diff.txt', "w") as outfile: subprocess.call(my_cmd, stdout=outfile) f_size = os.stat("/Users/jmatsumura/mana_dumps/GO_diff.txt").st_size result = "OK" if ((f_size > 2200000) and (f_size < 2900000)) else "FAILED" elif fileExtension == 'tbl' or fileExtension == 'gff3': newestFile = min(glob.iglob('/Users/jmatsumura/Downloads/VAC1_test2.annotation.*.'+fileExtension), key=os.path.getctime) result = "OK" if filecmp.cmp('/Users/jmatsumura/mana_dumps/VAC1_test2.annotation.20160329.'+fileExtension, newestFile) else "FAILED" elif fileExtension == 'sigp': newestFile = min(glob.iglob('/Users/jmatsumura/Downloads/sigp4.1_VAC.transcript.9803630972.1_pred.txt'), key=os.path.getctime) result = "OK" if filecmp.cmp('/Users/jmatsumura/mana_dumps/sigpOut.txt', newestFile) else "FAILED" else: newestFile = min(glob.iglob('/Users/jmatsumura/Downloads/VAC1_test2_'+fileExtension), key=os.path.getctime) result = "OK" if filecmp.cmp('/Users/jmatsumura/mana_dumps/VAC1_test2_'+fileExtension, newestFile) else "FAILED" return result
def copyUsedDLLs(source_dir, dist_dir, standalone_entry_points): # This is terribly complex, because we check the list of used DLLs # trying to avoid duplicates, and detecting errors with them not # being binary identical, so we can report them. And then of course # we also need to handle OS specifics. # pylint: disable=too-many-branches,too-many-locals used_dlls = detectUsedDLLs(source_dir, standalone_entry_points) # Fist make checks and remove some. for dll_filename1, sources1 in tuple(iterItems(used_dlls)): for dll_filename2, sources2 in tuple(iterItems(used_dlls)): if dll_filename1 == dll_filename2: continue # Colliding basenames are an issue to us. if os.path.basename(dll_filename1) != \ os.path.basename(dll_filename2): continue # May already have been removed earlier if dll_filename1 not in used_dlls: continue if dll_filename2 not in used_dlls: continue dll_name = os.path.basename(dll_filename1) if Options.isShowInclusion(): info("""Colliding DLL names for %s, checking identity of \ '%s' <-> '%s'.""" % ( dll_name, dll_filename1, dll_filename2, )) # Check that if a DLL has the same name, if it's identical, # happens at least for OSC and Fedora 20. import filecmp if filecmp.cmp(dll_filename1, dll_filename2): del used_dlls[dll_filename2] continue # So we have conflicting DLLs, in which case we do not proceed. sys.exit("""Error, conflicting DLLs for '%s'. %s used by: %s different from %s used by %s""" % (dll_name, dll_filename1, "\n ".join(sources1), dll_filename2, "\n ".join(sources2))) dll_map = [] for dll_filename, sources in iterItems(used_dlls): dll_name = os.path.basename(dll_filename) target_path = os.path.join(dist_dir, dll_name) shutil.copy(dll_filename, target_path) dll_map.append((dll_filename, dll_name)) if Options.isShowInclusion(): info("Included used shared library '%s' (used by %s)." % (dll_filename, ", ".join(sources))) if Utils.getOS() == "Darwin": # For MacOS, the binary and the DLLs needs to be changed to reflect # the relative DLL location in the ".dist" folder. for standalone_entry_point in standalone_entry_points: fixupBinaryDLLPaths( binary_filename=standalone_entry_point[1], is_exe=standalone_entry_point is standalone_entry_points[0], dll_map=dll_map) for _original_path, dll_filename in dll_map: fixupBinaryDLLPaths(binary_filename=os.path.join( dist_dir, dll_filename), is_exe=False, dll_map=dll_map) if Utils.getOS() == "Linux": # For Linux, the "rpath" of libraries may be an issue and must be # removed. for standalone_entry_point in standalone_entry_points[1:]: removeSharedLibraryRPATH(standalone_entry_point[1]) for _original_path, dll_filename in dll_map: removeSharedLibraryRPATH(os.path.join(dist_dir, dll_filename))
def _test_checkout(self): self.assertTrue(os.path.isfile(self.FOO)) self.assertTrue(filecmp.cmp(self.FOO, self.orig, shallow=False))
def main(): parser = MyParser(description='This application create components files from cdsl files or .ice from idsl\n' '\ta) to generate code from a CDSL file: ' + sys.argv[0].split('/')[-1] + ' INPUT_FILE.CDSL OUTPUT_PATH\n' +'\tb) to generate a new CDSL file: ' + sys.argv[0].split('/')[-1] + ' NEW_COMPONENT_DESCRIPTOR.CDSL', formatter_class=argparse.RawTextHelpFormatter) parser.add_argument("-I", "--include_dirs", nargs='*', help="Include directories", action=FullPaths, default=[]) parser.add_argument("-d", '--diff', dest='diff', choices=DIFF_TOOLS, action='store') parser.add_argument("input_file", help="The input dsl file") parser.add_argument("output_path", nargs='?', help="The path to put the files") args = parser.parse_args() if args.output_path is None: if args.input_file.endswith(".cdsl"): generateDummyCDSL(args.input_file) generateDummySMDSL("statemachine.smdsl") sys.exit(0) else: print(args.output_path, args.input_file) print(parser.error("No output path with non .cdsl file")) sys.exit(-1) inputFile = args.input_file outputPath = args.output_path sys.path.append('/opt/robocomp/python') new_existing_files = {} if inputFile.endswith(".cdsl"): component = DSLFactory().from_file(inputFile, includeDirectories=args.include_dirs) imports = ''.join( [ imp+'#' for imp in component['imports'] ] ) # verification pool = IDSLPool(imports, args.include_dirs) interface_list = component['requires'] + component['implements'] + component['subscribesTo'] + component['publishes'] for interface_required in interface_list: interface_required = interface_required if isinstance(interface_required, str) else interface_required[0] if not pool.moduleProviding(interface_required): raise rcExceptions.InterfaceNotFound(interface_required, pool.interfaces()) if component['language'].lower() == 'cpp' or component['language'].lower() == 'cpp11': # # Check output directory # if not os.path.exists(outputPath): create_directory(outputPath) # Create directories within the output directory try: create_directory(outputPath + "/bin") create_directory(outputPath + "/etc") create_directory(outputPath + "/src") except: print('There was a problem creating a directory') sys.exit(1) pass # # Generate regular files # files = [ 'CMakeLists.txt', 'DoxyFile', 'README-STORM.txt', 'README.md', 'etc/config', 'src/main.cpp', 'src/CMakeLists.txt', 'src/CMakeListsSpecific.txt', 'src/commonbehaviorI.h', 'src/commonbehaviorI.cpp', 'src/genericmonitor.h', 'src/genericmonitor.cpp', 'src/config.h', 'src/specificmonitor.h', 'src/specificmonitor.cpp', 'src/genericworker.h', 'src/genericworker.cpp', 'src/specificworker.h', 'src/specificworker.cpp', 'src/mainUI.ui' ] specificFiles = [ 'src/specificworker.h', 'src/specificworker.cpp', 'src/CMakeListsSpecific.txt', 'src/mainUI.ui', 'src/specificmonitor.h', 'src/specificmonitor.cpp', 'README.md', 'etc/config' ] for f in files: ofile = outputPath + '/' + f if f in specificFiles and os.path.exists(ofile): print('Not overwriting specific file "'+ ofile +'", saving it to '+ofile+'.new') new_existing_files[os.path.abspath(ofile)] = os.path.abspath(ofile)+'.new' ofile += '.new' ifile = "/opt/robocomp/share/robocompdsl/templateCPP/" + f if f != 'src/mainUI.ui' or component['gui'] is not None: print('Generating', ofile) run = "cog.py -z -d -D theCDSL="+inputFile + " -D theIDSLs="+imports + ' -D theIDSLPaths='+ '#'.join(args.include_dirs) + " -o " + ofile + " " + ifile run = run.split(' ') ret = Cog().main(run) if ret != 0: print('ERROR') sys.exit(-1) replaceTagsInFile(ofile) # # Generate interface-dependent files # for ima in component['implements']: im = ima if type(im) != type(''): im = im[0] if communication_is_ice(ima): for f in [ "SERVANT.H", "SERVANT.CPP"]: ofile = outputPath + '/src/' + im.lower() + 'I.' + f.split('.')[-1].lower() print('Generating ', ofile, ' (servant for', im + ')') # Call cog run = "cog.py -z -d -D theCDSL="+inputFile + " -D theIDSLs="+imports + ' -D theIDSLPaths='+ '#'.join(args.include_dirs) + " -D theInterface="+im + " -o " + ofile + " " + "/opt/robocomp/share/robocompdsl/templateCPP/" + f run = run.split(' ') ret = Cog().main(run) if ret != 0: print('ERROR') sys.exit(-1) replaceTagsInFile(ofile) for imp in component['subscribesTo']: im = imp if type(im) != type(''): im = im[0] if communication_is_ice(imp): for f in [ "SERVANT.H", "SERVANT.CPP"]: ofile = outputPath + '/src/' + im.lower() + 'I.' + f.split('.')[-1].lower() print('Generating ', ofile, ' (servant for', im + ')') # Call cog theInterfaceStr = im if type(theInterfaceStr) == type([]): theInterfaceStr = str(';'.join(im)) run = "cog.py -z -d -D theCDSL="+inputFile + " -D theIDSLs="+imports + ' -D theIDSLPaths='+ '#'.join(args.include_dirs) + " -D theInterface="+theInterfaceStr + " -o " + ofile + " " + "/opt/robocomp/share/robocompdsl/templateCPP/" + f #print(run run = run.split(' ') ret = Cog().main(run) if ret != 0: print('ERROR') sys.exit(-1) replaceTagsInFile(ofile) elif component['language'].lower() == 'python': # # Check output directory # if not os.path.exists(outputPath): create_directory(outputPath) # Create directories within the output directory try: create_directory(outputPath + "/etc") create_directory(outputPath + "/src") except: print('There was a problem creating a directory') sys.exit(1) pass needStorm = False for pub in component['publishes']: if communication_is_ice(pub): needStorm = True for sub in component['subscribesTo']: if communication_is_ice(sub): needStorm = True # # Generate regular files # files = [ 'CMakeLists.txt', 'DoxyFile', 'README-STORM.txt', 'README.md', 'etc/config', 'src/main.py', 'src/genericworker.py', 'src/specificworker.py', 'src/mainUI.ui' ] specificFiles = [ 'src/specificworker.py', 'src/mainUI.ui', 'README.md', 'etc/config' ] for f in files: if f == 'src/main.py': ofile = outputPath + '/src/' + component['name'] + '.py' else: ofile = outputPath + '/' + f if f in specificFiles and os.path.exists(ofile): print('Not overwriting specific file "'+ ofile +'", saving it to '+ofile+'.new') new_existing_files[os.path.abspath(ofile)] = os.path.abspath(ofile) + '.new' ofile += '.new' ifile = "/opt/robocomp/share/robocompdsl/templatePython/" + f ignoreFile = False if f == 'src/mainUI.ui' and component['gui'] is None: ignoreFile = True if f == 'CMakeLists.txt' and component['gui'] is None: ignoreFile = True if f == 'README-STORM.txt' and needStorm == False: ignoreFile = True if not ignoreFile: print('Generating', ofile) run = "cog.py -z -d -D theCDSL="+inputFile + " -D theIDSLs="+imports + ' -D theIDSLPaths='+ '#'.join(args.include_dirs) + " -o " + ofile + " " + ifile run = run.split(' ') ret = Cog().main(run) if ret != 0: print('ERROR') sys.exit(-1) replaceTagsInFile(ofile) if f == 'src/main.py': os.chmod(ofile, os.stat(ofile).st_mode | 0o111) # # Generate interface-dependent files # for imp in component['implements']+component['subscribesTo']: if type(imp) != type(''): im = imp[0] else: im = imp if communication_is_ice(imp): for f in [ "SERVANT.PY"]: ofile = outputPath + '/src/' + im.lower() + 'I.' + f.split('.')[-1].lower() print('Generating', ofile, ' (servant for', im + ')') # Call cog run = "cog.py -z -d -D theCDSL="+inputFile + " -D theIDSLs="+imports + ' -D theIDSLPaths='+ '#'.join(args.include_dirs) + " -D theInterface="+im + " -o " + ofile + " " + "/opt/robocomp/share/robocompdsl/templatePython/" + f run = run.split(' ') ret = Cog().main(run) if ret != 0: print('ERROR') sys.exit(-1) replaceTagsInFile(ofile) else: print('Unsupported language', component['language']) if component['usingROS'] == True: for imp in component['imports']: generate_ROS_headers(imp, outputPath + "/src", component, args.include_dirs) # Code to launch diff tool on .new files to be compared with their old version if args.diff is not None: diff_tool,_ = get_diff_tool(prefered=args.diff) print("Executing diff tool for existing files. Close if no change is needed.") for o_file, n_file in new_existing_files.items(): if not filecmp.cmp(o_file,n_file): print([diff_tool, o_file, n_file]) try: subprocess.call([diff_tool, o_file, n_file]) except KeyboardInterrupt as e: print("Comparasion interrupted. All files have been generated. Check this .new files manually:") for o_file2, n_file2 in new_existing_files.items(): if not filecmp.cmp(o_file2, n_file2): print("%s %s"%(o_file2,n_file2)) break except Exception as e: print("Exception trying to execute %s"%(diff_tool)) print(e.message) else: print("Binary equal files %s and %s"%(o_file, n_file)) elif inputFile.endswith(".idsl"): # idsl = IDSLParsing.fromFileIDSL(inputFile) print('Generating ICE file ', outputPath) # Call cog run = "cog.py -z -d" + " -D theIDSL="+inputFile + ' -D theIDSLPaths='+ '#'.join(args.include_dirs) +" -o " + outputPath + " /opt/robocomp/share/robocompdsl/TEMPLATE.ICE" run = run.split(' ') ret = Cog().main(run) if ret != 0: print('ERROR') sys.exit(-1) replaceTagsInFile(outputPath)
def test_common_convert_input_folder(self, copy_test_data_to_data_folder): # Execute the target tool on the input file with the arguments. command = [ str(self._executable) ] # Insert options. for key, value in self._data.options.items(): command.append(key) # Handle file arguments by discarding the empty value. if value != "": command.append(value) # Handle files to auto include. for f in self._data.input: if f.auto_include: target_path = self._data_folder / f.relative_path / f.name command.append(str(target_path)) # Ensure the data path exists. if not self._data_folder.exists(): self._data_folder.mkdir(parents=True) if self._data.name is not None: print(f"Starting test with comment: {self._data.name}") print(f"Executing command: {command}") # Perform test. call_result = subprocess.run(command, cwd=self._data_folder) # Ensure the return code matches the expected code. return_code = call_result.returncode assert return_code == self._data.return_code, f"Unexpected returncode. Expected {self._data.return_code} but got {return_code}." # Ensure the expected files are present. expected_output_files = [] for output_file in self._data.output: target_path = self._data_folder / output_file.relative_path / output_file.name expected_output_files.append(target_path) missing_files = [] for expected_output_file in expected_output_files: if not expected_output_file.exists(): missing_files.append(expected_output_file) assert len(missing_files) == 0, f"Missing expected files: {missing_files}" # Ensure the file contents match. comparison_errors = [] for output_file in self._data.output: # Find the data file to compare against. if output_file.comparision_data is None: output_file.comparision_data = output_file.name comparison_file = get_tool_data(tool=self._data.tool, name=output_file.comparision_data) data_file = self._data_folder / output_file.relative_path / output_file.name # Determine comparison to perform based on file extension. extension = data_file.suffix.lower() if extension in [".log", ".trc", ".asc", ".txt", ".csv"]: comparison_result = compare_ignore_line_ending(comparison_file, data_file) else: comparison_result = filecmp.cmp(comparison_file, data_file) if not comparison_result: comparison_errors.append(output_file.name) assert len(comparison_errors) == 0, f"Mismatch in output data for the following file(s): {comparison_errors}" return
import os, urllib, json, filecmp, shutil ## set params airtable_url = "<insert the airtable api url for your table>" airtable_key = "<insert your airtable api key>" url = "{}?api_key={}".format(airtable_url, airtable_key) dest_file = os.path.join("example", "table-data.json") temp_file = dest_file.replace(".json", "-tmp.json") ## retrieve data from airtable and write to file response = urllib.urlopen(url) data = json.loads(response.read()) with open(temp_file, "wb") as outfile: json.dump(data, outfile, indent=1) ## compare new file to existing file and replace if necessary if not filecmp.cmp(temp_file, dest_file): shutil.copy(temp_file, dest_file) os.remove(temp_file)
f.close() # you can omit in most cases as the destructor will call if f = open('redirect_target.html', 'w') f.close() with open("./.htaccess", "a") as myfile: myfile.write("RewriteRule ^redirect_target.html$ $2fetch_target.html [L]") chmod('./.htaccess', 0644) chmod('fetch_target.html', 0644) chmod('redirect_target.html', 0644) system( "wget -O- https://www.cs.ubc.ca/~mglgms/mta/redirect_target.html >out.txt 2> /dev/null" ) import filecmp if filecmp.cmp('out.txt', 'fetch_target.html'): print 'htaccess works. Hurray!' else: remove('.htaccess') remove('out.txt') user = raw_input("Administrator User: "******"admin" print "Administrator '" + user + "' created" while True: password = getpass.getpass("Administrator Password: "******"Re-type Administrator Password: ") if password == password_match: break print 'Password was re-typed incorrectly' entries = []
def compare(file1, file2): if (filecmp.cmp(file1, file2) == False): print("Failed test in {}".format(os.path.dirname(file1))) else: os.remove(file2)
def run_ota(source, target, payload_path, tempdir, output_dir): """Run an OTA on host side""" payload = update_payload.Payload(payload_path) payload.Init() if source and zipfile.is_zipfile(source): source = zipfile.ZipFile(source) if target and zipfile.is_zipfile(target): target = zipfile.ZipFile(target) source_exist = source and (isinstance(source, zipfile.ZipFile) or os.path.exists(source)) target_exist = target and (isinstance(target, zipfile.ZipFile) or os.path.exists(target)) old_partitions = [] new_partitions = [] expected_new_partitions = [] for part in payload.manifest.partitions: name = part.partition_name old_image = os.path.join(tempdir, "source_" + name + ".img") new_image = os.path.join(tempdir, "target_" + name + ".img") if part.HasField("old_partition_info"): assert source_exist, \ "source target file must point to a valid zipfile or directory " + \ source print("Extracting source image for", name) extract_img(source, name, old_image) if target_exist: print("Extracting target image for", name) extract_img(target, name, new_image) old_partitions.append(old_image) scratch_image_name = new_image + ".actual" new_partitions.append(scratch_image_name) with open(scratch_image_name, "wb") as fp: fp.truncate(part.new_partition_info.size) expected_new_partitions.append(new_image) delta_generator_args = ["delta_generator", "--in_file=" + payload_path] partition_names = [ part.partition_name for part in payload.manifest.partitions ] if payload.manifest.partial_update: delta_generator_args.append("--is_partial_update") if payload.is_incremental: delta_generator_args.append("--old_partitions=" + ":".join(old_partitions)) delta_generator_args.append("--partition_names=" + ":".join(partition_names)) delta_generator_args.append("--new_partitions=" + ":".join(new_partitions)) print("Running ", " ".join(delta_generator_args)) subprocess.check_output(delta_generator_args) valid = True if not target_exist: for part in new_partitions: print("Output written to", part) shutil.copy(part, output_dir) return for (expected_part, actual_part, part_name) in \ zip(expected_new_partitions, new_partitions, partition_names): if filecmp.cmp(expected_part, actual_part): print("Partition `{}` is valid".format(part_name)) else: valid = False print( "Partition `{}` is INVALID expected image: {} actual image: {}" .format(part_name, expected_part, actual_part)) if not valid and sys.stdout.isatty(): input( "Paused to investigate invalid partitions, press any key to exit.")
def recreate_cm(folder_path: str): # read inputs gather_designs = {} with open(os.path.join(folder_path, "FINAL_summary"), "r") as input_sum: input_sum.readline() for line in input_sum: if line.strip() == '': continue parts = line.strip().split('\t') gather_designs[parts[0]] = parts[5] gather_results = {} with open(os.path.join(folder_path, "FINAL_all"), 'r') as input_all: input_all.readline() for line in input_all: if line.strip() == '': continue parts = line.strip().split('\t') res_map = gather_results.get(parts[0], {}) res_map[parts[1]] = parts[4] gather_results[parts[0]] = res_map # start calculations folder = vienna.LiveRNAfold() folder.start() for design_code, sequence in gather_designs.items(): structure = folder.fold(sequence)['MFE'] cm_path = os.path.join(folder_path, "{}.cm".format(design_code)) sto_path = os.path.join(folder_path, "{}.sto".format(design_code)) if os.path.exists(cm_path): continue temp_cm_path = "{}_tmp".format(cm_path) temp_sto_path = "{}_tmp".format(sto_path) if not infernal.generate_single_seq_cm(sequence, cm_path, structure): print("Could not generate single cm for {}".format(design_code)) exit(-1) if not infernal.align_sequences({'{}'.format(design_code): sequence}, cm_path, sto_path): print("Could not generate single sto for {}".format(design_code)) exit(-1) design_results = gather_results.get(design_code) no_found = 0 temp_fasta = infernal.generate_fasta(design_results) while no_found < len(design_results): results = infernal.search_cm(cm_path, temp_fasta.name, inc_e=10.0) sto_parts = {} sto_target = get_sto_targets(sto_path) for item in results: if item['target name'] not in sto_target: sto_parts[item['target name']] = item['sequence'] if len(sto_parts) == 0: print( "ERROR: no new sequences found for {} maxed at {} sequences out of {} original\nListing: {}" .format(design_code, len(sto_target), len(design_results), [ res for res in design_results.keys() if res not in get_sto_targets(sto_path) ])) break if not infernal.align_sequences( sto_parts, cm_path, temp_sto_path, in_align_path=sto_path): print("Could not generate sto for {}".format(design_code)) exit(-1) if filecmp.cmp(sto_path, temp_sto_path, shallow=False): print("ERROR: {} missing codes: {}".format( design_code, [ res for res in design_results.keys() if res not in get_sto_targets(sto_path) ])) shutil.move(temp_sto_path, sto_path) break shutil.move(temp_sto_path, sto_path) if not infernal.generate_cm(sto_path, temp_cm_path): print("Could not generate cm for {}".format(design_code)) exit(-1) shutil.move(temp_cm_path, cm_path) no_found = len(results) os.remove(temp_fasta.name)
def cmp(f1, f2): return filecmp.cmp(f1, f2, shallow=True)
def test_imagefolder(remove_json_files=True): """ Test simulating resnet50 dataset pipeline. """ data_dir = "../data/dataset/testPK/data" ds.config.set_seed(1) # define data augmentation parameters rescale = 1.0 / 255.0 shift = 0.0 resize_height, resize_width = 224, 224 weights = [ 1.0, 0.1, 0.02, 0.3, 0.4, 0.05, 1.2, 0.13, 0.14, 0.015, 0.16, 1.1 ] # Constructing DE pipeline sampler = ds.WeightedRandomSampler(weights, 11) data1 = ds.ImageFolderDatasetV2(data_dir, sampler=sampler) data1 = data1.repeat(1) data1 = data1.map(input_columns=["image"], operations=[vision.Decode(True)]) rescale_op = vision.Rescale(rescale, shift) resize_op = vision.Resize((resize_height, resize_width), Inter.LINEAR) data1 = data1.map(input_columns=["image"], operations=[rescale_op, resize_op]) data1 = data1.batch(2) # Serialize the dataset pre-processing pipeline. # data1 should still work after saving. ds.serialize(data1, "imagenet_dataset_pipeline.json") ds1_dict = ds.serialize(data1) assert validate_jsonfile("imagenet_dataset_pipeline.json") is True # Print the serialized pipeline to stdout ds.show(data1) # Deserialize the serialized json file data2 = ds.deserialize(json_filepath="imagenet_dataset_pipeline.json") # Serialize the pipeline we just deserialized. # The content of the json file should be the same to the previous serialize. ds.serialize(data2, "imagenet_dataset_pipeline_1.json") assert validate_jsonfile("imagenet_dataset_pipeline_1.json") is True assert filecmp.cmp('imagenet_dataset_pipeline.json', 'imagenet_dataset_pipeline_1.json') # Deserialize the latest json file again data3 = ds.deserialize(json_filepath="imagenet_dataset_pipeline_1.json") data4 = ds.deserialize(input_dict=ds1_dict) num_samples = 0 # Iterate and compare the data in the original pipeline (data1) against the deserialized pipeline (data2) for item1, item2, item3, item4 in zip(data1.create_dict_iterator(), data2.create_dict_iterator(), data3.create_dict_iterator(), data4.create_dict_iterator()): assert np.array_equal(item1['image'], item2['image']) assert np.array_equal(item1['image'], item3['image']) assert np.array_equal(item1['label'], item2['label']) assert np.array_equal(item1['label'], item3['label']) assert np.array_equal(item3['image'], item4['image']) assert np.array_equal(item3['label'], item4['label']) num_samples += 1 logger.info("Number of data in data1: {}".format(num_samples)) assert num_samples == 6 # Remove the generated json file if remove_json_files: delete_json_files()
def test_cmap2act(self): """Export colormap to act file.""" colors.cmap2act('viridis', filename=self.f) ref = os.path.join(self.ref_dir, 'viridis.act') assert filecmp.cmp(self.f, ref)
from_file = FileSource() image_sources.append(from_file) image_sources.append(CalCamSaveSource()) example_file = os.path.join(paths.calcampath, 'usercode_examples', 'image_source.py_') user_files = [ fname for fname in os.listdir(paths.image_sources) if fname.endswith('.py') ] # See if the user already has a CAD model definition example file, and if it's up to date. If not, create it. # If the definitions might have changed, warn the user. if 'Example.py' in user_files: is_current_version = filecmp.cmp(os.path.join(paths.image_sources, 'Example.py'), example_file, shallow=False) if not is_current_version: shutil.copy2(example_file, os.path.join(paths.image_sources, 'Example.py')) print( '[Calcam Import] The latest image source definition example is different from your user copy. Your existing copy has been updated. If you get image source related errors, you may need to check and edit the CAD definition files in ' + paths.image_sources) user_files.remove('Example.py') else: shutil.copy2(example_file, os.path.join(paths.image_sources, 'Example.py')) print('[Calcam Import] Created image source definition example in ' + os.path.join(paths.image_sources, 'Example.py')) user_im_sources = [] # Go through all the python files which aren't examples, and import the CAD definitions
def compare_files(self, file1: PATH_TYPES, file2: PATH_TYPES) -> bool: """Compare two files, returning true if they are the same and False if not.""" return filecmp.cmp(str(file1), str(file2), shallow=False)
def generate_exercise(env, spec_path, exercise, check=False): """ Renders test suite for exercise and if check is: True: verifies that current tests file matches rendered False: saves rendered to tests file """ slug = os.path.basename(exercise) meta_dir = os.path.join(exercise, ".meta") plugins_module = None plugins_name = "plugins" plugins_source = os.path.join(meta_dir, f"{plugins_name}.py") try: if os.path.isfile(plugins_source): plugins_spec = importlib.util.spec_from_file_location( plugins_name, plugins_source ) plugins_module = importlib.util.module_from_spec(plugins_spec) sys.modules[plugins_name] = plugins_module plugins_spec.loader.exec_module(plugins_module) spec = load_canonical(slug, spec_path) additional_tests = load_additional_tests(slug) spec["additional_cases"] = additional_tests template_path = posixpath.join(slug, ".meta", "template.j2") template = env.get_template(template_path) tests_path = os.path.join(exercise, f"{to_snake(slug)}_test.py") spec["has_error_case"] = has_error_case(spec["cases"]) if plugins_module is not None: spec[plugins_name] = plugins_module logger.debug(f"{slug}: attempting render") rendered = template.render(**spec) with NamedTemporaryFile("w", delete=False) as tmp: logger.debug(f"{slug}: writing render to tmp file {tmp.name}") tmp.write(rendered) try: logger.debug(f"{slug}: formatting tmp file {tmp.name}") format_file(tmp.name) except FileNotFoundError as e: logger.error(f"{slug}: the black utility must be installed") return False if check: try: check_ok = True if not os.path.isfile(tmp.name): logger.debug(f"{slug}: tmp file {tmp.name} not found") check_ok = False if not os.path.isfile(tests_path): logger.debug(f"{slug}: tests file {tests_path} not found") check_ok = False if check_ok and not filecmp.cmp(tmp.name, tests_path): with open(tests_path) as f: current_lines = f.readlines() with open(tmp.name) as f: rendered_lines = f.readlines() diff = difflib.unified_diff( current_lines, rendered_lines, fromfile=f"[current] {os.path.basename(tests_path)}", tofile=f"[generated] {tmp.name}", ) logger.debug(f"{slug}: ##### DIFF START #####") for line in diff: logger.debug(line.strip()) logger.debug(f"{slug}: ##### DIFF END #####") check_ok = False if not check_ok: logger.error( f"{slug}: check failed; tests must be regenerated with bin/generate_tests.py" ) return False logger.debug(f"{slug}: check passed") finally: logger.debug(f"{slug}: removing tmp file {tmp.name}") os.remove(tmp.name) else: logger.debug(f"{slug}: moving tmp file {tmp.name}->{tests_path}") shutil.move(tmp.name, tests_path) print(f"{slug} generated at {tests_path}") except (TypeError, UndefinedError, SyntaxError) as e: logger.debug(str(e)) logger.error(f"{slug}: generation failed") return False except TemplateNotFound as e: logger.debug(str(e)) logger.info(f"{slug}: no template found; skipping") except FileNotFoundError as e: logger.debug(str(e)) logger.info(f"{slug}: no canonical data found; skipping") return True
try: conf = json.loads(cs) except: logging.error('json parse err') return False version_url = conf['AutoUpdate']['version'] zip_url = conf['AutoUpdate']['package'] logging.debug('to download %s' % version_url) if not __dlfile(version_url, 'tmp_version'): logging.error('can\'t download version file, to check url') return False logging.debug('to compare version') if os.path.isfile('local_version') and filecmp.cmp('local_version', 'tmp_version'): logging.debug('version NOT changed!') return False logging.debug('to download update zip package, from %s' % zip_url) if not __dlfile(zip_url, 'tmp_package.zip'): logging.error('can\'t download update package, to check url!!!') return False try: extract_path = tempfile.mkdtemp('.zonekey', 'au.') logging.debug('using temp path of %s' % extract_path) except: logging.error( 'can\'t create tmp path for extract?? to check disk free space!') return False
def hsdis(args, copyToDir=None): """download the hsdis library This is needed to support HotSpot's assembly dumping features. By default it downloads the Intel syntax version, use the 'att' argument to install AT&T syntax.""" flavor = None if mx.get_arch() == "amd64": flavor = mx.get_env('HSDIS_SYNTAX') if flavor is None: flavor = 'intel' if 'att' in args: flavor = 'att' libpattern = mx.add_lib_suffix('hsdis-' + mx.get_arch() + '-' + mx.get_os() + '-%s') sha1s = { 'att/hsdis-amd64-windows-%s.dll': 'bcbd535a9568b5075ab41e96205e26a2bac64f72', 'att/hsdis-amd64-linux-%s.so': '36a0b8e30fc370727920cc089f104bfb9cd508a0', 'att/hsdis-amd64-darwin-%s.dylib': 'c1865e9a58ca773fdc1c5eea0a4dfda213420ffb', 'intel/hsdis-amd64-windows-%s.dll': '6a388372cdd5fe905c1a26ced614334e405d1f30', 'intel/hsdis-amd64-linux-%s.so': '0d031013db9a80d6c88330c42c983fbfa7053193', 'intel/hsdis-amd64-darwin-%s.dylib': '67f6d23cbebd8998450a88b5bef362171f66f11a', 'hsdis-sparcv9-solaris-%s.so': '970640a9af0bd63641f9063c11275b371a59ee60', 'hsdis-sparcv9-linux-%s.so': '0c375986d727651dee1819308fbbc0de4927d5d9', } if flavor: flavoredLib = flavor + "/" + libpattern else: flavoredLib = libpattern if flavoredLib not in sha1s: mx.warn( "hsdis with flavor '{}' not supported on this platform or architecture" .format(flavor)) return sha1 = sha1s[flavoredLib] lib = flavoredLib % (sha1) path = join(_suite.get_output_root(), lib) if not exists(path): sha1path = path + '.sha1' mx.download_file_with_sha1( 'hsdis', path, ['https://lafo.ssw.uni-linz.ac.at/pub/hsdis/' + lib], sha1, sha1path, True, True, sources=False) overwrite = True if copyToDir is None: # Try install hsdis into JAVA_HOME overwrite = False base = mx.get_jdk().home if exists(join(base, 'jre')): copyToDir = join(base, 'jre', 'lib') else: copyToDir = join(base, 'lib') if exists(copyToDir): dest = join(copyToDir, mx.add_lib_suffix('hsdis-' + mx.get_arch())) if exists(dest) and not overwrite: import filecmp # Only issue warning if existing lib is different if filecmp.cmp(path, dest) == False: mx.warn('Not overwriting existing {} with {}'.format( dest, path)) else: try: shutil.copy(path, dest) mx.log('Copied {} to {}'.format(path, dest)) except IOError as e: mx.warn('Could not copy {} to {}: {}'.format( path, dest, str(e)))
def find_mozconfig(self, env=os.environ): """Find the active mozconfig file for the current environment. This emulates the logic in mozconfig-find. 1) If ENV[MOZCONFIG] is set, use that 2) If $TOPSRCDIR/mozconfig or $TOPSRCDIR/.mozconfig exists, use it. 3) If both exist or if there are legacy locations detected, error out. The absolute path to the found mozconfig will be returned on success. None will be returned if no mozconfig could be found. A MozconfigFindException will be raised if there is a bad state, including conditions from #3 above. """ # Check for legacy methods first. if 'MOZ_MYCONFIG' in env: raise MozconfigFindException(MOZ_MYCONFIG_ERROR) env_path = env.get('MOZCONFIG', None) or None if env_path is not None: if not os.path.isabs(env_path): potential_roots = [self.topsrcdir, os.getcwd()] # Attempt to eliminate duplicates for e.g. # self.topsrcdir == os.curdir. potential_roots = set( os.path.abspath(p) for p in potential_roots) existing = [ root for root in potential_roots if os.path.exists(os.path.join(root, env_path)) ] if len(existing) > 1: # There are multiple files, but we might have a setup like: # # somedirectory/ # srcdir/ # objdir/ # # MOZCONFIG=../srcdir/some/path/to/mozconfig # # and be configuring from the objdir. So even though we # have multiple existing files, they are actually the same # file. mozconfigs = [ os.path.join(root, env_path) for root in existing ] if not all( map( lambda p1, p2: filecmp.cmp( p1, p2, shallow=False), mozconfigs[:-1], mozconfigs[1:])): raise MozconfigFindException( 'MOZCONFIG environment variable refers to a path that ' + 'exists in more than one of ' + ', '.join(potential_roots) + '. Remove all but one.') elif not existing: raise MozconfigFindException( 'MOZCONFIG environment variable refers to a path that ' + 'does not exist in any of ' + ', '.join(potential_roots)) env_path = os.path.join(existing[0], env_path) elif not os.path.exists(env_path): # non-relative path raise MozconfigFindException( 'MOZCONFIG environment variable refers to a path that ' 'does not exist: ' + env_path) if not os.path.isfile(env_path): raise MozconfigFindException( 'MOZCONFIG environment variable refers to a ' 'non-file: ' + env_path) srcdir_paths = [ os.path.join(self.topsrcdir, p) for p in self.DEFAULT_TOPSRCDIR_PATHS ] existing = [p for p in srcdir_paths if os.path.isfile(p)] if env_path is None and len(existing) > 1: raise MozconfigFindException('Multiple default mozconfig files ' 'present. Remove all but one. ' + ', '.join(existing)) path = None if env_path is not None: path = env_path elif len(existing): assert len(existing) == 1 path = existing[0] if path is not None: return os.path.abspath(path) deprecated_paths = [ os.path.join(self.topsrcdir, s) for s in self.DEPRECATED_TOPSRCDIR_PATHS ] home = env.get('HOME', None) if home is not None: deprecated_paths.extend( [os.path.join(home, s) for s in self.DEPRECATED_HOME_PATHS]) for path in deprecated_paths: if os.path.exists(path): raise MozconfigFindException(MOZCONFIG_LEGACY_PATH % (path, self.topsrcdir)) return None
def generate_report(): # Reference to variables containing details for each rack global rack_1_name, rack_2_name global rack_1_fullpath, rack_2_fullpath global rack_1_outputdir, rack_2_outputdir global rack_1_shortpath, rack_2_shortpath global rack_1_node_id_map, rack_2_node_id_map html_report_path = os.getcwd() + "/rack_comparison_" + strftime( "%y%m%d_%H%M%S", localtime()) + ".html" # Initiate a html report fd = open(html_report_path, 'a') try: # Print the pre-formatted basic html tags to the output file before processing the main content fd.write(print_header()) # Print the header row of the main table which will contain the details on each parameters fd.write(""" <table class="main"> <tr border=5px> <th class="param_name" rowspan=2>Parameter Name</th> <th class="param_name" rowspan=2>Component Type</th> <th class="rack_col" colspan=2>Rack """ + rack_1_shortpath + """</th> <th class="rack_col" colspan=2>Rack """ + rack_2_shortpath + """</th> <th class="diff_col" rowspan=2>Difference</th> <th class="status" rowspan=2>Status</th> </tr> <tr border=5px> <th width: 7%;>Node Name</th> <th width: 20%;>Details</th> <th width: 7%>Node Name</th> <th width:20%>Details</th> </tr> """) global temp_identical_path iden_fd = open(temp_identical_path, 'a+') global temp_different_path diff_fd = open(temp_different_path, 'a+') # Loop through each check as gathered in the check_list set for checkname in sorted(check_list.keys()): rowspan = str(len(check_list[checkname])) count = 0 global total_count, identical_count total_count += 1 identical_bool = "true" buffer = "" # Loop through data from each node regarding 'checkname' variable for node_id in sorted(check_list[checkname]): buffer = buffer + "<tr border=5px>\n" checkname_string = checkname.replace('_', ' ') if count == 0: # Prepare the cell containing the name of the parameter to be checked buffer = buffer + "<td rowspan=" + rowspan + ">" + checkname_string + "</td>\n" #print checkname #print sorted(check_list[checkname]) buffer = buffer + "<td>" + to_string(node_id) + "</td>" rack_1_check_filename = rack_1_outputdir + checkname + "__" + node_id rack_2_check_filename = rack_2_outputdir + checkname + "__" + node_id # Prepare the columns containing data for each parameter to be checked, gathered from each rack respectively buffer = buffer + prepare_rack_columns( fd, rack_1_node_id_map, node_id, rack_1_shortpath, checkname, rack_1_check_filename) buffer = buffer + prepare_rack_columns( fd, rack_2_node_id_map, node_id, rack_2_shortpath, checkname, rack_2_check_filename) # Prepare the column containing differences for each parameter between two racks buffer = buffer + "<td>" if (os.path.exists(rack_1_check_filename) and os.path.exists(rack_2_check_filename)): diff_str = subprocess.Popen( ["diff", rack_1_check_filename, rack_2_check_filename], stdout=subprocess.PIPE).communicate()[0] if not diff_str or diff_str.isspace(): diff_str = "No difference" diff_stream = cStringIO.StringIO(diff_str) div_id = "diff_" + node_id + "_" + checkname buffer = buffer + print_content(diff_stream, div_id) else: if (not os.path.exists(rack_1_check_filename)): buffer = buffer + "Data from Rack 1 is unavailable for this parameter<br/>" if (not os.path.exists(rack_2_check_filename)): buffer = buffer + "Data from Rack 2 is unavailable for this parameter<br/>" buffer = buffer + "</td>" # Prepare the column containing the status of each parameter, i.e. whether it's identical on both racks or not if (os.path.exists(rack_1_check_filename) and os.path.exists(rack_2_check_filename) and filecmp.cmp(rack_1_check_filename, rack_2_check_filename)): status = "<span class=\"status_PASS\">IDENTICAL</span>" else: status = "<span class=\"status_FAIL\">DIFFERENT</span>" identical_bool = "false" buffer = buffer + "<td>" + status + "</td>" buffer = buffer + "</tr>" count += 1 if identical_bool is "true": identical_count += 1 iden_fd.write(buffer) else: diff_fd.write(buffer) # Reset the boolean tracker for the next check identical_bool = "true" # After all the rows have been processed, flush data rows to the html report # organized by parameters that are different to be shown earlier, and identical latter towards the bottom diff_fd.seek(0) fd.write(diff_fd.read()) diff_fd.close() iden_fd.seek(0) fd.write(iden_fd.read()) iden_fd.close() # End of the loop, print the rest of the footer html tags to complete the html report fd.write(""" </table> </body> <br><a href=\"#\" onclick=\"javascript:processForm();\"><div id=\"results\">Switch to old format</div></a> </html> """) finally: fd.close() # Append a summary table at the top of the html report insert_summary_table(html_report_path) return html_report_path
def copyfile(src, dst, rnx_ver=2): """ Copies a file from path src to path dst. If a file already exists at dst, it will not be overwritten, but: * If it is the same as the source file, do nothing * If it is different to the source file, pick a new name for the copy that is different and unused, then copy the file there (if rnx_ver=2) * If because rinex 3 files have names that are more comprehensive (include start time and duration) if a rnx_ver == 3 then copy the file unless it already exists (in which case it does nothing) Returns the path to the copy. """ if not os.path.exists(src): raise ValueError('Source file does not exist: {}'.format(src)) # make the folders if they don't exist # careful! racing condition between different workers try: dst_dir = os.path.dirname(dst) if not os.path.exists(dst_dir): os.makedirs(dst_dir) except OSError: # some other process created the folder an instant before pass # Keep trying to copy the file until it works if rnx_ver < 3: # only use this method for RINEX 2 # RINEX 3 files should have distinct names as a default if the files are different dst_gen = _increment_filename(dst) while True: if rnx_ver < 3: dst = next(dst_gen) # Check if there is a file at the destination location if os.path.exists(dst): # If the namesake is the same as the source file, then we don't # need to do anything else. if filecmp.cmp(src, dst): return dst else: # DDG: if the rinex version is == 3 and the files have the same name: # 1) if dst size is < than src, replace file # 2) if dst size is > than src, do nothing # for RINEX 2 files, loop over and find a different filename if rnx_ver >= 3: if os.path.getsize(src) > os.path.getsize(dst): os.remove(dst) if do_copy_op(src, dst): return dst else: raise OSError('File exists during copy of RINEX 3 file: ' + dst) else: return dst else: if do_copy_op(src, dst): # If we get to this point, then the write has succeeded return dst else: if rnx_ver >= 3: raise OSError('Problem while copying RINEX 3 file: ' + dst)
def gen_equal_output_map(config): """Formats 'in_files' with configs inside the 'out_dir' with Uncrustify and groups formatted files with equal content together. Expects config filename format generated by write_config_files :param config: configuration object, expects that it was processed by check_config :return: dict of files with equal content key -- group index value -- filepath list """ # maps that will hold configurations that produce the same formatted files equal_output_map = {} # map len counter map_val_idx = 0 # iterate through all generated config file names for cfg_path in sorted(iglob('%s/*.cfg' % config["out_dir"])): for in_file_idx in range(len(config["in_files"])): # extract substring form config gile name (removes __unc.cfg) splits_file = cfg_path.split("__unc") if len(splits_file) < 1: raise Exception('split with "__unc" | Wrong split len: %d' % len(splits_file)) out_path = ("%s__%d" % (splits_file[0], in_file_idx)) # gen formatted files with uncrustify binary proc = Popen([ config["unc_bin"], "-c", cfg_path, "-f", config["in_files"][in_file_idx], "-o", out_path, ]) proc.wait() if proc.returncode != 0: continue # populate 'equal_output_map' map if len(equal_output_map) == 0: equal_output_map[0] = [out_path] map_val_idx += 1 else: found_flag = False for i in range(map_val_idx): # compare first file of group i with the generated file if cmp(equal_output_map[i][0], out_path): equal_output_map[i].append(out_path) found_flag = True break # create new group if files do not match if not found_flag: equal_output_map[map_val_idx] = [out_path] map_val_idx += 1 return equal_output_map
def test(self): if not self.should_test(): return cache = self.scheme + self.scheme_sep + self.bucket + self.sep + str( uuid.uuid4()) ret = main(['config', 'cache.' + self.cache_scheme, 'myrepo']) self.assertEqual(ret, 0) ret = main(['remote', 'add', 'myrepo', cache]) self.assertEqual(ret, 0) remote_name = 'myremote' remote_key = str(uuid.uuid4()) remote = self.scheme + self.scheme_sep + self.bucket + self.sep + remote_key ret = main(['remote', 'add', remote_name, remote]) self.assertEqual(ret, 0) self.dvc = Project('.') foo_key = remote_key + self.sep + self.FOO bar_key = remote_key + self.sep + self.BAR foo_path = self.scheme + self.scheme_sep + self.bucket + self.sep + foo_key bar_path = self.scheme + self.scheme_sep + self.bucket + self.sep + bar_key # Using both plain and remote notation out_foo_path = 'remote://' + remote_name + '/' + self.FOO out_bar_path = bar_path self.write(self.bucket, foo_key, self.FOO_CONTENTS) sleep() import_stage = self.dvc.imp(out_foo_path, 'import') self.assertTrue(os.path.exists('import')) self.assertTrue(filecmp.cmp('import', self.FOO, shallow=False)) import_remote_stage = self.dvc.imp(out_foo_path, out_foo_path + '_imported') cmd_stage = self.dvc.run(outs=[out_bar_path], deps=[out_foo_path], cmd=self.cmd(foo_path, bar_path)) self.write(self.bucket, foo_key, self.BAR_CONTENTS) sleep() self.dvc.status() stages = self.dvc.reproduce(import_stage.path) self.assertEqual(len(stages), 1) self.assertTrue(os.path.exists('import')) self.assertTrue(filecmp.cmp('import', self.BAR, shallow=False)) stages = self.dvc.reproduce(cmd_stage.path) self.assertEqual(len(stages), 1) self.dvc.gc() self.dvc.remove(cmd_stage.path, outs_only=True) self.dvc.checkout(cmd_stage.path)
def test_bulk_put_default_options(): result = invoke([ 'os', 'object', 'bulk-upload', '--namespace', util.NAMESPACE, '--bucket-name', bulk_put_bucket_name, '--src-dir', root_bulk_put_folder ]) # No failures or skips and we uploaded everything parsed_result = parse_json_response_from_mixed_output(result.output) assert parsed_result['skipped-objects'] == [] assert parsed_result['upload-failures'] == {} assert len( parsed_result['uploaded-objects'] ) == get_count_of_files_in_folder_and_subfolders(root_bulk_put_folder) # Pull everything down and verify that the files match (everything in source appears in destination and they are equal) download_folder = 'tests/temp/verify_files_{}'.format(bulk_put_bucket_name) invoke([ 'os', 'object', 'bulk-download', '--namespace', util.NAMESPACE, '--bucket-name', bulk_put_bucket_name, '--download-dir', download_folder ]) object_name_set = set() for dir_name, subdir_list, file_list in os.walk(root_bulk_put_folder): for file in file_list: source_file_path = os.path.join(dir_name, file) downloaded_file_path = source_file_path.replace( root_bulk_put_folder, download_folder) assert os.path.exists(downloaded_file_path) assert filecmp.cmp(source_file_path, downloaded_file_path, shallow=False) # Sanity check that we're reporting back that we uploaded the right files assert get_object_name_from_path( root_bulk_put_folder, source_file_path) in parsed_result['uploaded-objects'] object_name_set.add( get_object_name_from_path(root_bulk_put_folder, source_file_path)) # If we try and put it in the same bucket without --overwrite then everything should be skipped. There should be prompts result = invoke([ 'os', 'object', 'bulk-upload', '--namespace', util.NAMESPACE, '--bucket-name', bulk_put_bucket_name, '--src-dir', root_bulk_put_folder ]) parsed_result = parse_json_response_from_mixed_output(result.output) assert 'Are you sure you want to overwrite it?' in result.output assert set(parsed_result['skipped-objects']) == object_name_set assert parsed_result['upload-failures'] == {} assert parsed_result['uploaded-objects'] == {} # If we say to --no-overwrite then everything should be skipped. There should be no prompts result = invoke([ 'os', 'object', 'bulk-upload', '--namespace', util.NAMESPACE, '--bucket-name', bulk_put_bucket_name, '--src-dir', root_bulk_put_folder, '--no-overwrite' ]) parsed_result = parse_json_response_from_mixed_output(result.output) assert 'Are you sure you want to overwrite it?' not in result.output assert set(parsed_result['skipped-objects']) == object_name_set assert parsed_result['upload-failures'] == {} assert parsed_result['uploaded-objects'] == {} # Now we force it result = invoke([ 'os', 'object', 'bulk-upload', '--namespace', util.NAMESPACE, '--bucket-name', bulk_put_bucket_name, '--src-dir', root_bulk_put_folder, '--overwrite' ]) parsed_result = parse_json_response_from_mixed_output(result.output) assert parsed_result['skipped-objects'] == [] assert parsed_result['upload-failures'] == {} assert len(parsed_result['uploaded-objects']) == len(object_name_set) for object_name in object_name_set: assert object_name in parsed_result['uploaded-objects'] shutil.rmtree(download_folder)
import filecmp import os test_res = os.system( 'python3 frequentwords.py --mainfile b-test1.txt --file2 b-test2.txt > b-test-r.txt' ) if (filecmp.cmp('b-test-r-correct.txt', 'b-test-r.txt')): print("Test: PASSED") else: print("Test: FAILED")
def _perform_mng_test(mng): # test dry run for new asset data_path = os.path.join(test_path, "testdata", "some_data.json") mng.storage_provider.new(data_path, "category-test/some-data", "0.0", dry_run=True) with pytest.raises(Exception): mng.fetch_asset("category-test/some-data") # test updating an inexistant asset data_path = os.path.join(test_path, "testdata", "some_data.json") with pytest.raises(errors.AssetDoesNotExistError): mng.storage_provider.update(data_path, "category-test/some-data", version="0.0") # create the asset mng.storage_provider.new(data_path, "category-test/some-data", "0.0") # check metadata mng.storage_provider.get_asset_meta("category-test/some-data", "0.0") # test dry run for update asset mng.storage_provider.update(data_path, "category-test/some-data", version="0.1", dry_run=True) with pytest.raises(Exception): mng.fetch_asset("category-test/some-data:0.1") # update the asset mng.storage_provider.update(data_path, "category-test/some-data", version="0.1") # check that it is present mng.storage_provider.get_asset_meta("category-test/some-data", "0.1") # rewrite the asset fails with pytest.raises(errors.AssetAlreadyExistsError): mng.storage_provider.update(data_path, "category-test/some-data", version="0.1") # pushing via new fails with pytest.raises(errors.AssetAlreadyExistsError): mng.storage_provider.new(data_path, "category-test/some-data", version="0.0") mng.storage_provider.update(data_path, "category-test/some-data", version="1.0") # # update a major version that does not exist # with pytest.raises(errors.AssetMajorVersionDoesNotExistError): # mng.storage_provider.update(data_path, "category-test/some-data", major="10") # check that it is present mng.storage_provider.get_asset_meta("category-test/some-data", "1.0") # fetch the pinned asset fetched_path = mng.fetch_asset("category-test/some-data:1.0") assert filecmp.cmp(fetched_path, data_path) # fetch the major version asset fetched_path = mng.fetch_asset("category-test/some-data:1") assert filecmp.cmp(fetched_path, data_path) # fetch the latest asset fetched_path = mng.fetch_asset("category-test/some-data") assert filecmp.cmp(fetched_path, data_path) # fetch the latest asset from cache with full info fetched_asset_dict = mng.fetch_asset("category-test/some-data", return_info=True) assert fetched_asset_dict["path"], fetched_path assert fetched_asset_dict["from_cache"] is True assert fetched_asset_dict["version"] == "1.0" assert list(mng.storage_provider.iterate_assets()) == [ ("category-test/some-data", ["1.0", "0.1", "0.0"]) ] # pushing via new works mng.storage_provider.update(data_path, "category-test/some-data", version="1.1") # check that it is present mng.storage_provider.get_asset_meta("category-test/some-data", "1.1") fetched_asset_dict = mng.fetch_asset("category-test/some-data", return_info=True) assert fetched_asset_dict["path"], fetched_path assert fetched_asset_dict["from_cache"] is False assert fetched_asset_dict["version"] == "1.1" assert list(mng.storage_provider.iterate_assets()) == [ ("category-test/some-data", ["1.1", "1.0", "0.1", "0.0"]), ]
def test_bulk_put_get_delete_with_exclusions(object_storage_client): exclusion_test_folder = os.path.join('tests', 'temp', 'os_bulk_upload_exclusion_test') if not os.path.exists(exclusion_test_folder): os.makedirs(exclusion_test_folder) # Make some files for include/exclude folders_to_files = { '': ['test_file1.txt', 'test_file2.png'], 'subfolder': ['blah.pdf', 'hello.txt', 'testfile3.png'], 'subfolder/subfolder2': ['xyz.jpg', 'blag.txt', 'byz.jpg', 'testfile4.png'] } for folder, files in six.iteritems(folders_to_files): folder_path = os.path.join(exclusion_test_folder, folder) if not os.path.exists(folder_path): os.makedirs(folder_path) for file in files: file_path = os.path.join(folder_path, file) with open(file_path, 'w') as f: # For non-text extension types this won't create a valid file, but for testing is probably OK f.write(generate_random_string(CONTENT_STRING_LENGTH)) result = invoke([ 'os', 'object', 'bulk-upload', '--namespace', util.NAMESPACE, '--bucket-name', bulk_put_bucket_name, '--src-dir', exclusion_test_folder, '--object-prefix', 'exclusion_test/', '--exclude', '*.txt', '--exclude', '*.ps1', # Shouldn't match anything '--exclude', 'subfolder/subfolder2/xyz.jpg', '--exclude', 'subfolder/[spqr]lah.pdf' # blah.pdf should still be included because it's not slah.pdf, plah.pdf, qlah.pdf or rlah.pdf ]) parsed_result = parse_json_response_from_mixed_output(result.output) assert parsed_result['skipped-objects'] == [] assert parsed_result['upload-failures'] == {} expected_uploaded_files = [ '{}{}'.format('exclusion_test/', 'test_file2.png'), '{}{}'.format('exclusion_test/', 'subfolder/blah.pdf'), '{}{}'.format('exclusion_test/', 'subfolder/testfile3.png'), '{}{}'.format('exclusion_test/', 'subfolder/subfolder2/byz.jpg'), '{}{}'.format('exclusion_test/', 'subfolder/subfolder2/testfile4.png') ] # Check that we uploaded what we said we did assert len( parsed_result['uploaded-objects']) == len(expected_uploaded_files) for f in expected_uploaded_files: assert f in parsed_result['uploaded-objects'] download_folder_base = os.path.join( 'tests', 'temp', 'verify_os_bulk_upload_exclusion_test') verify_downloaded_folders_for_inclusion_exclusion_tests( expected_uploaded_files=expected_uploaded_files, source_folder=exclusion_test_folder, download_folder=download_folder_base, download_prefix_no_slash='exclusion_test') # Download objects with exclusions to make sure that works target_download_folder = os.path.join(download_folder_base, 'get_with_exclude') invoke([ 'os', 'object', 'bulk-download', '--namespace', util.NAMESPACE, '--bucket-name', bulk_put_bucket_name, '--download-dir', target_download_folder, '--prefix', 'exclusion_test/', '--exclude', '*.jpg', '--exclude', 'subfolder/subfolder2/*.png', '--exclude', 'subfolder/blah.pdf', ]) assert not os.path.exists( os.path.join(target_download_folder, 'exclusion_test', 'subfolder', 'blah.pdf')) assert not os.path.exists( os.path.join(target_download_folder, 'exclusion_test', 'subfolder', 'subfolder2', 'byz.jpg')) assert not os.path.exists( os.path.join(target_download_folder, 'exclusion_test', 'subfolder', 'subfolder2', 'testfile4.png')) assert get_count_of_files_in_folder_and_subfolders( target_download_folder) == 2 assert os.path.exists( os.path.join(target_download_folder, 'exclusion_test', 'test_file2.png')) assert os.path.exists( os.path.join(target_download_folder, 'exclusion_test', 'subfolder', 'testfile3.png')) assert filecmp.cmp( os.path.join(exclusion_test_folder, 'test_file2.png'), os.path.join(target_download_folder, 'exclusion_test', 'test_file2.png')) assert filecmp.cmp( os.path.join(exclusion_test_folder, 'subfolder', 'testfile3.png'), os.path.join(target_download_folder, 'exclusion_test', 'subfolder', 'testfile3.png')) # Delete objects with exclusions result = invoke([ 'os', 'object', 'bulk-delete', '--namespace', util.NAMESPACE, '--bucket-name', bulk_put_bucket_name, '--prefix', 'exclusion_test/', '--exclude', '*.jpg', '--exclude', 'subfolder/blah.pdf', '--dry-run' ]) parsed_dry_run_result = parse_json_response_from_mixed_output( result.output) assert len(parsed_dry_run_result['deleted-objects']) == 3 result = invoke([ 'os', 'object', 'bulk-delete', '--namespace', util.NAMESPACE, '--bucket-name', bulk_put_bucket_name, '--prefix', 'exclusion_test/', '--exclude', '*.jpg', '--exclude', 'subfolder/blah.pdf', '--force' ]) parsed_result = parse_json_response_from_mixed_output(result.output) assert parsed_result['delete-failures'] == {} assert set(parsed_result['deleted-objects']) == set( parsed_dry_run_result['deleted-objects']) list_objects_responses = oci_cli.objectstorage_cli_extended.retrying_list_objects( client=object_storage_client, request_id=None, namespace=util.NAMESPACE, bucket_name=bulk_put_bucket_name, prefix='exclusion_test/', start=None, end=None, limit=1000, delimiter=None, fields='name', retrieve_all=True) remaining_objects = [] for response in list_objects_responses: remaining_objects.extend( map(lambda obj: obj.name, response.data.objects)) assert len(remaining_objects) == 2 assert '{}{}'.format('exclusion_test/', 'subfolder/blah.pdf') in remaining_objects assert '{}{}'.format('exclusion_test/', 'subfolder/subfolder2/byz.jpg') in remaining_objects shutil.rmtree(target_download_folder) shutil.rmtree(exclusion_test_folder)
def run(self, server): """Execute the test assuming it's a python program. If the test aborts, print its output to stdout, and raise an exception. Else, comprare result and reject files. If there is a difference, print it to stdout and raise an exception. The exception is raised only if is_force flag is not set.""" diagnostics = "unknown" save_stdout = sys.stdout try: self.skip = 0 if os.path.exists(self.skip_cond): sys.stdout = FilteredStream(self.tmp_result) stdout_fileno = sys.stdout.stream.fileno() execfile(self.skip_cond, dict(locals(), **server.__dict__)) sys.stdout.close() sys.stdout = save_stdout if not self.skip: sys.stdout = FilteredStream(self.tmp_result) stdout_fileno = sys.stdout.stream.fileno() self.execute(server) self.is_executed_ok = True except Exception as e: traceback.print_exc(e) diagnostics = str(e) finally: if sys.stdout and sys.stdout != save_stdout: sys.stdout.close() sys.stdout = save_stdout; self.is_executed = True if not self.skip: if self.is_executed_ok and os.path.isfile(self.result): self.is_equal_result = filecmp.cmp(self.result, self.tmp_result) else: self.is_equal_result = 1 if self.args.valgrind: self.is_valgrind_clean = \ check_valgrind_log(server.valgrind_log) == False if self.skip: print "[ skip ]" if os.path.exists(self.tmp_result): os.remove(self.tmp_result) elif self.is_executed_ok and self.is_equal_result and self.is_valgrind_clean: print "[ pass ]" if os.path.exists(self.tmp_result): os.remove(self.tmp_result) elif (self.is_executed_ok and not self.is_equal_result and not os.path.isfile(self.result)): os.rename(self.tmp_result, self.result) print "[ NEW ]" else: os.rename(self.tmp_result, self.reject) print "[ fail ]" where = "" if not self.is_executed_ok: self.print_diagnostics(self.reject, "Test failed! Last 10 lines of the result file:") where = ": test execution aborted, reason '{0}'".format(diagnostics) elif not self.is_equal_result: self.print_unidiff() where = ": wrong test output" elif not self.is_valgrind_clean: os.remove(self.reject) self.print_diagnostics(server.valgrind_log, "Test failed! Last 10 lines of valgrind.log:") where = ": there were warnings in valgrind.log" if not self.args.is_force: raise RuntimeError("Failed to run test " + self.name + where)
def _ValidateShardMaps(args): """Validate that the shard maps, csv files, etc. are consistent.""" del args errors = [] tempdir = tempfile.mkdtemp() try: builders = _GetBuilderPlatforms(builders=None, waterfall='all') for builder in builders: output_file = os.path.join( tempdir, os.path.basename(builder.timing_file_path)) _FilterTimingData(builder, output_file) if not filecmp.cmp(builder.timing_file_path, output_file): errors.append( '{timing_data} is not up to date. Please run ' '`./generate_perf_sharding.py update-timing --filter-only` ' 'to regenerate it.'.format( timing_data=builder.timing_file_path)) finally: shutil.rmtree(tempdir) # Check that bot_platforms.py matches the actual shard maps for platform in bot_platforms.ALL_PLATFORMS: platform_benchmark_names = set( b.name for b in platform.benchmark_configs) | set( e.name for e in platform.executables) shard_map_benchmark_names = _ParseBenchmarks( platform.shards_map_file_path) for benchmark in platform_benchmark_names - shard_map_benchmark_names: errors.append( 'Benchmark {benchmark} is supposed to be scheduled on platform ' '{platform} according to ' 'bot_platforms.py, but it is not yet scheduled. If this is a new ' 'benchmark, please rename it to UNSCHEDULED_{benchmark}, and then ' 'contact ' 'Telemetry and Chrome Client Infra team to schedule the benchmark. ' 'You can email chrome-benchmarking-request@ to get started.'. format(benchmark=benchmark, platform=platform.name)) for benchmark in shard_map_benchmark_names - platform_benchmark_names: errors.append( 'Benchmark {benchmark} is scheduled on shard map {path}, but ' 'bot_platforms.py ' 'says that it should not be on that shard map. This could be because ' 'the benchmark was deleted. If that is the case, you can use ' '`generate_perf_sharding deschedule` to deschedule the benchmark ' 'from the shard map.'.format( benchmark=benchmark, path=platform.shards_map_file_path)) # Check that every official benchmark is scheduled on some shard map. # TODO(crbug.com/963614): Note that this check can be deleted if we # find some way other than naming the benchmark with prefix "UNSCHEDULED_" # to make it clear that a benchmark is not running. scheduled_benchmarks = set() for platform in bot_platforms.ALL_PLATFORMS: scheduled_benchmarks = scheduled_benchmarks | _ParseBenchmarks( platform.shards_map_file_path) for benchmark in (bot_platforms.OFFICIAL_BENCHMARK_NAMES - scheduled_benchmarks): errors.append( 'Benchmark {benchmark} is an official benchmark, but it is not ' 'scheduled to run anywhere. please rename it to ' 'UNSCHEDULED_{benchmark}'.format(benchmark=benchmark)) for error in errors: print('*', textwrap.fill(error, 70), '\n', file=sys.stderr) if errors: return 1 return 0
def _main(): out_dir = args.out_dir if not os.path.exists(out_dir): os.mkdir(out_dir) log_file = args.log_file if args.log_file is not None else os.path.join( out_dir, "image_organization.log") logger = create_logger(log_file, "image_organizer") logger.info("started new session") print("Listing subtree...") all_files = list_subtree(args.source_dir, recursive=args.recursive) media_files = [] for f in tqdm(all_files, desc="Filtering non-media files"): try: if is_media(f): media_files.append(f) except OSError: logger.warning(f"OS error while checking if '{f}' is a media file") for src_file in tqdm(media_files, desc="Organizing Media"): try: time_taken = get_media_time(src_file, args.valid_mod_time) except ValueError: logger.warning(f"failed to get time from '{src_file}'") continue dst_dir = os.path.join(out_dir, f"{time_taken.year:04}_{time_taken.month:02}") if not os.path.exists(dst_dir): os.mkdir(dst_dir) dst_filename = os.path.basename(src_file) dst_file = os.path.join(dst_dir, dst_filename) if os.path.exists(dst_file): if filecmp.cmp(src_file, dst_file): if not args.copy: if args.dry_run: logger.info( f"Would remove '{src_file}' - duplicate of '{dst_file}'" ) else: os.remove(src_file) logger.info( f"Remove '{src_file}' - duplicate of '{dst_file}'") else: logger.info( f"Ignoring '{src_file}' - duplicate of '{dst_file}'") else: logger.warning( f"failed to handle '{src_file}' - destination file '{dst_file}' already exists" ) continue if args.dry_run: if args.copy: logger.info(f"Would copy '{src_file}' to '{dst_file}'") else: logger.info(f"Would move '{src_file}' to '{dst_file}'") else: try: if args.copy: logger.info(f"Copy '{src_file}' to '{dst_file}'") shutil.copy2(src_file, dst_file) else: logger.info(f"Move '{src_file}' to '{dst_file}'") shutil.move(src_file, dst_file) except: logger.error(f"Error copying/moving {src_file} -> {dst_file}") if not args.copy and not args.dry_run: delete_empty_dirs(args.source_dir, args.recursive)
not isSingleFolderUpdate) #Get res directory targetResDirectory = helper.GetResDirectory(options.path) #Copy the files if not len(matchingFiles): print "No matching files found" for k, v in matchingFiles.iteritems(): targetStringsXml = helper.GetTargetStringsXml(targetResDirectory, k) if not os.path.exists(os.path.dirname(targetStringsXml)): os.makedirs(os.path.dirname(targetStringsXml)) try: if not os.path.exists(targetStringsXml) or not filecmp.cmp( matchingFiles[k], targetStringsXml): print "Replacing", targetStringsXml shutil.copy(matchingFiles[k], targetStringsXml) else: print "Skipping", targetStringsXml, ", it is already up to date" except: print "Error. No Crowdin file for ", targetStringsXml print "Deleting", extractDir shutil.rmtree(extractDir) print "Deleting", zipPath[0] os.remove(zipPath[0])