def run_server_in_temp_folder(files=[], port=8999): """ Set up a server with some known files to run captures against. Example: with run_server_in_temp_folder(['test/assets/test.html','test/assets/test.pdf']): assert(requests.get("http://localhost/test.html") == contents_of_file("test.html")) """ # Run in temp dir. # We have to (implicitly) cwd to this so SimpleHTTPRequestHandler serves the files for us. old_cwd = os.getcwd() with tempdir.in_tempdir(): # Copy over files to current temp dir, stripping paths. for file in files: shutil.copyfile(os.path.join(old_cwd, file), os.path.basename(file)) # start server httpd = HTTPServer(('', port), SimpleHTTPRequestHandler) httpd._BaseServer__is_shut_down = multiprocessing.Event() server_thread = Process(target=httpd.serve_forever) server_thread.start() try: # run body of `with` block yield finally: # shut down server server_thread.terminate()
def test_hello_world(self): with tempdir.in_tempdir(): with tempfile.NamedTemporaryFile() as fasta: with tempfile.NamedTemporaryFile() as tax: fasta.write(Tests.extra_mcra_fasta) fasta.flush() tax.write(Tests.extra_mcra_taxonomy) tax.flush() prev_path = os.path.join(path_to_data,'mcrA.10seqs.gpkg') cmd1 = "%s update --graftm_package %s --sequences %s --taxonomy %s --output %s" %( path_to_script, prev_path, fasta.name, tax.name, 'updated.gpkg') extern.run(cmd1) prev = GraftMPackage.acquire(prev_path) up = GraftMPackage.acquire('updated.gpkg') prevhash = prev.taxonomy_hash() taxhash = up.taxonomy_hash() self.assertEqual(len(prevhash)+1, len(taxhash)) self.assertEqual(['mcrA','Euryarchaeota_mcrA','Methanofastidiosa'], taxhash['KYC55281.1']) self.assertEqual(prevhash['638165755'], taxhash['638165755']) seqio = SequenceIO() self.assertEqual( len(seqio.read_fasta_file(prev.unaligned_sequence_database_path()))+1, len(seqio.read_fasta_file(up.unaligned_sequence_database_path())))
def test_autodecorate(self): with tempdir.in_tempdir(): with tempfile.NamedTemporaryFile() as fasta: fasta.write(Tests.extra_mcra_fasta) fasta.flush() prev_path = os.path.join(path_to_data,'mcrA.10seqs.gpkg') update = Update(prerequisites) update.update( input_sequence_path = fasta.name, input_graftm_package_path = prev_path, output_graftm_package_path = 'updated.gpkg') prev = GraftMPackage.acquire(prev_path) up = GraftMPackage.acquire('updated.gpkg') prevhash = prev.taxonomy_hash() taxhash = up.taxonomy_hash() self.assertEqual(11, len(taxhash)) #hard-code 11 because of #https://github.com/geronimp/graftM/issues/204 self.assertEqual(['mcrA','Euryarchaeota_mcrA', 'Methanomicrobia'], taxhash['KYC55281.1']) self.assertEqual(prevhash['638165755'], taxhash['638165755']) seqio = SequenceIO() self.assertEqual( len(seqio.read_fasta_file(prev.unaligned_sequence_database_path()))+1, len(seqio.read_fasta_file(up.unaligned_sequence_database_path())))
def test_get_writer_container(self): if not self.ref.container_format: return with tempdir.in_tempdir(): self.outfile = 'test-writer' + self.ref.ext with self.reader.Writer(self.outfile) as W: assert_equal(isinstance(W, self.ref.writer), True) assert_equal(W.n_atoms, self.reader.n_atoms)
def test_xtc_striding(): """testing MDAnalysis.analysis.helanal xtc striding: Check for resolution of Issue #188.""" u = MDAnalysis.Universe(GRO, XTC) u.trajectory[1] with tempdir.in_tempdir(): assert_raises(FinishTimeException, MDAnalysis.analysis.helanal.helanal_trajectory, u, selection="name CA", finish=5)
def test_create_protein_pkg(self): with tempdir.in_tempdir(): PackageCreator().create(input_graftm_package=os.path.join( path_to_data, '4.11.22seqs.gpkg.spkg', '4.11.22seqs'), output_singlem_package='protein.spkg', hmm_position=76, force=False) self.assertTrue(os.path.isdir('protein.spkg')) j = json.load(open('protein.spkg/CONTENTS.json')) self.assertEqual(76, j['singlem_hmm_position'])
def test_create_nuc_pkg(self): with tempdir.in_tempdir(): PackageCreator().create(input_graftm_package=os.path.join( path_to_data, '61_otus.v3.gpkg.spkg', '61_otus.v3'), output_singlem_package='nuc.spkg', hmm_position=888, force=False) self.assertTrue(os.path.isdir('nuc.spkg')) j = json.load(open('nuc.spkg/CONTENTS.json')) self.assertEqual(888, j['singlem_hmm_position'])
def test_get_writer(self): with tempdir.in_tempdir(): self.outfile = 'test-trz-writer.trz' W = self.trz.Writer(self.outfile) assert_equal(isinstance(W, mda.coordinates.TRZ.TRZWriter), True) assert_equal(W.n_atoms, self.trz.n_atoms) try: os.unlink(self.outfile) except OSError: pass
def run_calc(*funcargs, basedir='.'): ''' :param funcargs: of tuples of function, **kwargs ''' results = list() with in_tempdir(basedir=basedir): for f, a in funcargs: if isinstance(a, dict): results.append(f(**a)) else: results.append(f(*a)) return results
def test_create_protein_pkg(self): with tempdir.in_tempdir(): PackageCreator().create( input_graftm_package = os.path.join( path_to_data, '4.11.22seqs.gpkg.spkg', '4.11.22seqs'), output_singlem_package = 'protein.spkg', hmm_position = 76, window_size = 63, force = False) self.assertTrue(os.path.isdir('protein.spkg')) j = json.load(open('protein.spkg/CONTENTS.json')) self.assertEqual(76, j['singlem_hmm_position']) self.assertEqual(63, j['singlem_window_size'])
def test_create_nuc_pkg(self): with tempdir.in_tempdir(): PackageCreator().create( input_graftm_package = os.path.join( path_to_data, '61_otus.v3.gpkg.spkg', '61_otus.v3'), output_singlem_package = 'nuc.spkg', hmm_position = 888, window_size = 57, force = False) self.assertTrue(os.path.isdir('nuc.spkg')) j = json.load(open('nuc.spkg/CONTENTS.json')) self.assertEqual(888, j['singlem_hmm_position']) self.assertEqual(57, j['singlem_window_size'])
def test_writer_trz_from_other(self): with tempdir.in_tempdir(): outfile = 'trz-writer-2.trz' W = mda.coordinates.TRZ.TRZWriter(outfile, n_atoms=len(self.u.atoms)) W.write(self.u.trajectory.ts) W.close() u2 = mda.Universe(two_water_gro, outfile) assert_array_almost_equal(self.u.atoms.positions, u2.atoms.positions, 3)
def test_helanal_trajectory(reference=HELANAL_BENDING_MATRIX, outfile="helanal_bending_matrix.dat"): u = mda.Universe(PSF, DCD) with tempdir.in_tempdir(): # Helix 8: 161 - 187 http://www.rcsb.org/pdb/explore.do?structureId=4AKE MDAnalysis.analysis.helanal.helanal_trajectory(u, selection="name CA and resnum 161-187") bendingmatrix = read_bending_matrix(outfile) ref = read_bending_matrix(reference) assert_equal(sorted(bendingmatrix.keys()), sorted(ref.keys()), err_msg="different contents in bending matrix data file") for label in ref.keys(): assert_array_almost_equal(bendingmatrix[label], ref[label], err_msg="bending matrix stats for {0} mismatch".format(label))
def test_01(self): with tempdir.in_tempdir() as td: os.mkdir('src') os.mkdir('dst') with open(os.path.join('src', 'Moby Dick.txt'), 'wb') as w: w.write(TEST_FILES['Moby Dick.txt']) with open(os.path.join('src', 'Great Expectations.txt'), 'wb') as w: w.write(TEST_FILES['Great Expectations.txt']) pycopy.pycopy('src', 'dst') self.assertTrue(os.path.exists(os.path.join('dst', 'Moby Dick.txt' ))) self.assertTrue(os.path.exists(os.path.join('dst', 'Great Expectations.txt')))
def test_01(self): with tempdir.in_tempdir() as td: os.mkdir('src') os.mkdir('dst') with open(os.path.join('src', 'Moby Dick.txt'), 'wb') as w: w.write(TEST_FILES['Moby Dick.txt']) with open(os.path.join('src', 'Great Expectations.txt'), 'wb') as w: w.write(TEST_FILES['Great Expectations.txt']) pycopy.pycopy('src', 'dst') self.assertTrue( os.path.exists(os.path.join('dst', 'Moby Dick.txt'))) self.assertTrue( os.path.exists(os.path.join('dst', 'Great Expectations.txt')))
def test_conect_bonds_conect(self): conect = self.conect assert_equal(len(conect.atoms), 1890) assert_equal(len(conect.bonds), 1922) with tempdir.in_tempdir(): try: outfile = 'test-pdb-hbonds.pdb' self.conect.atoms.write(outfile, bonds="conect") u1 = mda.Universe(outfile, guess_bonds=True) finally: os.unlink(outfile) assert_equal(len(u1.atoms), 1890) assert_equal(len(u1.bonds), 1922)
def test_conect_bonds_all(self): conect = self.conect assert_equal(len(conect.atoms), 1890) assert_equal(len(conect.bonds), 1922) with tempdir.in_tempdir(): try: outfile = 'pdb-connect-bonds.pdb' self.conect.atoms.write(outfile, bonds="all") u2 = mda.Universe(outfile, guess_bonds=True) finally: os.unlink(outfile) assert_equal(len(u2.atoms), 1890) assert_equal(len([b for b in u2.bonds if not b.is_guessed]), 1922)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--config-file', default='backup.yml') parser.add_argument('--output-dir', default=None) parser.add_argument('--upload', default=False, action='store_true') parser.add_argument('--quiet', default=False, action='store_true') parser.add_argument('--debug', default=False, action='store_true') opts = parser.parse_args() if not os.path.exists(opts.config_file): sys.exit("The specified config file does not exist!") if not opts.quiet: if opts.debug: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.INFO) else: logging.basicConfig(level=logging.ERROR) config = None if opts.output_dir: opts.output_dir = os.path.abspath(opts.output_dir) with open(opts.config_file) as fp: logging.debug("Loading config from {0}".format(opts.config_file)) config = yaml.safe_load(fp) config['quiet'] = opts.quiet with tempdir.in_tempdir() as dir_path: logging.debug("Chdir to {0}".format(dir_path)) backup_folder = datetime.datetime.utcnow().strftime( config['output_prefix']) logging.info("Creating backup folder: {0}".format(backup_folder)) os.mkdir(backup_folder, 0o700) os.chdir(backup_folder) _backup_db(config) _backup_files(config) logging.debug("Chdir to {0}".format(dir_path)) os.chdir(dir_path) archive_name = _create_archive(backup_folder, config) enc_archive_name = _encrypt_file(archive_name, config) if opts.upload: _upload_file(enc_archive_name, config) if opts.output_dir: shutil.move(enc_archive_name, opts.output_dir)
def check_density_from_Universe(self, atomselection, ref_meandensity, **kwargs): import MDAnalysis.analysis.density with tempdir.in_tempdir(): D = MDAnalysis.analysis.density.density_from_Universe( self.universe, atomselection=atomselection, delta=self.delta, **kwargs) assert_almost_equal(D.grid.mean(), ref_meandensity, err_msg="mean density does not match") D.export(self.outfile) D2 = MDAnalysis.analysis.density.Density(self.outfile) assert_almost_equal(D.grid, D2.grid, decimal=self.precision, err_msg="DX export failed: different grid sizes")
def test_File_write(self): with tempdir.in_tempdir(): outfile = "lookingglas.txt" try: obj = open(outfile, "w") ns = util.NamedStream(obj, outfile, close=True) ns.writelines(self.text) ns.close() text = open(outfile).readlines() assert_equal(ns.name, outfile) assert_equal(str(ns), outfile) assert_equal(len(text), len(self.text)) assert_equal("".join(text), "".join(self.text)) finally: ns.close() obj.close()
def test_helanal_trajectory(reference=HELANAL_BENDING_MATRIX, outfile="helanal_bending_matrix.dat"): u = mda.Universe(PSF, DCD) with tempdir.in_tempdir(): # Helix 8: 161 - 187 http://www.rcsb.org/pdb/explore.do?structureId=4AKE MDAnalysis.analysis.helanal.helanal_trajectory( u, selection="name CA and resnum 161-187") bendingmatrix = read_bending_matrix(outfile) ref = read_bending_matrix(reference) assert_equal(sorted(bendingmatrix.keys()), sorted(ref.keys()), err_msg="different contents in bending matrix data file") for label in ref.keys(): assert_array_almost_equal( bendingmatrix[label], ref[label], err_msg="bending matrix stats for {0} mismatch".format(label))
def extract(self, archive_path, output_package_path, **kwargs): '''Extract an archived GraftM package. Parameters ---------- archive_path: str path to archive output_package_path: str path to where to put the extracted file kwargs: force: bool overwrite an existing directory ''' force = kwargs.pop('force', ArchiveDefaultOptions.force) if len(kwargs) > 0: raise Exception("Unexpected arguments detected: %s" % kwargs) logging.info("Un-archiving GraftM package '%s' from '%s'" % (output_package_path, archive_path)) archive = os.path.abspath(archive_path) output = os.path.abspath(output_package_path) self._setup_output(output, force) with tempdir.in_tempdir(): with tarfile.open(archive) as tar: tar.extractall() for tarinfo in tar: folder = os.path.dirname(tarinfo.name) break # recreate diamond file gpkg = GraftMPackage.acquire(folder) if gpkg.version != 3: raise Exception( "Encountered an archived gpkg of unexpected version: %d" % gpkg.version) if gpkg.diamond_database_path(): logging.debug("Creating diamond DB") gpkg.create_diamond_db() shutil.move(folder, output) logging.info("Archive successfully extracted")
def test_hole_module_fd_closure(self): """MDAnalysis.analysis.hole: Issue 129: ensure low level file descriptors to PDB files used by Hole program are properly closed""" # If Issue 129 isn't resolved, this function will produce an OSError on # the system, and cause many other tests to fail as well. # # Successful test takes ~10 s, failure ~2 s. try: # Hasten failure by setting "ulimit -n 64" (can't go too low because of open modules etc...) import resource resource.setrlimit(resource.RLIMIT_NOFILE, (64, self.hard_max_open_files)) except ImportError: raise NotImplementedError( "Test cannot be run without the resource module.") with tempdir.in_tempdir(): try: # will need to have the 'hole' command available in the path H = HOLEtraj(self.universe, cvect=[0, 1, 0], sample=20.0) except OSError as err: if err.errno == errno.ENOENT: raise OSError(errno.ENOENT, "HOLE binary not found") raise finally: self._restore_rlimits() # pretty unlikely that the code will get through 2 rounds if the MDA # issue 129 isn't fixed, although this depends on the file descriptor # open limit for the machine in question try: for i in xrange(2): # will typically get an OSError for too many files being open after # about 2 seconds if issue 129 isn't resolved H.run() except OSError as err: if err.errno == errno.EMFILE: raise AssertionError( "HOLEtraj does not close file descriptors (Issue 129)") elif err.errno == errno.ENOENT: raise OSError(errno.ENOENT, "HOLE binary not found") raise finally: # make sure to restore open file limit !! self._restore_rlimits()
def test_hole_module_fd_closure(self): """MDAnalysis.analysis.hole: Issue 129: ensure low level file descriptors to PDB files used by Hole program are properly closed""" # If Issue 129 isn't resolved, this function will produce an OSError on # the system, and cause many other tests to fail as well. # # Successful test takes ~10 s, failure ~2 s. try: # Hasten failure by setting "ulimit -n 64" (can't go too low because of open modules etc...) import resource resource.setrlimit(resource.RLIMIT_NOFILE, (64, self.hard_max_open_files)) except ImportError: raise NotImplementedError("Test cannot be run without the resource module.") with tempdir.in_tempdir(): try: # will need to have the 'hole' command available in the path H = HOLEtraj(self.universe, cvect=[0, 1, 0], sample=20.0) except OSError as err: if err.errno == errno.ENOENT: raise OSError(errno.ENOENT, "HOLE binary not found") raise finally: self._restore_rlimits() # pretty unlikely that the code will get through 2 rounds if the MDA # issue 129 isn't fixed, although this depends on the file descriptor # open limit for the machine in question try: for i in xrange(2): # will typically get an OSError for too many files being open after # about 2 seconds if issue 129 isn't resolved H.run() except OSError as err: if err.errno == errno.EMFILE: raise AssertionError("HOLEtraj does not close file descriptors (Issue 129)") elif err.errno == errno.ENOENT: raise OSError(errno.ENOENT, "HOLE binary not found") raise finally: # make sure to restore open file limit !! self._restore_rlimits()
def extract(self, archive_path, output_package_path, **kwargs): '''Extract an archived GraftM package. Parameters ---------- archive_path: str path to archive output_package_path: str path to where to put the extracted file kwargs: force: bool overwrite an existing directory ''' force = kwargs.pop('force', ArchiveDefaultOptions.force) if len(kwargs) > 0: raise Exception("Unexpected arguments detected: %s" % kwargs) logging.info("Un-archiving GraftM package '%s' from '%s'" % (output_package_path, archive_path)) archive = os.path.abspath(archive_path) output = os.path.abspath(output_package_path) self._setup_output(output, force) with tempdir.in_tempdir(): with tarfile.open(archive) as tar: tar.extractall() for tarinfo in tar: folder = os.path.dirname(tarinfo.name) break # recreate diamond file gpkg = GraftMPackage.acquire(folder) if gpkg.version != 3: raise Exception("Encountered an archived gpkg of unexpected version: %d" % gpkg.version) if gpkg.diamond_database_path(): logging.debug("Creating diamond DB") gpkg.create_diamond_db() shutil.move(folder, output) logging.info("Archive successfully extracted")
def test_round_trip(self): with tempdir.in_tempdir(): archiver = Archive() ingpkg = os.path.join(path_to_data, 'mcrA.gpkg') arc = 'compressed.gpkg.tar.gz' out = 'reconstituted.gpkg' archiver.create(ingpkg, arc) archiver.extract(arc, out) file_list = ['CONTENTS.json', 'mcrA.faa', 'mcrA.hmm'] dmnd = 'mcra.faa.dmnd' refpkg = 'mcrA.refpkg' for f in file_list: self.assertTrue( filecmp.cmp(os.path.join(ingpkg, f), os.path.join(out, f))) self.assertTrue( filecmp.dircmp(os.path.join(ingpkg, refpkg), os.path.join(out, refpkg))) # Do not test actual equality of diamond DB because this changes # dependant on diamond version. self.assertTrue(os.path.exists(os.path.join(out, dmnd))) self.assertEquals(sorted(file_list + [dmnd, refpkg]), sorted(os.listdir(out)))
def test_round_trip(self): with tempdir.in_tempdir(): archiver = Archive() ingpkg = os.path.join(path_to_data, 'mcrA.gpkg') arc = 'compressed.gpkg.tar.gz' out = 'reconstituted.gpkg' archiver.create(ingpkg, arc) archiver.extract(arc, out) file_list = ['CONTENTS.json', 'mcrA.faa', 'mcrA.hmm'] dmnd = 'mcra.faa.dmnd' refpkg = 'mcrA.refpkg' for f in file_list: self.assertTrue(filecmp.cmp( os.path.join(ingpkg, f), os.path.join(out, f))) self.assertTrue(filecmp.dircmp( os.path.join(ingpkg, refpkg), os.path.join(out, refpkg))) # Do not test actual equality of diamond DB because this changes # dependant on diamond version. self.assertTrue(os.path.exists(os.path.join(out, dmnd))) self.assertEquals( sorted(file_list+[dmnd, refpkg]), sorted(os.listdir(out)))
def in_tmpexport(commit): with in_tempdir() as tmpdir: commit.export_to(tmpdir) yield tmpdir
def test_no_container(self): with tempdir.in_tempdir(): if self.ref.container_format: self.ref.writer('foo') else: assert_raises(TypeError, self.ref.writer, 'foo')
def test_get_writer(self): with tempdir.in_tempdir(): self.outfile = 'test-writer' + self.ref.ext with self.reader.Writer(self.outfile, n_atoms=100) as W: assert_equal(isinstance(W, self.ref.writer), True) assert_equal(W.n_atoms, 100)
def test_startframe(self): """test_startframe: TestContactAnalysis1: start frame set to 0 (resolution of Issue #624)""" with tempdir.in_tempdir(): CA1 = self._run_ContactAnalysis1() self.assertEqual(CA1.timeseries.shape[1], self.universe.trajectory.n_frames)
def test_end_zero(self): """test_end_zero: TestContactAnalysis1: stop frame 0 is not ignored""" with tempdir.in_tempdir(): CA1 = self._run_ContactAnalysis1(stop=0) self.assertEqual(len(CA1.timeseries), 0)
def test_slicing(self): start, stop, step = 10, 30, 5 with tempdir.in_tempdir(): CA1 = self._run_ContactAnalysis1(start=start, stop=stop, step=step) frames = np.arange(self.universe.trajectory.n_frames)[start:stop:step] self.assertEqual(CA1.timeseries.shape[1], len(frames))