def test_one_chrom(self): logging.info("test_one_chrom") output_file = self.file_name("one_chrom") storage = LocalCache("local_cache/one_chrom") test_storage = storage.join('test_snps') test_storage.rmtree('') test_snps3 = self.bed[:, self.bed.pos[:, 0] == 3] # Test only on chromosome 3 test_snps3_dist = DistributedBed.write(test_storage, test_snps3, piece_per_chrom_count=2) for test_snps, ref, clear_cache, name in ( (test_snps3, "old_one", True, "Run with just chrom3"), (test_snps3_dist, "old_one", True, "Run with distributed test SNPs"), (test_snps3, "old_one", False, "Run with just chrom3 (use cache)"), (test_snps3_dist, "old_one", False, "Run with distributed test SNPs (use cache)"), ): logging.info("=========== " + name + " ===========") results_df = single_snp_scale( test_snps=test_snps, pheno=self.phen_fn, covar=self.cov_fn, K0=self.bed, cache=self._cache_dict(storage, clear_cache=clear_cache), output_file_name=output_file, ) self.compare_files(results_df, ref)
def test_local_distribute(self): logging.info("test_local_distribute") force_python_only = False output_file = self.file_name("local_distribute") storage = LocalCache("local_cache/local_distribute") test_storage = storage.join('test_snps') test_storage.rmtree('') test_snps = DistributedBed.write(test_storage, self.bed, piece_per_chrom_count=2) results_df = single_snp_scale(test_snps=test_snps, pheno=self.phen_fn, covar=self.cov_fn, G0=self.bed, cache=self._cache_dict(storage, clear_cache=True), output_file_name=output_file, force_python_only=force_python_only) self.compare_files(results_df, "old") results_df = single_snp_scale(test_snps=self.bed, pheno=self.phen_fn, covar=self.cov_fn, G0=self.bed, cache=self._cache_dict( storage, clear_cache=False), output_file_name=output_file) self.compare_files(results_df, "old")
def scan_local(local_directory, url=None, logging_level=logging.WARNING): ''' Bootstrap a Hashdown by recursively walking a local directory and finding the local MD5 hashes. (A local hash might be wrong if the files are out of date or have OS-dependent line endings.) Typically, you'll then want to save the result to a JSON file and then edit that JSON file manually to remove uninteresting files. :param local_directory: Local directory to recursively walk :type path: string :param url: URL to give to the Hashdown. (It will not be checked.) :type path: string :param logging_level: Logging level for printing progress of the walk. Default is logging.WARNING) :rtype: :class:`.Hashdown` ''' from pysnptools.util.filecache import LocalCache file_to_hash = {} localcache = LocalCache(local_directory) with log_in_place("scanning", logging_level) as updater: for file in localcache.walk(): updater(file) with localcache.open_read(file) as full_file: hash = Hashdown._get_hash(full_file) file_to_hash[file] = hash return Hashdown(url, file_to_hash=file_to_hash)
def test_low(self): logging.info("test_low") output_file = self.file_name("low") storage = LocalCache("local_cache/low") for clear_cache in (True, False): if clear_cache: storage.rmtree() results_df = single_snp_scale(test_snps=self.bed, pheno=self.phen_fn, covar=self.cov_fn, cache=storage, output_file_name=output_file) self.compare_files(results_df, "old")
def _fixup(cache_value,default_subfolder='filecache'): if isinstance(cache_value,FileCache): return cache_value if cache_value is None: dirpath = tempfile.mkdtemp()+'/'+default_subfolder return FileCache._fixup(dirpath) if isinstance(cache_value,str): from pysnptools.util.filecache import LocalCache return LocalCache(cache_value) raise Exception("Do not know how to make a FileCache from '{0}'".format(cache_value))
def test_one_fast(self): logging.info("test_one_fast") output_file = self.file_name("one_fast") storage = LocalCache("local_cache") test_storage = storage.join('one_fast') test_storage.rmtree() test_snps3 = self.bed[:, self.bed.pos[:, 0] == 3] # Test only on chromosome 3 test_snps3_dist = DistributedBed.write(test_storage, test_snps3, piece_per_chrom_count=2) results_df = single_snp_scale(test_snps=test_snps3_dist, pheno=self.phen_fn, covar=self.cov_fn, G0=self.bed, output_file_name=output_file) self.compare_files(results_df, "old_one")
def __init__(self, common_directory, id_and_path_function, leave_space=0): from pysnptools.util.filecache import LocalCache super(PeerToPeer, self).__init__() shared_directory_str = str(common_directory) if isinstance(common_directory, str): common_directory = LocalCache(common_directory) self.common_directory = common_directory self.id_and_path_function = id_and_path_function self.leave_space = leave_space assert not self.common_directory._simple_file_exists( "main.txt" ), "A common_directory cannot exist where a file already exists." self._str = "{0}('{1}',id_and_path_function=...{2}')".format( self.__class__.__name__, shared_directory_str, '' if self.leave_space == 0 else ',leave_space={0}'.format(leave_space))
def storage_closure(): return LocalCache(temp_dir)