def test_one_chrom(self):
logging.info("test_one_chrom")
output_file = self.file_name("one_chrom")
storage = LocalCache("local_cache/one_chrom")
test_storage = storage.join('test_snps')
test_storage.rmtree('')
test_snps3 = self.bed[:, self.bed.pos[:, 0] ==
3] # Test only on chromosome 3
test_snps3_dist = DistributedBed.write(test_storage,
test_snps3,
piece_per_chrom_count=2)
for test_snps, ref, clear_cache, name in (
(test_snps3, "old_one", True, "Run with just chrom3"),
(test_snps3_dist, "old_one", True,
"Run with distributed test SNPs"),
(test_snps3, "old_one", False, "Run with just chrom3 (use cache)"),
(test_snps3_dist, "old_one", False,
"Run with distributed test SNPs (use cache)"),
):
logging.info("=========== " + name + " ===========")
results_df = single_snp_scale(
test_snps=test_snps,
pheno=self.phen_fn,
covar=self.cov_fn,
K0=self.bed,
cache=self._cache_dict(storage, clear_cache=clear_cache),
output_file_name=output_file,
)
self.compare_files(results_df, ref)
def test_local_distribute(self):
logging.info("test_local_distribute")
force_python_only = False
output_file = self.file_name("local_distribute")
storage = LocalCache("local_cache/local_distribute")
test_storage = storage.join('test_snps')
test_storage.rmtree('')
test_snps = DistributedBed.write(test_storage,
self.bed,
piece_per_chrom_count=2)
results_df = single_snp_scale(test_snps=test_snps,
pheno=self.phen_fn,
covar=self.cov_fn,
G0=self.bed,
cache=self._cache_dict(storage,
clear_cache=True),
output_file_name=output_file,
force_python_only=force_python_only)
self.compare_files(results_df, "old")
results_df = single_snp_scale(test_snps=self.bed,
pheno=self.phen_fn,
covar=self.cov_fn,
G0=self.bed,
cache=self._cache_dict(
storage, clear_cache=False),
output_file_name=output_file)
self.compare_files(results_df, "old")
def scan_local(local_directory, url=None, logging_level=logging.WARNING):
'''
Bootstrap a Hashdown by recursively walking a local directory and finding the local MD5 hashes.
(A local hash might be wrong if the files are out of date or have OS-dependent line endings.)
Typically, you'll then want to save the result to a JSON file and then edit that JSON file
manually to remove uninteresting files.
:param local_directory: Local directory to recursively walk
:type path: string
:param url: URL to give to the Hashdown. (It will not be checked.)
:type path: string
:param logging_level: Logging level for printing progress of the walk. Default
is logging.WARNING)
:rtype: :class:`.Hashdown`
'''
from pysnptools.util.filecache import LocalCache
file_to_hash = {}
localcache = LocalCache(local_directory)
with log_in_place("scanning", logging_level) as updater:
for file in localcache.walk():
updater(file)
with localcache.open_read(file) as full_file:
hash = Hashdown._get_hash(full_file)
file_to_hash[file] = hash
return Hashdown(url, file_to_hash=file_to_hash)
def test_low(self):
logging.info("test_low")
output_file = self.file_name("low")
storage = LocalCache("local_cache/low")
for clear_cache in (True, False):
if clear_cache:
storage.rmtree()
results_df = single_snp_scale(test_snps=self.bed,
pheno=self.phen_fn,
covar=self.cov_fn,
cache=storage,
output_file_name=output_file)
self.compare_files(results_df, "old")
def _fixup(cache_value,default_subfolder='filecache'):
if isinstance(cache_value,FileCache):
return cache_value
if cache_value is None:
dirpath = tempfile.mkdtemp()+'/'+default_subfolder
return FileCache._fixup(dirpath)
if isinstance(cache_value,str):
from pysnptools.util.filecache import LocalCache
return LocalCache(cache_value)
raise Exception("Do not know how to make a FileCache from '{0}'".format(cache_value))
def test_one_fast(self):
logging.info("test_one_fast")
output_file = self.file_name("one_fast")
storage = LocalCache("local_cache")
test_storage = storage.join('one_fast')
test_storage.rmtree()
test_snps3 = self.bed[:, self.bed.pos[:, 0] ==
3] # Test only on chromosome 3
test_snps3_dist = DistributedBed.write(test_storage,
test_snps3,
piece_per_chrom_count=2)
results_df = single_snp_scale(test_snps=test_snps3_dist,
pheno=self.phen_fn,
covar=self.cov_fn,
G0=self.bed,
output_file_name=output_file)
self.compare_files(results_df, "old_one")
def __init__(self, common_directory, id_and_path_function, leave_space=0):
from pysnptools.util.filecache import LocalCache
super(PeerToPeer, self).__init__()
shared_directory_str = str(common_directory)
if isinstance(common_directory, str):
common_directory = LocalCache(common_directory)
self.common_directory = common_directory
self.id_and_path_function = id_and_path_function
self.leave_space = leave_space
assert not self.common_directory._simple_file_exists(
"main.txt"
), "A common_directory cannot exist where a file already exists."
self._str = "{0}('{1}',id_and_path_function=...{2}')".format(
self.__class__.__name__, shared_directory_str, '' if
self.leave_space == 0 else ',leave_space={0}'.format(leave_space))
def storage_closure():
return LocalCache(temp_dir)