def _download_atlas_flatiron(file_image, FLAT_IRON_ATLAS_REL_PATH, par):
file_image.parent.mkdir(exist_ok=True, parents=True)
url = (par.HTTP_DATA_SERVER + '/' +
'/'.join(FLAT_IRON_ATLAS_REL_PATH.parts) + '/' + file_image.name)
http_download_file(url, cache_dir=Path(par.CACHE_DIR).joinpath(FLAT_IRON_ATLAS_REL_PATH),
username=par.HTTP_DATA_SERVER_LOGIN,
password=par.HTTP_DATA_SERVER_PWD)
def download_raw_partial(self, url_cbin, url_ch, first_chunk=0, last_chunk=0):
assert url_cbin.endswith('.cbin')
assert url_ch.endswith('.ch')
relpath = Path(url_cbin.replace(self._par.HTTP_DATA_SERVER, '.')).parents[0]
target_dir = Path(self._get_cache_dir(None), relpath)
Path(target_dir).mkdir(parents=True, exist_ok=True)
# First, download the .ch file.
ch_local_path = Path(wc.http_download_file(
url_ch,
username=self._par.HTTP_DATA_SERVER_LOGIN,
password=self._par.HTTP_DATA_SERVER_PWD,
cache_dir=target_dir, clobber=True, offline=False, return_md5=False))
ch_local_path = remove_uuid_file(ch_local_path)
ch_local_path = ch_local_path.rename(ch_local_path.with_suffix('.chopped.ch'))
assert ch_local_path.exists()
# Load the .ch file.
with open(ch_local_path, 'r') as f:
cmeta = json.load(f)
# Get the first byte and number of bytes to download.
i0 = cmeta['chunk_bounds'][first_chunk]
cmeta['chunk_bounds'] = cmeta['chunk_bounds'][first_chunk:last_chunk + 2]
cmeta['chunk_bounds'] = [_ - i0 for _ in cmeta['chunk_bounds']]
assert len(cmeta['chunk_bounds']) >= 2
assert cmeta['chunk_bounds'][0] == 0
first_byte = cmeta['chunk_offsets'][first_chunk]
cmeta['chunk_offsets'] = cmeta['chunk_offsets'][first_chunk:last_chunk + 2]
cmeta['chunk_offsets'] = [_ - first_byte for _ in cmeta['chunk_offsets']]
assert len(cmeta['chunk_offsets']) >= 2
assert cmeta['chunk_offsets'][0] == 0
n_bytes = cmeta['chunk_offsets'][-1]
assert n_bytes > 0
# Save the chopped chunk bounds and ossets.
cmeta['sha1_compressed'] = None
cmeta['sha1_uncompressed'] = None
cmeta['chopped'] = True
with open(ch_local_path, 'w') as f:
json.dump(cmeta, f, indent=2, sort_keys=True)
# Download the requested chunks
cbin_local_path = wc.http_download_file(
url_cbin,
username=self._par.HTTP_DATA_SERVER_LOGIN,
password=self._par.HTTP_DATA_SERVER_PWD,
cache_dir=target_dir, clobber=True, offline=False, return_md5=False,
chunks=(first_byte, n_bytes))
cbin_local_path = remove_uuid_file(cbin_local_path)
cbin_local_path = cbin_local_path.rename(cbin_local_path.with_suffix('.chopped.cbin'))
assert cbin_local_path.exists()
import mtscomp
reader = mtscomp.decompress(cbin_local_path, cmeta=ch_local_path)
return reader[:]
def download_histology_data(subject, lab):
if lab == 'hoferlab':
lab_temp = 'mrsicflogellab'
else:
lab_temp = lab
par = params.read('one_params')
try:
FLAT_IRON_HIST_REL_PATH = Path('histology', lab_temp, subject,
'downsampledStacks_25', 'sample2ARA')
baseurl = (par.HTTP_DATA_SERVER + '/' + '/'.join(FLAT_IRON_HIST_REL_PATH.parts))
r = requests.get(baseurl, auth=(par.HTTP_DATA_SERVER_LOGIN, par.HTTP_DATA_SERVER_PWD))
r.raise_for_status()
except Exception as err:
print(err)
try:
subject_rem = subject.replace("_", "")
FLAT_IRON_HIST_REL_PATH = Path('histology', lab_temp, subject_rem,
'downsampledStacks_25', 'sample2ARA')
baseurl = (par.HTTP_DATA_SERVER + '/' + '/'.join(FLAT_IRON_HIST_REL_PATH.parts))
r = requests.get(baseurl, auth=(par.HTTP_DATA_SERVER_LOGIN, par.HTTP_DATA_SERVER_PWD))
r.raise_for_status()
except Exception as err:
print(err)
path_to_nrrd = None
return path_to_nrrd
tif_files = []
for line in r.text.splitlines():
result = re.findall('href="(.*).tif"', line)
if result:
tif_files.append(result[0] + '.tif')
CACHE_DIR = Path(Path.home(), 'Downloads', 'FlatIron', lab, 'Subjects', subject, 'histology')
CACHE_DIR.mkdir(exist_ok=True, parents=True)
path_to_files = []
for file in tif_files:
path_to_image = Path(CACHE_DIR, file)
if not path_to_image.exists():
url = (baseurl + '/' + file)
http_download_file(url, cache_dir=CACHE_DIR,
username=par.HTTP_DATA_SERVER_LOGIN,
password=par.HTTP_DATA_SERVER_PWD)
path_to_nrrd = tif2nrrd(path_to_image)
path_to_files.append(path_to_nrrd)
if len(path_to_files) > 3:
path_to_files = path_to_files[1:3]
return path_to_files
def _download_file(self,
url,
cache_dir,
clobber=False,
offline=False,
keep_uuid=False,
file_size=None,
hash=None):
local_path = cache_dir + os.sep + os.path.basename(url)
if not keep_uuid:
local_path = remove_uuid_file(local_path, dry=True)
if Path(local_path).exists():
# overwrites the file if the expected filesize is different from the cached filesize
if file_size and Path(local_path).stat().st_size != file_size:
clobber = True
# overwrites the file if the expected hash is different from the cached hash
if hash and hashfile.md5(Path(local_path)) != hash:
clobber = True
# if there is no cached file, download
else:
clobber = True
if clobber:
local_path = wc.http_download_file(
url,
username=self._par.HTTP_DATA_SERVER_LOGIN,
password=self._par.HTTP_DATA_SERVER_PWD,
cache_dir=str(cache_dir),
clobber=clobber,
offline=offline)
if keep_uuid:
return local_path
else:
return remove_uuid_file(local_path)
def test_download_datasets(self):
# test downloading a single file
full_link_to_file = r'http://ibl.flatironinstitute.org/mainenlab/Subjects/clns0730'\
'/2018-08-24/1/licks.times.51852a2f-c76e-4c0c-95cb-9c7ba54be0f9.npy'
file_name = wc.http_download_file(full_link_to_file,
username=par.HTTP_DATA_SERVER_LOGIN,
password=par.HTTP_DATA_SERVER_PWD)
a = np.load(file_name)
self.assertTrue(len(a) > 0)
# test downloading a list of files
links = [
r'http://ibl.flatironinstitute.org/mainenlab/Subjects/clns0730'
'/2018-08-24/1/licks.times.51852a2f-c76e-4c0c-95cb-9c7ba54be0f9.npy',
r'http://ibl.flatironinstitute.org/mainenlab/Subjects/clns0730'
'/2018-08-24/1/probes.sitePositions.3ddd45be-7d24-4fc7-9dd3-a98717342af6.npy'
]
file_list = wc.http_download_file_list(
links,
username=par.HTTP_DATA_SERVER_LOGIN,
password=par.HTTP_DATA_SERVER_PWD)
a = np.load(file_list[0])
b = np.load(file_list[1])
self.assertTrue(len(a) > 0)
self.assertTrue(len(b) > 0)
def _download_file(self, url, cache_dir, clobber=False):
local_path = wc.http_download_file(
url,
username=self._par.HTTP_DATA_SERVER_LOGIN,
password=self._par.HTTP_DATA_SERVER_PWD,
cache_dir=str(cache_dir),
clobber=clobber)
return local_path
def _download_file(self,
url,
target_dir,
clobber=False,
offline=False,
keep_uuid=False,
file_size=None,
hash=None):
"""
Downloads a single file from an HTTP webserver
:param url:
:param cache_dir:
:param clobber: (bool: False) overwrites local dataset if any
:param offline:
:param keep_uuid:
:param file_size:
:param hash:
:return:
"""
Path(target_dir).mkdir(parents=True, exist_ok=True)
local_path = str(target_dir) + os.sep + os.path.basename(url)
if not keep_uuid:
local_path = alfio.remove_uuid_file(local_path, dry=True)
if Path(local_path).exists() and not offline:
# the local file hash doesn't match the dataset table cached hash
hash_mismatch = hash and hashfile.md5(Path(local_path)) != hash
file_size_mismatch = file_size and Path(
local_path).stat().st_size != file_size
if hash_mismatch or file_size_mismatch:
clobber = True
_logger.warning(
f" local md5 or size mismatch, re-downloading {local_path}"
)
# if there is no cached file, download
else:
clobber = True
if clobber:
local_path, md5 = wc.http_download_file(
url,
username=self._par.HTTP_DATA_SERVER_LOGIN,
password=self._par.HTTP_DATA_SERVER_PWD,
cache_dir=str(target_dir),
clobber=clobber,
offline=offline,
return_md5=True)
# post download, if there is a mismatch between Alyx and the newly downloaded file size
# or hash flag the offending file record in Alyx for database maintenance
hash_mismatch = hash and md5 != hash
file_size_mismatch = file_size and Path(
local_path).stat().st_size != file_size
if hash_mismatch or file_size_mismatch:
self._tag_mismatched_file_record(url)
if keep_uuid:
return local_path
else:
return alfio.remove_uuid_file(local_path)
def _download_file(self,
url,
target_dir,
clobber=False,
offline=False,
keep_uuid=False,
file_size=None,
hash=None):
"""
Downloads a single file from an HTTP webserver
:param url:
:param cache_dir:
:param clobber: (bool: False) overwrites local dataset if any
:param offline:
:param keep_uuid:
:param file_size:
:param hash:
:return:
"""
Path(target_dir).mkdir(parents=True, exist_ok=True)
local_path = str(target_dir) + os.sep + os.path.basename(url)
if not keep_uuid:
local_path = remove_uuid_file(local_path, dry=True)
if Path(local_path).exists():
# overwrites the file if the expected filesize is different from the cached filesize
if file_size and Path(local_path).stat().st_size != file_size:
clobber = True
# overwrites the file if the expected hash is different from the cached hash
if hash and hashfile.md5(Path(local_path)) != hash:
clobber = True
# if there is no cached file, download
else:
clobber = True
if clobber:
local_path = wc.http_download_file(
url,
username=self._par.HTTP_DATA_SERVER_LOGIN,
password=self._par.HTTP_DATA_SERVER_PWD,
cache_dir=str(target_dir),
clobber=clobber,
offline=offline)
if keep_uuid:
return local_path
else:
return remove_uuid_file(local_path)
def _download_file(self,
url,
cache_dir,
clobber=False,
offline=False,
keep_uuid=False):
local_path = cache_dir + os.sep + os.path.basename(url)
if not keep_uuid:
local_path = remove_uuid_file(local_path, dry=True)
if clobber or not Path(local_path).exists():
local_path = wc.http_download_file(
url,
username=self._par.HTTP_DATA_SERVER_LOGIN,
password=self._par.HTTP_DATA_SERVER_PWD,
cache_dir=str(cache_dir),
clobber=clobber,
offline=offline)
if keep_uuid:
return local_path
else:
return remove_uuid_file(local_path)
