def image_api():
image_api = ImageDownloadApi()
image_api.retrieve_file_over_http = \
MagicMock(name='retrieve_file_over_http')
return image_api
def image_api():
image_api = ImageDownloadApi()
image_api.retrieve_file_over_http = \
MagicMock(name='retrieve_file_over_http')
return image_api
def image_api():
image_api = ImageDownloadApi()
image_api.retrieve_file_over_http = \
MagicMock(name='retrieve_file_over_http')
image_api.json_msg_query = MagicMock(name='json_msg_query')
return image_api
def image_api():
image_api = ImageDownloadApi()
image_api.retrieve_file_over_http = \
MagicMock(name='retrieve_file_over_http')
image_api.json_msg_query = MagicMock(name='json_msg_query')
return image_api
class ImageDownloadApiTests(unittest.TestCase):
def __init__(self, *args, **kwargs):
super(ImageDownloadApiTests, self).__init__(*args, **kwargs)
self.ida = None
def setUp(self):
self.ida = ImageDownloadApi()
def tearDown(self):
self.ida = None
def test_api_doc_url_download_image_downsampled(self):
'''
Notes
-----
See: `Experimental Overview and Metadata `<http://help.brain-map.org/display/mouseconnectivity/API#API-ExperimentalOverviewandMetadata>_
, link labeled 'Download image downsampled by factor of 6 using default thresholds'.
'''
expected = 'http://api.brain-map.org/api/v2/section_image_download/126862575?downsample=6&range=0,932,0,1279,0,4095'
path = '126862575.jpg'
self.ida.retrieve_file_over_http = \
MagicMock(name='retrieve_file_over_http')
section_image_id = 126862575
self.ida.download_section_image(section_image_id,
downsample=6,
range=[0,932, 0,1279, 0,4095])
self.ida.retrieve_file_over_http.assert_called_once_with(expected, path)
def test_api_doc_url_download_image_full_resolution(self):
'''
Notes
-----
See: `Experimental Overview and Metadata `<http://help.brain-map.org/display/mouseconnectivity/API#API-ExperimentalOverviewandMetadata>_
, link labeled 'Download a region of interest at full resolution using default thresholds'.
'''
expected = 'http://api.brain-map.org/api/v2/section_image_download/126862575?left=19045&top=11684&width=1000&height=1000&range=0,932,0,1279,0,4095'
path = '126862575.jpg'
self.ida.retrieve_file_over_http = \
MagicMock(name='retrieve_file_over_http')
section_image_id = 126862575
self.ida.download_section_image(section_image_id,
left=19045,
top=11684,
width=1000,
height=1000,
range=[0,932, 0,1279, 0,4095])
self.ida.retrieve_file_over_http.assert_called_once_with(expected, path)
def __init__(self):
SvgApi.__init__(
self
) # https://github.com/AllenInstitute/AllenSDK/blob/master/allensdk/api/queries/svg_api.py
ImageDownloadApi.__init__(
self
) # https://github.com/AllenInstitute/AllenSDK/blob/master/allensdk/api/queries/image_download_api.py
self.annsetsapi = AnnotatedSectionDataSetsApi(
) # https://github.com/AllenInstitute/AllenSDK/blob/master/allensdk/api/queries/annotated_section_data_sets_api.py
self.oapi = OntologiesApi(
) # https://github.com/AllenInstitute/AllenSDK/blob/master/allensdk/api/queries/ontologies_api.py
# Get metadata about atlases
self.atlases = pd.DataFrame(self.oapi.get_atlases_table())
self.atlases_names = sorted(list(self.atlases['name'].values))
self.mouse_coronal_atlas_id = int(self.atlases.loc[
self.atlases['name'] == "Mouse, P56, Coronal"].id.values[0])
self.mouse_sagittal_atlas_id = int(self.atlases.loc[
self.atlases['name'] == "Mouse, P56, Sagittal"].id.values[0])
self.mouse_3D_atlas_id = int(self.atlases.loc[
self.atlases['name'] == "Mouse, Adult, 3D Coronal"].id.values[0])
# Get metadata about products
if connected_to_internet():
self.products = pd.DataFrame(
send_query(
"http://api.brain-map.org/api/v2/data/query.json?criteria=model::Product"
))
self.mouse_brain_reference_product_id = 12
self.mouse_brain_ish_data_product_id = 1
self.products_names = sorted(list(self.products["name"].values))
self.mouse_products_names = sorted(
list(self.products.loc[self.products.species == "Mouse"]
["name"].values))
else:
raise ConnectionError(
"It seems that you are not connected to the internet, you won't be able to download stuff."
)
def __init__(self):
SvgApi.__init__(
self
) # https://github.com/AllenInstitute/AllenSDK/blob/master/allensdk/api/queries/svg_api.py
ImageDownloadApi.__init__(
self
) # https://github.com/AllenInstitute/AllenSDK/blob/master/allensdk/api/queries/image_download_api.py
self.annsetsapi = (
AnnotatedSectionDataSetsApi()
) # https://github.com/AllenInstitute/AllenSDK/blob/master/allensdk/api/queries/annotated_section_data_sets_api.py
self.oapi = (
OntologiesApi()
) # https://github.com/AllenInstitute/AllenSDK/blob/master/allensdk/api/queries/ontologies_api.py
# Get metadata about atlases
self.atlases = pd.DataFrame(self.oapi.get_atlases_table())
self.atlases_names = sorted(list(self.atlases["name"].values))
self.mouse_coronal_atlas_id = int(self.atlases.loc[
self.atlases["name"] == self.mouse_coronal].id.values[0])
self.mouse_sagittal_atlas_id = int(self.atlases.loc[
self.atlases["name"] == self.mouse_sagittal].id.values[0])
self.mouse_3D_atlas_id = int(self.atlases.loc[
self.atlases["name"] == self.mouse3d].id.values[0])
# Get metadata about products
self.products = pd.DataFrame(
send_query(
"http://api.brain-map.org/api/v2/data/query.json?criteria=model::Product"
))
self.mouse_brain_reference_product_id = 12
self.mouse_brain_ish_data_product_id = 1
self.products_names = sorted(list(self.products["name"].values))
self.mouse_products_names = sorted(
list(self.products.loc[self.products.species == "Mouse"]
["name"].values))
def __init__(self, input_dir, process_dir, output_dir, structure_map_dir,
structs, connectivity_dir, _processor_number,
brightness_threshold, strains):
super().__init__(input_dir, process_dir, output_dir,
f'experiment-images-downloader-{_processor_number}')
self.brightness_threshold = brightness_threshold
self.structs = ast.literal_eval(structs)
self.segmentation_dir = structure_map_dir
self.mcc = MouseConnectivityCache(
manifest_file=f'{connectivity_dir}/mouse_connectivity_manifest.json'
)
struct_tree = self.mcc.get_structure_tree()
structure_ids = [
i for sublist in struct_tree.descendant_ids(self.structs)
for i in sublist
]
self.structure_ids = set(structure_ids)
self.image_api = ImageDownloadApi()
self.bbox_dilation_kernel = cv2.getStructuringElement(
cv2.MORPH_ELLIPSE, (14, 14))
exps = self.mcc.get_experiments(dataframe=True)
items = []
for s, gs in strains.items():
strain_items = []
for g in gs:
strain_items += [
sorted(exps[(exps.strain == s)
& (exps.gender == g)].id.tolist())
]
if strain_items:
min_len = min([len(i) for i in strain_items])
strain_items = [i[:min_len] for i in strain_items]
items += [str(i) for j in zip(*strain_items) for i in j]
self.initial_items = [i for i in items if i in self.initial_items] + [
i for i in self.initial_items if i not in items
]
def download_section(savepath, section_id, downsample):
# Downloading all of the images from a section data set
image_api = ImageDownloadApi()
input_directory = str(section_id) + '_input'
output_directory = str(section_id) + '_output'
format_str = '.jpg'
section_images = image_api.section_image_query(section_id)
section_image_ids = [si['id'] for si in section_images]
# You have probably noticed that the AllenSDK has a logger which notifies you of file downloads.
# Since we are downloading ~300 images, we don't want to see messages for each one.
# The following line will temporarily disable the download logger.(optional)
logging.getLogger(
'allensdk.api.api.retrieve_file_over_http').disabled = True
for section_image_id in section_image_ids:
file_name = str(section_image_id) + format_str
input_file_path = os.path.join(savepath, input_directory, file_name)
output_file_path = os.path.join(savepath, output_directory, file_name)
Manifest.safe_make_parent_dirs(input_file_path)
image_api.download_section_image(section_image_id,
file_path=input_file_path,
downsample=downsample,
expression=0)
Manifest.safe_make_parent_dirs(output_file_path)
image_api.download_section_image(section_image_id,
file_path=output_file_path,
downsample=downsample,
expression=1)
# re-enable the logger (optional)
logging.getLogger(
'allensdk.api.api.retrieve_file_over_http').disabled = False
file_names = os.listdir(os.path.join(savepath, input_directory))
print(len(file_names))
def download_brain_slice(df):
# create an image download API
image_api = ImageDownloadApi()
format_str = ".jpg"
# You have probably noticed that the AllenSDK has a logger which notifies you of file downloads.
# Since we are downloading ~300 images, we don't want to see messages for each one.
# The following line will temporarily disable the download logger.
logging.getLogger("allensdk.api.api.retrieve_file_over_http").disabled = True
# get parameters
path, downsample, indices = ask_parameters_for_downloading(df)
print(
"Downloads initiated", end="...", file=sys.stderr, flush=True,
)
for index in indices:
# from indices, get experiment id and gene symbol from df
exp_id = df["Experiment"][index]
# set the dirname as the gene symbol
dirname = df["Gene Symbol"][index]
plane = df["Plane"][index]
section_data_set_id = exp_id
section_image_directory = os.path.join(path, dirname)
# get the image ids for all of the images in this data set
section_images = image_api.section_image_query(
section_data_set_id
) # Should be a dicionary of the features of section images
section_image_ids = [
si["id"] for si in section_images
] # Take value of 'id' from the dictionary
# Create a progress bar
pbar_image = tqdm(total=len(section_image_ids), desc=dirname + " " + plane)
for section_image_id in section_image_ids:
file_name = str(section_image_id) + format_str
file_path = os.path.join(section_image_directory, file_name)
Manifest.safe_make_parent_dirs(file_path)
# Check if the file is already downloaded, which happens if the downloads have been interrupted.
saved_file_names = os.listdir(section_image_directory)
if file_name in saved_file_names:
pass
else:
image_api.download_section_image(
section_image_id, file_path=file_path, downsample=downsample
)
pbar_image.update()
pbar_image.close()
# re-enable the logger
logging.getLogger("allensdk.api.api.retrieve_file_over_http").disabled = False
print(
"Downloads completed.", file=sys.stderr, flush=True,
)
from allensdk.api.queries.image_download_api import ImageDownloadApi from allensdk.api.queries.svg_api import SvgApi from allensdk.config.manifest import Manifest import matplotlib.pyplot as plt from skimage.io import imread import pandas as pd import logging import os from base64 import b64encode from IPython.display import HTML, display #product_id = 8 #file_name = str(product_id) + format_str #file_path = os.path.join(section_image_directory, file_name) #Manifest.safe_make_parent_dirs(file_path) image_api = ImageDownloadApi() print(image_api.get_section_data_sets_by_product(product_ids=8))
def setUp(self):
self.ida = ImageDownloadApi()
def verify_svg(file_path, width_scale, height_scale):
# we're using this function to display scaled svg in the rendered notebook.
# we suggest that in your own work you use a tool such as inkscape or illustrator to view svg
with open(file_path, 'rb') as svg_file:
svg = svg_file.read()
encoded_svg = b64encode(svg)
decoded_svg = encoded_svg.decode('ascii')
st = r'<img class="figure" src="data:image/svg+xml;base64,{}" width={}% height={}%></img>'.format(decoded_svg, width_scale, height_scale)
display(HTML(st))
image_api = ImageDownloadApi()
svg_api = SvgApi()
#Downloading all of the images from a section data set
section_data_set_id = 71724696
downsample = 4
expression = 1
section_image_directory = str(section_data_set_id) + '_section_images'
format_str = '.jpg'
section_images = image_api.section_image_query(section_data_set_id)
section_image_ids = [si['id'] for si in section_images]
class ExperimentImagesDownloader(DirWatcher):
def __init__(self, input_dir, process_dir, output_dir, structure_map_dir,
structs, connectivity_dir, _processor_number,
brightness_threshold, strains):
super().__init__(input_dir, process_dir, output_dir,
f'experiment-images-downloader-{_processor_number}')
self.brightness_threshold = brightness_threshold
self.structs = ast.literal_eval(structs)
self.segmentation_dir = structure_map_dir
self.mcc = MouseConnectivityCache(
manifest_file=f'{connectivity_dir}/mouse_connectivity_manifest.json'
)
struct_tree = self.mcc.get_structure_tree()
structure_ids = [
i for sublist in struct_tree.descendant_ids(self.structs)
for i in sublist
]
self.structure_ids = set(structure_ids)
self.image_api = ImageDownloadApi()
self.bbox_dilation_kernel = cv2.getStructuringElement(
cv2.MORPH_ELLIPSE, (14, 14))
exps = self.mcc.get_experiments(dataframe=True)
items = []
for s, gs in strains.items():
strain_items = []
for g in gs:
strain_items += [
sorted(exps[(exps.strain == s)
& (exps.gender == g)].id.tolist())
]
if strain_items:
min_len = min([len(i) for i in strain_items])
strain_items = [i[:min_len] for i in strain_items]
items += [str(i) for j in zip(*strain_items) for i in j]
self.initial_items = [i for i in items if i in self.initial_items] + [
i for i in self.initial_items if i not in items
]
def on_process_error(self, item, exception):
retval = super().on_process_error(item, exception)
self.logger.error(f"Error occurred during processing", exc_info=True)
if any(
map(lambda x: issubclass(type(exception), x), [
urllib.error.HTTPError, OSError, ValueError,
http.client.error
])):
return False
else:
return retval
def process_item(self, item, directory):
experiment_id = int(item)
retries = 0
images = []
while True:
try:
time.sleep(2**(retries // 2))
images = self.image_api.section_image_query(experiment_id)
break
except simplejson.errors.JSONDecodeError as e:
if retries > 10:
raise e
else:
self.logger.info(f"Exception invoking image API, retrying")
retries += 1
continue
images = {i['section_number']: i for i in images}
segmentation = np.load(
f'{self.segmentation_dir}/{item}/{item}-sections.npz')['arr_0']
mask = np.isin(segmentation, list(self.structure_ids))
locs = np.where(mask)
sections = [
s for s in sorted(np.unique(locs[2]).tolist()) if s in images
]
bboxes = {
section: self.extract_bounding_boxes(mask[:, :, section])
for section in sections
}
valid_sections = list(filter(lambda s: bboxes[s], sections))
self.logger.info(
f"Experiment {experiment_id}, evaluating brightness...")
brightness = self.calculate_brightness(directory, experiment_id,
images, valid_sections, mask)
if brightness < self.brightness_threshold:
self.logger.info(
f"Experiment {experiment_id}: brightness less than required minimum, removing..."
)
return False
with open(f'{directory}/bboxes.pickle', 'wb') as f:
pickle.dump(bboxes, f)
for section in valid_sections:
self.process_section(directory, experiment_id, images, section,
bboxes[section])
def calculate_brightness(self, directory, experiment_id, images,
valid_sections, mask):
pixels = np.zeros_like(mask, dtype=np.uint8)
for section in valid_sections:
self.download_snapshot(experiment_id, section, images[section],
directory)
filename = f'{directory}/thumbnail-{experiment_id}-{section}.jpg'
image = cv2.imread(filename, cv2.IMREAD_GRAYSCALE)
pixels[:, :, section] = image[:mask.shape[0], :mask.shape[1]]
return np.median(pixels[mask != 0])
def process_section(self, directory, experiment_id, images, section,
bboxes):
self.logger.info(
f"Experiment {experiment_id}, downloading section {section}...")
self.download_snapshot(experiment_id, section, images[section],
directory)
for bbox in bboxes:
self.download_fullres(experiment_id, section, bbox,
images[section], directory)
def extract_bounding_boxes(self, mask, area_threshold=0):
bboxes = self.get_bounding_boxes(mask)
bbmask = np.zeros_like(mask, dtype=np.uint8)
for bbox in bboxes:
cv2.rectangle(bbmask, *bbox.corners(), color=1, thickness=-1)
bbmask = cv2.dilate(bbmask, self.bbox_dilation_kernel)
bboxes = [
bbox for bbox in self.get_bounding_boxes(bbmask)
if bbox.area() > area_threshold
]
return bboxes
@staticmethod
def get_bounding_boxes(mask):
contours, hierarchy = cv2.findContours(mask.astype(np.uint8),
cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)[-2:]
rects = []
for cnt in contours:
x, y, w, h = cv2.boundingRect(cnt)
if w > 5 and h > 5:
rects.append(Rect(x=x, y=y, w=w, h=h))
return rects
def download_fullres(self, experiment_id, section, bbox, image_desc,
directory):
url = f'https://connectivity.brain-map.org/cgi-bin/imageservice?path={image_desc["path"]}&' \
f'mime=1&zoom={8}&&filter=range&filterVals=0,534,0,1006,0,4095'
x, y, w, h = bbox.scale(64)
url += f'&top={y}&left={x}&width={w}&height={h}'
filename = f'{directory}/full-{experiment_id}-{section}-{x}_{y}_{w}_{h}.jpg'
for retries in range(3):
fname, _, downloaded = self.retrieve_url(filename, url)
if downloaded:
try:
image = Image.open(fname)
image.load()
break
except OSError or FileNotFoundError as e:
os.remove(fname)
if retries == 2:
raise e
else:
self.logger.info(
f"Corrupted file {fname}, re-downloading {filename}"
)
else:
self.logger.info(
f"Cached version of {filename} used, skipping verification"
)
return filename
def download_snapshot(self, experiment_id, section, image_desc, directory):
url = f'https://connectivity.brain-map.org/cgi-bin/imageservice?path={image_desc["path"]}&' \
f'mime=1&zoom={2}&&filter=range&filterVals=0,534,0,1006,0,4095'
filename = f'{directory}/thumbnail-{experiment_id}-{section}.jpg'
filename, _, _ = self.retrieve_url(filename, url)
return filename
def download_brightness_snapshot(self, experiment_id, section, image_desc,
directory):
url = f'https://connectivity.brain-map.org/cgi-bin/imageservice?path={image_desc["path"]}&' \
f'mime=1&zoom={2}&&filter=range&filterVals=0,534,0,1006,0,4095'
filename = f'{directory}/thumbnail-{experiment_id}-{section}.jpg'
filename, _, _ = self.retrieve_url(filename, url)
return filename
def retrieve_url(self, filename, url, retries=10):
if os.path.isfile(filename):
self.logger.info(f"File {filename} already downloaded")
return filename, None, False
backoff = 0
urllib.request.urlcleanup()
while True:
try:
time.sleep(2**backoff)
fname, msg = urllib.request.urlretrieve(
url, filename=f'{filename}.partial')
os.replace(fname, filename)
return filename, msg, True
except http.client.HTTPException or OSError or urllib.error.HTTPError as e:
backoff += 1
retries -= 1
if retries > 0:
self.logger.info(
f"Transient error downloading {url}, "
f"retrying ({retries} retries left) ...",
exc_info=True)
continue
else:
self.logger.exception(
f"Retry count exceeded or permanent error for {url} ({filename}), exiting..."
)
raise e
'experiment_id':
int(entry['filename'].split('.')[0].split('-')[0]),
'slice_id':
int(entry['filename'].split('.')[0].split('-')[1]),
'shape':
entry['regions'][0]['shape_attributes'],
'filename':
entry['filename']
} for entry in entries]
experiments = defaultdict(list)
for info in scan_info:
experiments[info['experiment_id']] += [info]
image_api = ImageDownloadApi()
for experiment_id, infos in experiments.items():
images = {
s['section_number']: s
for s in image_api.section_image_query(experiment_id)
}
for info in infos:
img = images[info["slice_id"]]
y = int(info["shape"]["y"]) // 4
x = int(info["shape"]["x"]) // 4
width = int(info["shape"]["width"]) // 4
height = int(info["shape"]["height"]) // 4
zoom_factor = 8
print(f'\tProcessing slice {info["slice_id"]}')
url = f'http://connectivity.brain-map.org/cgi-bin/imageservice?path={img["path"]}&zoom={zoom_factor}&'\
def get_image(image_id, folder=None, **kwargs):
"""Get any image from Allen's database just by its id.
Notes
-----
All requested images are stored in the `CACHED_FOLDER` and then read.
Parameters
----------
image_id : int
Integer representing an id of the section image.
folder : str or LocalPath or None
Local folder where image saved. If None then automatically defaults to `CACHE_FOLDER`.
**kwargs
Additional parameters to be passed onto the `download_image` method of ``ImageDownloadApi``. See
See references for details.
Returns
-------
img : np.ndarray
Downloaded/locally loaded image. The dtype is np.uint8.
References
----------
[1] https://allensdk.readthedocs.io/en/latest/allensdk.api.queries.image_download_api.html#allensdk.api.queries.image_download_api.ImageDownloadApi # noqa
"""
folder = folder or CACHE_FOLDER
folder = str(
folder
) # this should guarantee that also LocalPath works (pytest uses it)
# Check the folder exists
if not os.path.exists(folder):
os.makedirs(folder)
# Create full path
additional_speficier = "_".join(
sorted(["{}_{}".format(k, v) for k, v in kwargs.items()])
)
if additional_speficier:
additional_speficier = "_{}".format(additional_speficier)
path = "{}{}{}.jpg".format(folder, image_id, additional_speficier)
# Check image exists
if os.path.exists(path):
img = plt.imread(path)
if not img.dtype == np.uint8:
raise ValueError("The dtype needs to be uint8")
return img
else:
img_api = ImageDownloadApi()
img_api.download_image(image_id, file_path=path, **kwargs)
return get_image(image_id, **kwargs)
