def migrate_optical_images(ds_id):
output.print('Migrating optical images')
with timeit():
output.print('Transferring images and updating database...')
db = DB()
rows = db.select(SEL_OPTICAL_IMGS, params=(ds_id,))
for opt_image_id, opt_image_url in rows:
if not opt_image_url and opt_image_id:
transfer_images(
ds_id,
'optical_images',
image_storage.OPTICAL,
[opt_image_id],
)
opt_image_url = image_storage.get_image_url(
image_storage.OPTICAL, ds_id, opt_image_id
)
db.alter(UPD_OPTICAL_IMGS, params=(opt_image_url, opt_image_id))
opt_thumb_id, opt_thumb_url = db.select_one(SEL_OPT_THUMB, params=(ds_id,))
if not opt_thumb_url and opt_thumb_id:
transfer_images(
ds_id,
'optical_images',
image_storage.OPTICAL,
[opt_thumb_id],
)
opt_thumb_url = image_storage.get_image_url(image_storage.OPTICAL, ds_id, opt_thumb_id)
db.alter(UPD_OPT_THUMB, params=(opt_thumb_url, ds_id))
def save_diagnostic_image(ds_id: str,
data: Any,
key,
index=None,
fmt=DiagnosticImageFormat.NPY) -> DiagnosticImage:
assert key in DiagnosticImageKey
if fmt == DiagnosticImageFormat.NPY:
image_id = save_npy_image(ds_id, data)
elif fmt == DiagnosticImageFormat.JSON:
image_id = image_storage.post_image(
image_storage.DIAG, ds_id,
BytesIO(numpy_json_dumps(data).encode()))
elif fmt == DiagnosticImageFormat.PARQUET:
image_id = save_parquet_image(ds_id, data)
else:
raise ValueError(f'Unknown format: {fmt}')
image = DiagnosticImage(
key=key,
image_id=image_id,
url=image_storage.get_image_url(image_storage.DIAG, ds_id, image_id),
format=fmt,
)
if index is not None:
image['index'] = index
return image
def save_image(img):
buf = _save_jpeg(img)
scaled_img_id = image_storage.post_image(image_storage.OPTICAL, ds.id,
buf.read())
scaled_img_url = image_storage.get_image_url(image_storage.OPTICAL,
ds.id, scaled_img_id)
return scaled_img_id, scaled_img_url
def _index_ds_annotations(self, ds_id, moldb, ds_doc, isocalc):
annotation_docs = self._db.select_with_fields(ANNOTATIONS_SEL,
params=(ds_id, moldb.id))
logger.info(
f'Indexing {len(annotation_docs)} documents: {ds_id}, {moldb}')
annotation_counts = defaultdict(int)
mol_by_formula = self._get_mol_by_formula_dict(moldb)
for doc in annotation_docs:
self._expand_stats_field(doc)
self._add_ds_fields_to_ann(doc, ds_doc)
doc['db_id'] = moldb.id
doc['db_name'] = moldb.name
doc['db_version'] = moldb.version
formula = doc['formula']
ion_without_pol = format_ion_formula(formula, doc['chem_mod'],
doc['neutral_loss'],
doc['adduct'])
doc['ion'] = ion_without_pol + doc['polarity']
doc['comp_ids'], doc['comp_names'] = mol_by_formula[formula]
mzs, _ = isocalc.centroids(ion_without_pol)
doc['centroid_mzs'] = list(mzs) if mzs is not None else []
doc['mz'] = mzs[0] if mzs is not None else 0
doc['iso_image_urls'] = [
image_storage.get_image_url(image_storage.ISO, ds_id, image_id)
if image_id else None for image_id in doc['iso_image_ids']
]
if moldb.targeted and ds_doc['ds_config'].get(
'analysis_version', 1) == 1:
fdr_level = doc['fdr'] = -1
else:
fdr_level = FDR.nearest_fdr_level(doc['fdr'])
annotation_counts[round(fdr_level * 100, 2)] += 1
self._add_isomer_fields_to_anns(annotation_docs)
ESExporterIsobars.add_isobar_fields_to_anns(annotation_docs, isocalc)
to_index = []
for doc in annotation_docs:
to_index.append({
'_index': self.index,
'_type': 'annotation',
'_id': f"{doc['ds_id']}_{doc['annotation_id']}",
'_source': doc,
})
for success, info in parallel_bulk(self._es,
actions=to_index,
timeout='60s'):
if not success:
logger.error(f'Document failed: {info}')
return annotation_counts
def _add_thumbnail_optical_image(db, ds, dims, optical_img, transform):
logger.debug(f'Saving optical image thumbnail: {optical_img}')
thumbnail_size = (200, 200)
db.alter(UPD_DATASET_THUMB_OPTICAL_IMAGE, params=(None, None, ds.id))
img = _transform_image_to_ion_space(optical_img, transform, dims,
zoom=1)[0]
img.thumbnail(thumbnail_size, Image.ANTIALIAS)
buf = _save_jpeg(img)
img_thumb_id = image_storage.post_image(image_storage.OPTICAL, ds.id,
buf.read())
img_thumb_url = image_storage.get_image_url(image_storage.OPTICAL, ds.id,
img_thumb_id)
db.alter(UPD_DATASET_THUMB_OPTICAL_IMAGE,
params=(img_thumb_id, img_thumb_url, ds.id))
def migrate_ion_thumbnail(ds_id):
output.print('Migrating ion thumbnail images')
with timeit():
output.print('Transferring images and updating database...')
db = DB()
ion_thumb_id, ion_thumbnail_url = db.select_one(SEL_ION_THUMB, params=(ds_id,))
if not ion_thumbnail_url and ion_thumb_id:
transfer_images(
ds_id,
'ion_thumbnails',
image_storage.THUMB,
[ion_thumb_id],
)
ion_thumb_url = image_storage.get_image_url(image_storage.THUMB, ds_id, ion_thumb_id)
db.alter(UPD_ION_THUMB, params=(ion_thumb_url, ds_id))
def generate_ion_thumbnail_lithops(
executor: Executor,
db,
ds: Dataset,
only_if_needed=False,
algorithm=DEFAULT_ALGORITHM,
):
try:
(existing_thumb_id,) = db.select_one(THUMB_SEL, [ds.id])
if existing_thumb_id and only_if_needed:
return
annotation_rows = db.select(ISO_IMAGE_SEL, [ds.id])
if not annotation_rows:
logger.warning('Could not create ion thumbnail - no annotations found')
return
ds_id = ds.id
sm_config = SMConfig.get_conf()
def generate(annotation_rows):
return _generate_ion_thumbnail_image(
image_storage.ImageStorage(sm_config), ds_id, annotation_rows, algorithm
)
thumbnail = executor.call(
generate, (annotation_rows,), runtime_memory=2048, include_modules=['png']
)
image_id = _save_ion_thumbnail_image(ds.id, thumbnail)
image_url = image_storage.get_image_url(image_storage.THUMB, ds.id, image_id)
db.alter(THUMB_UPD, [image_id, image_url, ds.id])
if existing_thumb_id:
image_storage.delete_image(image_storage.THUMB, ds.id, existing_thumb_id)
except Exception:
logger.error('Error generating ion thumbnail image', exc_info=True)
def generate_ion_thumbnail(db, ds, only_if_needed=False, algorithm=DEFAULT_ALGORITHM):
try:
(existing_thumb_id,) = db.select_one(THUMB_SEL, [ds.id])
if existing_thumb_id and only_if_needed:
return
annotation_rows = db.select(ISO_IMAGE_SEL, [ds.id])
if not annotation_rows:
logger.warning('Could not create ion thumbnail - no annotations found')
return
thumbnail = _generate_ion_thumbnail_image(image_storage, ds.id, annotation_rows, algorithm)
image_id = _save_ion_thumbnail_image(ds.id, thumbnail)
image_url = image_storage.get_image_url(image_storage.THUMB, ds.id, image_id)
db.alter(THUMB_UPD, [image_id, image_url, ds.id])
if existing_thumb_id:
image_storage.delete_image(image_storage.THUMB, ds.id, existing_thumb_id)
except Exception:
logger.error('Error generating ion thumbnail image', exc_info=True)