def save_file(self, fn, **kw):
'''save a file in the image server and return info'''
with open(self.fullname(fn), 'rb') as src:
fr = etree.Element('file',
name=fn,
permission=str(self.permission_flag))
return blob_service.store_blob(fr, src)
def save_image(self, fn, **kw):
'''save a file in the image server and return info'''
log.debug("Store new image: " + fn + " " + str(self.image_info))
self.resource = etree.Element('image',
name=fn,
permission=str(self.permission_flag))
with open(self.fullname(fn), 'rb') as src:
self.resource = blob_service.store_blob(self.resource, src)
if 'uri' in self.resource.attrib:
self.import_files[fn] = self.resource.get('uri')
else:
log.debug("Image service could not create image %s" % fn)
raise BIXError("Image service could not create image %s" % fn)
def upload_dream3d_pipeline(uf, intags):
# analyze DREAM.3D pipeline and replace illegal operations with BisQue operations
pipeline = {}
with open(uf.localpath(), 'r') as fo:
pipeline = dream3d_to_json(fo)
uf.close()
# walk the pipeline and replace any incompatible steps with BisQue steps as well as possible
new_pipeline = { '__Header__': pipeline['__Header__'] }
new_step_id = 0
old_step_id = 0
converted_cnt = 0 # TODO: only keep up to 10 interactive params... URL gets too big otherwise
for step_id in range(old_step_id, len(pipeline)-1):
if pipeline[str(step_id)]['__Label__'] in ['Read H5EBSD File', 'Read DREAM.3D Data File']:
new_pipeline[str(new_step_id)] = { '__Label__': 'BisQueLoadTable',
'Parameters': [{'Filename': 'input.h5'}],
'__Meta__': {'module_num': str(new_step_id+1)}
}
new_step_id += 1
new_pipeline[str(new_step_id)] = pipeline[str(step_id)]
new_pipeline[str(new_step_id)]['__Meta__']['module_num'] = str(new_step_id+1)
_set_parameter(new_pipeline[str(new_step_id)], 'InputFile', 'input.h5')
new_step_id += 1
elif pipeline[str(step_id)]['__Label__'] == 'Write DREAM.3D Data File':
new_pipeline[str(new_step_id)] = pipeline[str(step_id)]
new_pipeline[str(new_step_id)]['__Meta__']['module_num'] = str(new_step_id+1)
_set_parameter(new_pipeline[str(new_step_id)], 'OutputFile', 'output.h5')
new_step_id += 1
outname = _get_parameters(pipeline[str(step_id)], 'OutputFile')[0]
new_pipeline[str(new_step_id)] = { '__Label__': 'BisQueSaveTable',
'Parameters': [{'Filename': os.path.basename(outname)}],
'__Meta__': {'module_num': str(new_step_id+1)}
}
new_step_id += 1
else:
# keep all others unchanged
# but check if some parameters could be treated as "interactive"
new_pipeline[str(new_step_id)] = pipeline[str(step_id)]
new_pipeline[str(new_step_id)]['__Meta__']['module_num'] = str(new_step_id+1)
new_parameters = []
for param in new_pipeline[str(new_step_id)]['Parameters']:
param_key, param_val = param.items()[0]
if converted_cnt < 10 and (any([param_key.lower().startswith(phrase) for phrase in ['max', 'min']]) or \
any([param_key.lower().endswith(phrase) for phrase in ['size', 'tolerance', 'value']])):
param_name = "Step %s (%s) - %s" % (step_id, new_pipeline[str(new_step_id)]['__Label__'], param_key)
try:
float(str(param_val)) # is this a number?
param_val = "@NUMPARAM|%s@%s" % (param_name, str(param_val))
converted_cnt += 1
new_parameters.append({param_key:param_val})
except ValueError:
# not a value... it may be a complex parameter (i.e., dictionary)
if isinstance(param_val, dict):
complex_val = {}
for key,val in param_val.iteritems():
try:
float(str(val)) # is this a number?
complex_val[key] = "@NUMPARAM|%s - %s@%s" % (param_name, key, str(val))
converted_cnt += 1
except ValueError:
complex_val[key] = val
new_parameters.append({param_key:complex_val})
else:
new_parameters.append({param_key:param_val})
else:
new_parameters.append({param_key:param_val})
new_pipeline[str(new_step_id)]['Parameters'] = new_parameters
new_step_id += 1
new_pipeline['__Header__']['ModuleCount'] = str(len(new_pipeline)-1)
# write modified pipeline back for ingest
ftmp = tempfile.NamedTemporaryFile(delete=False)
ftmp.write(json_to_dream3d(new_pipeline))
ftmp.close()
# ingest modified pipeline
res = []
with open(ftmp.name, 'rb') as fo:
res = [blob_service.store_blob(resource=uf.resource, fileobj=fo)]
return res
def upload_bisque_pipeline(uf, intags):
with open(uf.localpath(), 'rb') as fo:
res = [blob_service.store_blob(resource=uf.resource, fileobj=fo)]
uf.close()
return res
def upload_imagej_pipeline(uf, intags):
# analyze ImageJ macro and replace illegal operations with BisQue operations
pipeline = {}
with open(uf.localpath(), 'r') as fo:
pipeline = imagej_to_json(fo)
uf.close()
# walk the pipeline and replace any incompatible steps with BisQue steps as well as possible
new_pipeline = {'__Header__': pipeline['__Header__']}
new_step_id = 0
old_step_id = 0
converted_cnt = 0 # TODO: only keep up to 10 interactive params... URL gets too big otherwise
had_save_img = False
had_save_res = False
for step_id in range(old_step_id, len(pipeline) - 1):
new_pipeline[str(new_step_id)] = copy.deepcopy(pipeline[str(step_id)])
new_pipeline[str(new_step_id)]['__Meta__']['module_num'] = str(
new_step_id + 1)
if pipeline[str(step_id)]['__Label__'] == 'open':
new_pipeline[str(new_step_id)]['__Label__'] = 'BisQueLoadImage'
new_pipeline[str(new_step_id)]['Parameters'] = []
new_step_id += 1
elif pipeline[str(step_id)]['__Label__'] in [
'saveAs'
] and _get_parameters(
pipeline[str(step_id)],
'arg0')[0].lower() == 'tiff' and not had_save_img:
new_pipeline[str(new_step_id)]['__Label__'] = 'BisQueSaveImage'
out_name = _get_parameters(pipeline[str(step_id)], 'arg1')
new_pipeline[str(new_step_id)]['Parameters'] = [{
'arg0':
out_name[0] if len(out_name) > 0 else '"output.tif"'
}]
had_save_img = True
new_step_id += 1
elif pipeline[str(step_id)]['__Label__'] in [
'saveAs'
] and _get_parameters(
pipeline[str(step_id)],
'arg0')[0].lower() == 'results' and not had_save_res:
new_pipeline[str(new_step_id)]['__Label__'] = 'BisQueSaveResults'
out_name = _get_parameters(pipeline[str(step_id)], 'arg1')
new_pipeline[str(new_step_id)]['Parameters'] = [{
'arg0':
out_name[0] if len(out_name) > 0 else '"output.csv"'
}]
had_save_res = True
new_step_id += 1
else:
# keep all others unchanged
# but check if some parameters could be treated as "interactive"
new_pipeline[str(new_step_id)]['__Meta__']['module_num'] = str(
new_step_id + 1)
new_parameters = []
for param in new_pipeline[str(new_step_id)]['Parameters']:
param_key, param_val = param.items()[0]
if converted_cnt < 10 and any([
phrase in param_key.lower() for phrase in [
'threshold', 'size', 'diameter', 'distance',
'smoothing', 'bound', 'difference', 'intensity'
]
]):
try:
float(str(param_val)) # is this a number?
param_val = "@STRPARAM@%s" % str(param_val)
converted_cnt += 1
except ValueError:
pass # skip non-numerical parameters for now
new_parameters.append({param_key: param_val})
new_pipeline[str(new_step_id)]['Parameters'] = new_parameters
new_step_id += 1
new_pipeline['__Header__']['ModuleCount'] = str(len(new_pipeline) - 1)
# write modified pipeline back for ingest
ftmp = tempfile.NamedTemporaryFile(delete=False)
ftmp.write(json_to_imagej(new_pipeline))
ftmp.close()
# ingest modified pipeline
res = []
with open(ftmp.name, 'rb') as fo:
res = [blob_service.store_blob(resource=uf.resource, fileobj=fo)]
return res
def upload_cellprofiler_pipeline(uf, intags):
# analyze cellprofiler pipeline and replace illegal operations with BisQue operations
pipeline = {}
with open(uf.localpath(), 'r') as fo:
pipeline = cellprofiler_to_json(fo)
uf.close()
# walk the pipeline and replace any incompatible steps with BisQue steps as well as possible
# TODO: improve the quality of replacement and add BisQueExtractGObjects step generation
new_pipeline = {'__Header__': pipeline['__Header__']}
new_step_id = 0
old_step_id = 0
# TODO: handle more cases...
if pipeline['0']['__Label__'] == 'Images' and \
_get_parameters(pipeline['0'], 'Filter images?')[0] == 'Images only' and \
pipeline['1']['__Label__'] == 'Metadata' and \
pipeline['2']['__Label__'] == 'NamesAndTypes' and \
_get_parameters(pipeline['2'], 'Assign a name to')[0] == 'Images matching rules' and \
int(_get_parameters(pipeline['2'], 'Single images count')[0]) == 0 and \
int(_get_parameters(pipeline['2'], 'Assignments count')[0]) > 0 and \
pipeline['3']['__Label__'] == 'Groups' and \
_get_parameters(pipeline['3'], 'Do you want to group your images?')[0] == 'No':
assign_cnt = int(
_get_parameters(pipeline['2'], 'Assignments count')[0])
channel_id = [str(channel)
for channel in range(1, assign_cnt +
1)] if assign_cnt > 1 else ['all']
img_name = _get_parameters(pipeline['2'],
'Name to assign these images')
obj_name = _get_parameters(pipeline['2'],
'Name to assign these objects')
img_type = _get_parameters(pipeline['2'], 'Select the image type')
params = [{'Assignments count': str(assign_cnt)}]
for assign in range(assign_cnt, 0, -1):
params += [{
'Channel': channel_id[-assign]
}, {
'Name to assign these images': img_name[-assign]
}, {
'Name to assign these objects': obj_name[-assign]
}, {
'Select the image type': img_type[-assign]
}]
new_pipeline[str(new_step_id)] = {
'__Label__': 'BisQueLoadImages',
'__Meta__': {
'module_num': str(new_step_id + 1)
},
'Parameters': params
}
new_step_id += 1
old_step_id += 4
elif pipeline['0']['__Label__'] == 'LoadImages' and \
_get_parameters(pipeline['0'], 'File type to be loaded')[0] == 'individual images' and \
_get_parameters(pipeline['0'], 'Group images by metadata?')[0] == 'No':
channel_id = 'all'
img_name = _get_parameters(pipeline['0'], 'Name this loaded image')[0]
obj_name = _get_parameters(pipeline['0'], 'Name this loaded object')[0]
img_type = _get_parameters(pipeline['0'],
'Load the input as images or objects?')[0]
img_type = 'Color image' if img_type == 'Images' else 'Objects'
new_pipeline[str(new_step_id)] = {
'__Label__':
'BisQueLoadImages',
'__Meta__': {
'module_num': str(new_step_id + 1)
},
'Parameters': [{
'Channel': channel_id
}, {
'Name to assign these images': img_name
}, {
'Name to assign these objects': obj_name
}, {
'Select the image type': img_type
}]
}
new_step_id += 1
old_step_id += 1
# now the rest of the pipeline...
geo_objects = []
colors = ['#ff0000', '#00ff00', '#0000ff', '#ffff00', '#ff00ff', '#00ffff']
color_idx = 0
converted_cnt = 0 # TODO: only keep up to 10 interactive params... URL gets too big otherwise
for step_id in range(old_step_id, len(pipeline) - 1):
if pipeline[str(step_id)]['__Label__'] == 'SaveImages' and \
_get_parameters(pipeline[str(step_id)], 'Select the type of image to save')[0] == 'Image':
img_name = _get_parameters(pipeline[str(step_id)],
'Select the image to save')[0]
bit_depth = _get_parameters(pipeline[str(step_id)],
'Image bit depth')[0]
img_type = _get_parameters(pipeline[str(step_id)],
'Save as grayscale or color image?')[0]
colormap = _get_parameters(pipeline[str(step_id)],
'Select colormap')[0]
filename = _get_parameters(pipeline[str(step_id)],
'Enter single file name')[0]
new_pipeline[str(new_step_id)] = {
'__Label__':
'BisQueSaveImage',
'__Meta__': {
'module_num': str(new_step_id + 1)
},
'Parameters': [{
'Select the image to save': img_name
}, {
'Image bit depth': bit_depth
}, {
'Save as grayscale or color image?': img_type
}, {
'Select colormap': colormap
}, {
'Name': filename
}]
}
new_step_id += 1
elif pipeline[str(step_id)]['__Label__'] == 'ExportToSpreadsheet' and \
all([comb == 'No' for comb in _get_parameters(pipeline[str(step_id)], 'Combine these object measurements with those of the previous object?')]):
data_exports = _get_parameters(pipeline[str(step_id)],
'Data to export')
params = []
for data_export in data_exports:
params += [{
'Data to export': data_export
}, {
'Name': data_export
}]
new_pipeline[str(new_step_id)] = {
'__Label__': 'BisQueSaveTables',
'__Meta__': {
'module_num': str(new_step_id + 1)
},
'Parameters': params
}
new_step_id += 1
else:
# keep all others unchanged
# but check if some parameters could be treated as "interactive"
new_pipeline[str(new_step_id)] = pipeline[str(step_id)]
new_pipeline[str(new_step_id)]['__Meta__']['module_num'] = str(
new_step_id + 1)
new_parameters = []
for param in new_pipeline[str(new_step_id)]['Parameters']:
param_key, param_val = param.items()[0]
if converted_cnt < 10 and any([
phrase in param_key.lower() for phrase in [
'threshold', 'size', 'diameter', 'distance',
'smoothing', 'bound', 'difference', 'intensity'
]
]):
try:
float(str(param_val)) # is this a number?
param_val = "@STRPARAM@%s" % str(param_val)
converted_cnt += 1
except ValueError:
pass # skip non-numerical parameters for now
new_parameters.append({param_key: param_val})
new_pipeline[str(new_step_id)]['Parameters'] = new_parameters
new_step_id += 1
if pipeline[str(step_id)]['__Label__'] == 'IdentifyPrimaryObjects':
obj_name = _get_parameters(
pipeline[str(step_id)],
'Name the primary objects to be identified')[0]
geo_objects.append({
'name': obj_name,
'color': colors[color_idx % len(colors)]
})
color_idx += 1
elif pipeline[str(step_id)]['__Label__'] == 'IdentifySecondaryObjects':
obj_name = _get_parameters(pipeline[str(step_id)],
'Name the objects to be identified')[0]
geo_objects.append({
'name': obj_name,
'color': colors[color_idx % len(colors)]
})
color_idx += 1
elif pipeline[str(step_id)]['__Label__'] == 'IdentifyTertiaryObjects':
obj_name = _get_parameters(
pipeline[str(step_id)],
'Name the tertiary objects to be identified')[0]
geo_objects.append({
'name': obj_name,
'color': colors[color_idx % len(colors)]
})
color_idx += 1
# if there were any "IdentifyXXXObjects", add BisQueExportGObjects step(s) at the end
for geo_object in geo_objects:
new_pipeline[str(new_step_id)] = {
'__Label__':
'BisQueExtractGObjects',
'__Meta__': {
'module_num': str(new_step_id + 1)
},
'Parameters': [{
'Data to extract': geo_object['name']
}, {
'GObject type': 'Ellipse'
}, {
'GObject label': geo_object['name']
}, {
'GObject color': geo_object['color']
}, {
'XCoord column': 'AreaShape_Center_X'
}, {
'YCoord column': 'AreaShape_Center_Y'
}, {
'Orientation column': 'AreaShape_Orientation'
}, {
'MajorAxis column': 'AreaShape_MajorAxisLength'
}, {
'MinorAxis column': 'AreaShape_MinorAxisLength'
}]
}
new_step_id += 1
new_pipeline['__Header__']['ModuleCount'] = str(len(new_pipeline) - 1)
# write modified pipeline back for ingest
ftmp = tempfile.NamedTemporaryFile(delete=False)
ftmp.write(json_to_cellprofiler(new_pipeline))
ftmp.close()
# ingest modified pipeline
res = []
with open(ftmp.name, 'rb') as fo:
res = [blob_service.store_blob(resource=uf.resource, fileobj=fo)]
return res