class BotanicamTrainer(object):
"""
Botanicam Trainer Module
"""
def mex_parameter_parser(self, mex_xml):
"""
Parses input of the xml and add it to BotanicamTrainer's options attribute
@param: mex_xml
"""
mex_inputs = mex_xml.xpath('tag[@name="inputs"]')
if mex_inputs:
for tag in mex_inputs[0]:
if tag.tag == 'tag' and tag.attrib[
'type'] != 'system-input': #skip system input values
log.debug('Set options with %s as %s' %
(tag.attrib['name'], tag.attrib['value']))
setattr(self.options, tag.attrib['name'],
tag.attrib['value'])
else:
log.debug('BotanicamFS: No Inputs Found on MEX!')
def validate_input(self):
"""
Check to see if a mex with token or user with password was provided.
@return True is returned if validation credention was provided else
False is returned
"""
if (self.options.mexURL
and self.options.token): #run module through engine service
return True
if (self.options.user and self.options.pwd and
self.options.root): #run module locally (note: to test module)
return True
log.debug(
'BotanicamTrainer: Insufficient options or arguments to start this module'
)
return False
def setup(self):
"""
Fetches the mex and appends input_configurations to the option
attribute of BotanicamTrainer
"""
#initalizes if user and password are provided
if (self.options.user and self.options.pwd and self.options.root):
self.bqSession = BQSession().init_local(
self.options.user,
self.options.pwd,
bisque_root=self.options.root)
self.options.mexURL = self.bqSession.mex.uri
#initalizes if mex and mex token is provided
elif (self.options.mexURL and self.options.token):
self.bqSession = BQSession().init_mex(self.options.mexURL,
self.options.token)
else:
raise BotanicamTrainerError(
'BotanicamTrainer: Insufficient options or arguments to start this module'
)
self.bqSession.update_mex('Initializing...')
self.mex_parameter_parser(self.bqSession.mex.xmltree)
return
def run(self):
"""
The core of the Botanicam Trainer
Parses the tags and find all the corresponding values to from classes.
Classes that have no images in them are removed. Features are requested
on all the images and then trained using the tag value classes. The
resulting model file is stored on bisque as a zip file.
"""
#retieve tags
self.bqSession.update_mex('Parse Tags...')
if not self.options.tag_names:
raise BotanicamTrainerError('Tags are a required input!')
self.tag_names = self.options.tag_names.split(',')
#type check
resource_short = self.bqSession.fetchxml(self.options.resource_url,
view='short')
if resource_short.tag == 'dataset':
resource_url_values = '%s/value' % self.options.resource_url
else:
resource_url_values = self.options.resource_url
all_images = self.bqSession.fetchxml(resource_url_values,
view='full,clean')
tag_query_list = []
for name in self.tag_names:
name_list = []
for u in np.unique(
all_images.xpath('image/tag[@name="%s"]/@value' % name)):
name_list.append('"%s":"%s"' % (name, u))
tag_query_list.append(name_list)
#need to find the unique values to create lists of images
#hopefully the tag names and lists are not too complicated
tag_query_list = [
list(element) for element in itertools.product(*tag_query_list)
] #cartesian product
self.complete_tag_list = []
tag_query_url_list = []
#search for classes with images
#query all the values to see if images return
#removing query_tag from the resource_url and adding it back in later
resource_url_wo_query = resource_url_values
resource_query = None
from urlparse import urlsplit, urlunsplit, parse_qs
from urllib import urlencode
o = urlsplit(resource_url_values)
q = parse_qs(o.query)
if q.get('tag_query'):
resource_query = q['tag_query']
del q['tag_query']
query = urlencode(q)
resource_url_wo_query = urlunsplit(
(o.scheme, o.netloc, o.path, query, o.fragment))
log.debug(tag_query_list)
for tag_query in tag_query_list:
encoded_tag_query = tag_query
if resource_query:
encoded_tag_query += resource_query #adding back in the query_tag from the resource_url
encoded_tag_query = map(urllib.quote, tag_query)
encoded_tag_query = '%s' % ' AND '.join(encoded_tag_query)
query_xml = self.bqSession.fetchxml(resource_url_wo_query,
tag_query=encoded_tag_query,
view='full,clean')
if len(query_xml.xpath('image')) > 0:
name_value_pairs = {}
for t in tag_query: #create dictionary of clases with list of name value pairs
m = re.match('"(?P<name>[^"]+)":"(?P<value>[^"]+)"', t)
name_value_pairs[m.group('name')] = m.group('value')
self.complete_tag_list.append(name_value_pairs)
tag_query_url_list.append(query_xml.attrib['uri'])
feature_length = Feature().length(self.bqSession, FEATURE_NAME)
if len(tag_query_url_list) < 2:
raise BotanicamTrainerError(
'Requires atleast 2 classes to train found %s' %
len(tag_query_url_list))
#extracts all the features and then appends it to a larger table
_mkdir(os.path.join(self.options.stagingPath, FEATURE_TABLE_DIR))
main_table = os.path.join(self.options.stagingPath, FEATURE_TABLE_DIR,
'feature_table.h5')
with tables.open_file(main_table, 'w') as h5file:
columns = {
'label': tables.Int64Col(),
'feature': tables.Float32Col(shape=(feature_length))
}
table = h5file.create_table('/', 'values', columns)
table.flush()
self.bqSession.update_mex('Calculated features on (0/%s) spieces...' %
(len(tag_query_url_list)))
for i, tag_url_query in enumerate(tag_query_url_list):
try:
vectors = extract_bush_feature(
self.bqSession,
tag_url_query) #may need to be moved into local temp
feature_table = vectors.root.values
with tables.open_file(main_table, 'a') as h5file:
table = h5file.root.values
r = table.row
for fr in feature_table:
r['feature'] = fr['feature']
r['label'] = i
r.append()
table.flush()
vectors.close()
os.remove(vectors.filename)
except FeatureError as e:
raise BotanicamTrainerError(str(e))
self.bqSession.update_mex(
'Calculated features on (%s/%s) spieces...' %
(i + 1, len(tag_query_url_list)))
self.bqSession.update_mex('Classifying')
log.debug('Training model')
#classify the features
pca = RandomizedPCA(whiten=True)
clf = svm.SVC()
with tables.open_file(main_table, 'r') as h5file:
table = h5file.root.values
pca.fit(table[:]['feature'])
clf.fit(pca.transform(table[:]['feature']), table[:]['label'])
self.bqSession.update_mex('Posting model to bisque...')
log.debug('Storing and Zipping model')
self.model_dir = os.path.join(self.options.stagingPath, MODEL_DIR)
self.svm_model_file = os.path.join(self.options.stagingPath, MODEL_DIR,
'svm_model')
self.pca_model_file = os.path.join(self.options.stagingPath, MODEL_DIR,
'pca_model')
_mkdir(self.model_dir)
svm_files = joblib.dump(clf, self.svm_model_file)
pca_files = joblib.dump(pca, self.pca_model_file)
#zip file
import zipfile
with zipfile.ZipFile('%s.zip' % self.model_dir, 'w') as fzip:
for f in svm_files:
fzip.write(f, os.path.basename(f))
for f in pca_files:
fzip.write(f, os.path.basename(f))
def teardown(self):
"""
Post the results to the mex xml.
"""
#save the model and upload it to the data service with all the meta data
self.bqSession.update_mex('Returning results...')
#self.bqSession.update_mex('Returning home after a long day...')
#constructing and storing model file
#does not accept no name on the resource
cl_model = etree.Element('resource',
name=os.path.basename('%s.zip' % MODEL_DIR))
#classes
tag_classes = etree.SubElement(cl_model, 'tag', name='Classes')
for i, name_value_pairs in enumerate(self.complete_tag_list):
tag_labels = etree.SubElement(tag_classes,
'tag',
name='Class_%s' % str(i),
value=str(i))
for n in name_value_pairs.keys():
etree.SubElement(tag_labels,
'tag',
name=n,
value=name_value_pairs[n])
etree.SubElement(cl_model,
'tag',
name='Number of Classes',
value=str(len(self.complete_tag_list)))
#module identifier (a descriptor to be found by the botanicam model)
etree.SubElement(cl_model,
'tag',
name='module_identifier',
value='Botanicam')
#model filename
etree.SubElement(cl_model,
'tag',
name='filename',
value=os.path.basename('%s.zip' % MODEL_DIR))
#classification method
etree.SubElement(cl_model,
'tag',
name='Classification Method',
value='Bush Descriptor')
#Resource url
etree.SubElement(cl_model,
'tag',
name='Resource',
value=self.options.resource_url)
#description
etree.SubElement(cl_model,
'tag',
name='description',
value='Model File for the Botanicam Module')
#storing the model file in blobservice
r = self.bqSession.postblob('%s.zip' % self.model_dir, xml=cl_model)
r_xml = etree.fromstring(r)
outputTag = etree.Element('tag', name='outputs')
etree.SubElement(outputTag,
'tag',
name='classification_file',
value=r_xml[0].attrib['uri'],
type='model_viewer')
self.bqSession.finish_mex(tags=[outputTag])
self.bqSession.close()
def main(self):
"""
The main function that runs everything
"""
log.debug('sysargv : %s' % sys.argv)
parser = OptionParser()
parser.add_option('--resource_url',
dest="resource_url") #for now only datasets
parser.add_option('--Tags', dest="tag_names") #labels on the images
parser.add_option('--ClassifierMethod', dest="classifier_methods")
parser.add_option('--FeatureName', dest="feature_name")
parser.add_option('--mex_url', dest="mexURL")
parser.add_option('--module_dir', dest="modulePath")
parser.add_option('--staging_path', dest="stagingPath")
parser.add_option('--bisque_token', dest="token")
parser.add_option('--user', dest="user")
parser.add_option('--pwd', dest="pwd")
parser.add_option('--root', dest="root")
(options, args) = parser.parse_args()
try: #pull out the mex
if not options.mexURL:
options.mexURL = sys.argv[1]
if not options.token:
options.token = sys.argv[2]
except IndexError: #no argv were set
pass
if not options.stagingPath:
options.stagingPath = ''
log.debug('\n\nPARAMS : %s \n\n Options: %s' % (args, options))
self.options = options
if self.validate_input():
try:
self.setup()
except Exception as e:
log.exception("Exception during setup")
self.bqSession.fail_mex(msg="Exception during setup: %s" %
str(e))
return
try:
self.run()
except (Exception, BotanicamTrainerError) as e:
log.exception("Exception during run")
self.bqSession.fail_mex(msg="Exception during run: %s" %
str(e))
return
try:
self.teardown()
except (Exception, BotanicamTrainerError) as e:
log.exception("Exception during teardown %s")
self.bqSession.fail_mex(msg="Exception during teardown: %s" %
str(e))
return
class PythonScriptWrapper(object):
def run(self):
"""
Run Python script
"""
session = self.bqSession
# call script
outputs = run_script( session, log, **self.options.__dict__ )
# save output back to BisQue
for output in outputs:
self.output_resources.append(output)
def setup(self):
"""
Pre-run initialization
"""
self.bqSession.update_mex('Initializing...')
self.mex_parameter_parser(self.bqSession.mex.xmltree)
self.output_resources = []
def teardown(self):
"""
Post the results to the mex xml
"""
self.bqSession.update_mex( 'Returning results')
outputTag = etree.Element('tag', name ='outputs')
for r_xml in self.output_resources:
if isinstance(r_xml, basestring):
r_xml = etree.fromstring(r_xml)
#res_type = r_xml.get('type', None) or r_xml.get('resource_type', None) or r_xml.tag
# append reference to output
outputTag.append(r_xml)
self.bqSession.finish_mex(tags=[outputTag])
def mex_parameter_parser(self, mex_xml):
"""
Parses input of the xml and add it to options attribute (unless already set)
@param: mex_xml
"""
# inputs are all non-"script_params" under "inputs" and all params under "script_params"
mex_inputs = mex_xml.xpath('tag[@name="inputs"]/tag[@name!="script_params"] | tag[@name="inputs"]/tag[@name="script_params"]/tag')
if mex_inputs:
for tag in mex_inputs:
if tag.tag == 'tag' and tag.get('type', '') != 'system-input': #skip system input values
if not getattr(self.options,tag.get('name', ''), None):
log.debug('Set options with %s as %s'%(tag.get('name',''),tag.get('value','')))
setattr(self.options,tag.get('name',''),tag.get('value',''))
else:
log.debug('No Inputs Found on MEX!')
def validate_input(self):
"""
Check to see if a mex with token or user with password was provided.
@return True is returned if validation credention was provided else
False is returned
"""
if (self.options.mexURL and self.options.token): #run module through engine service
return True
if (self.options.user and self.options.pwd and self.options.root): #run module locally (note: to test module)
return True
log.debug('Insufficient options or arguments to start this module')
return False
def collect_outputs(self):
"""
Perform reduce phase (i.e., examine final (top) mex and create any additional outputs based on submexes)
"""
top_mex = self.bqSession.fetchxml(self.options.mexURL, view='deep')
# add output tag if needed
outputTag = top_mex.xpath('/mex/tag[@name="outputs"]')
if not outputTag:
# no "outputs" tag in mex => add it now
etree.SubElement(top_mex, 'tag', name='outputs')
top_mex = self.bqSession.postxml(url=top_mex.get('uri'), xml=top_mex, view='deep')
outputTag = top_mex.xpath('/mex/tag[@name="outputs"]')
outtag = outputTag[0]
# add multi-param output table based on submex outputs
# from inputs, collect everything except mex_url and bisque_token
# from outputs/statistics, collect everything except "meanMaxMisorient"
multiparam_name = 'output_table'
multitag = etree.SubElement(outtag, 'tag', name=multiparam_name, type='multiparam')
colnames = etree.SubElement(multitag, 'tag', name='title')
coltypes = etree.SubElement(multitag, 'tag', name='xmap')
colxpaths = etree.SubElement(multitag, 'tag', name='xpath')
etree.SubElement(multitag, 'tag', name='xreduce', value='vector')
inputs = top_mex.xpath('/mex/mex/tag[@name="inputs"]')[0]
for inp in inputs.xpath('./tag'):
if inp.get('name') not in ['mex_url', 'bisque_token']:
etree.SubElement(colnames, 'value', value=inp.get('name'))
etree.SubElement(coltypes, 'value', value=self._get_type(inp))
etree.SubElement(colxpaths, 'value', value='/mex/mex/tag[@name="inputs"]/tag[@name="%s"]' % inp.get('name'))
outputs = top_mex.xpath('/mex/mex/tag[@name="outputs"]')[0]
for outp in outputs.xpath('./tag[@name="statistics"]/tag'):
if outp.get('name') not in ['meanMaxMisorient']:
etree.SubElement(colnames, 'value', value=outp.get('name'))
etree.SubElement(coltypes, 'value', value=self._get_type(outp))
etree.SubElement(colxpaths, 'value', value='/mex/mex/tag[@name="outputs"]/tag[@name="statistics"]/tag[@name="%s"]' % outp.get('name'))
# last column is the submex URI
etree.SubElement(colnames, 'value', value="submex_uri")
etree.SubElement(coltypes, 'value', value="resource-uri")
etree.SubElement(colxpaths, 'value', value='./mex')
# write back to mex
self.bqSession.postxml(url=outtag.get('uri'), xml=outtag)
def _get_type(self, inp):
if inp.get('name') == 'dream3d_run':
return 'resource-uri'
else:
inptype = inp.get('type', 'string')
if inptype == 'combo':
inptype = 'string'
return "tag-value-%s" % inptype
def main(self):
parser = optparse.OptionParser()
parser.add_option('--mex_url' , dest="mexURL")
parser.add_option('--module_dir' , dest="modulePath")
parser.add_option('--staging_path' , dest="stagingPath")
parser.add_option('--bisque_token' , dest="token")
parser.add_option('--user' , dest="user")
parser.add_option('--pwd' , dest="pwd")
parser.add_option('--root' , dest="root")
# for the reduce phase: create output dataset of output HDFs (in this case, mexURL is top mex)
parser.add_option('--entrypoint' , dest="entrypoint")
(options, args) = parser.parse_args()
fh = logging.FileHandler('phase_%s.log' % (options.entrypoint or 'scriptrun.log'), mode='a')
fh.setLevel(logging.DEBUG)
formatter = logging.Formatter('[%(asctime)s] %(levelname)8s --- %(message)s ' +
'(%(filename)s:%(lineno)s)',datefmt='%Y-%m-%d %H:%M:%S')
fh.setFormatter(formatter)
log.addHandler(fh)
try: #pull out the mex
if not options.mexURL:
options.mexURL = sys.argv[1]
if not options.token:
options.token = sys.argv[2]
except IndexError: #no argv were set
pass
if not options.stagingPath:
options.stagingPath = ''
log.info('\n\nPARAMS : %s \n\n Options: %s' % (args, options))
self.options = options
if self.validate_input():
#initalizes if user and password are provided
if (self.options.user and self.options.pwd and self.options.root):
self.bqSession = BQSession().init_local( self.options.user, self.options.pwd, bisque_root=self.options.root)
self.options.mexURL = self.bqSession.mex.uri
#initalizes if mex and mex token is provided
elif (self.options.mexURL and self.options.token):
self.bqSession = BQSession().init_mex(self.options.mexURL, self.options.token)
else:
raise ScriptError('Insufficient options or arguments to start this module')
if not self.options.entrypoint:
# NOT a reduce phase => perform regular run processing
try:
self.setup()
except Exception as e:
log.exception("Exception during setup")
self.bqSession.fail_mex(msg = "Exception during setup: %s" % str(e))
return
try:
self.run()
except (Exception, ScriptError) as e:
log.exception("Exception during run")
self.bqSession.fail_mex(msg = "Exception during run: %s" % str(e))
return
try:
self.teardown()
except (Exception, ScriptError) as e:
log.exception("Exception during teardown")
self.bqSession.fail_mex(msg = "Exception during teardown: %s" % str(e))
return
else:
# in a reduce phase => run reduce code
if self.options.entrypoint != 'collect_outputs':
self.bqSession.fail_mex(msg = "Unknown entrypoint: %s" % self.options.entrypoint)
return
try:
self.collect_outputs()
except (Exception, ScriptError) as e:
log.exception("Exception during collect_outputs")
self.bqSession.fail_mex(msg = "Exception during collect_outputs: %s" % str(e))
return
self.bqSession.close()
class Dream3D(object):
"""
Dream3D Module
"""
def mex_parameter_parser(self, mex_xml):
"""
Parses input of the xml and add it to Dream3D's options attribute (unless already set)
@param: mex_xml
"""
# inputs are all non-"pipeline params" under "inputs" and all params under "pipeline_params"
mex_inputs = mex_xml.xpath(
'tag[@name="inputs"]/tag[@name!="pipeline_params"] | tag[@name="inputs"]/tag[@name="pipeline_params"]/tag'
)
if mex_inputs:
for tag in mex_inputs:
if tag.tag == 'tag' and tag.get(
'type',
'') != 'system-input': #skip system input values
if not getattr(self.options, tag.get('name', ''), None):
log.debug('Set options with %s as %s' %
(tag.get('name', ''), tag.get('value', '')))
setattr(self.options, tag.get('name', ''),
tag.get('value', ''))
else:
log.debug('Dream3D: No Inputs Found on MEX!')
def validate_input(self):
"""
Check to see if a mex with token or user with password was provided.
@return True is returned if validation credention was provided else
False is returned
"""
if (self.options.mexURL
and self.options.token): #run module through engine service
return True
if (self.options.user and self.options.pwd and
self.options.root): #run module locally (note: to test module)
return True
log.debug(
'Dream3D: Insufficient options or arguments to start this module')
return False
def setup(self):
"""
Fetches the mex and appends input_configurations to the option
attribute of Dream3D
"""
self.bqSession.update_mex('Initializing...')
self.mex_parameter_parser(self.bqSession.mex.xmltree)
self.output_resources = []
self.ppops = None
self.ppops_url = None
def run(self):
"""
The core of the Dream3D runner
"""
module_time = datetime.now()
#retrieve tags
self.bqSession.update_mex('Extracting properties')
#type check
hdf_resource = self.bqSession.fetchxml(self.options.InputFile,
view='deep,clean')
if (hdf_resource.tag != 'resource'
or hdf_resource.get('resource_type', '') != 'table'
) and hdf_resource.tag != 'table':
raise Dream3DError("trying to run Dream3D on non-table resource")
# run prerun operations
filelist_file = self._run_prerun_ops(
module_time,
pipeline_url=self.options.pipeline_url,
input_xml=hdf_resource)
# create pipeline with correct parameters
pipeline_params = self.bqSession.mex.xmltree.xpath(
'tag[@name="inputs"]/tag[@name="pipeline_params"]/tag')
params = {}
for tag in pipeline_params:
params[tag.get('name', '')] = getattr(self.options,
tag.get('name', ''))
pipeline_file, err_file = self._instantiate_pipeline(
pipeline_url=self.options.pipeline_url, params=params)
if not pipeline_file:
raise Dream3DError("trying to run incompatible Dream.3D pipeline")
# run Dream3D on the pipeline
self.bqSession.update_mex('Running Dream3D')
log.debug('run Dream3D on %s', pipeline_file)
res = 1
with open(err_file, 'w') as fo:
res = subprocess.call(
['/dream3d/bin/PipelineRunner', '-p', pipeline_file],
stderr=fo,
stdout=fo)
log.debug("Dream3D returned: %s", str(res))
if res > 0:
err_msg = 'pipeline execution failed\n'
with open(err_file, 'r') as fo:
err_msg += ''.join(fo.readlines())
if len(err_msg) > 1024:
err_msg = err_msg[:512] + '...' + err_msg[-512:]
raise Dream3DError(err_msg)
# run postrun operations
self.output_resources = self._run_postrun_ops(
module_time, pipeline_url=self.options.pipeline_url)
def _cache_ppops(self, pipeline_url):
if not self.ppops or self.ppops_url != pipeline_url:
pipeline_path = urlparse.urlsplit(pipeline_url).path.split('/')
pipeline_uid = pipeline_path[1] if is_uniq_code(
pipeline_path[1]) else pipeline_path[2]
url = self.bqSession.service_url('pipeline',
path='/'.join([pipeline_uid] +
['ppops:dream3d']))
self.ppops = json.loads(self.bqSession.c.fetch(url))
self.ppops_url = pipeline_url
def _run_prerun_ops(self, module_time, pipeline_url, input_xml):
"""
Perform any operations necessary before the pipeline runs (e.g., download input table) and return filelist file
"""
self._cache_ppops(pipeline_url)
post_ops = self.ppops['PreOps']
input_files = []
for op in post_ops:
input_files += self._run_single_op(module_time, op, input_xml)
filelist_file = os.path.join(self.options.stagingPath, 'filelist.txt')
with open(filelist_file, 'w') as fo:
for input_file in input_files:
fo.write(input_file + '\n')
return filelist_file
def _run_postrun_ops(self, module_time, pipeline_url):
"""
Perform any operations necessary after the pipeline finished (e.g., upload result tables) and return created resources
"""
self._cache_ppops(pipeline_url)
post_ops = self.ppops['PostOps']
created_resources = []
for op in post_ops:
created_resources += self._run_single_op(module_time, op)
return created_resources
def _run_single_op(self, module_time, op, input_xml=None):
"""
Perform single pre/post operation and return list of files or resources generated
"""
# replace special placeholders
if 'id' in op and op['id'] == '@INPUT':
op['id'] = input_xml.get('resource_uniq')
res = []
if op['service'] == 'postblob':
# upload image or table (check op['type'])
mex_id = self.bqSession.mex.uri.split('/')[-1]
dt = module_time.strftime('%Y%m%dT%H%M%S')
final_output_file = "ModuleExecutions/Dream3D/%s_%s_%s.h5" % (
self.options.OutputPrefix, dt, mex_id)
cl_model = etree.Element('resource',
resource_type=op['type'],
name=final_output_file)
# module identifier (a descriptor to be found by the Dream3D model)
etree.SubElement(cl_model,
'tag',
name='module_identifier',
value='Dream3D')
# hdf filename
etree.SubElement(cl_model,
'tag',
name='OutputFile',
value=final_output_file)
#description
etree.SubElement(cl_model,
'tag',
name='description',
value='output from Dream3D Module')
# post blob
output_file = os.path.join(self.options.stagingPath,
op['filename'])
resource = self.bqSession.postblob(output_file, xml=cl_model)
resource_xml = etree.fromstring(resource)
res += [resource_xml[0]]
elif op['service'] == 'getblob':
# download table
table_file = os.path.join(self.options.stagingPath, op['filename'])
hdf_url = self.bqSession.service_url('blob_service', path=op['id'])
self.bqSession.fetchblob(hdf_url, path=table_file)
res += [table_file]
return res
def _instantiate_pipeline(self, pipeline_url, params):
"""
instantiate dream.3d pipeline file with provided parameters
"""
pipeline_path = urlparse.urlsplit(pipeline_url).path.split('/')
pipeline_uid = pipeline_path[1] if is_uniq_code(
pipeline_path[1]) else pipeline_path[2]
url = self.bqSession.service_url(
'pipeline',
path='/'.join(
[pipeline_uid] +
["setvar:%s|%s" % (tag, params[tag])
for tag in params] + ['exbsteps:dream3d']),
query={'format': 'dream3d'})
pipeline = self.bqSession.c.fetch(url)
if not pipeline:
# bad pipeline
return None, None
out_pipeline_file = os.path.join(self.options.stagingPath,
'pipeline.json')
out_error_file = os.path.join(self.options.stagingPath,
'dream3d_error.txt')
with open(out_pipeline_file, 'w') as fo:
fo.write(pipeline)
return out_pipeline_file, out_error_file
def teardown(self):
"""
Post the results to the mex xml.
"""
self.bqSession.update_mex('Returning results')
outputTag = etree.Element('tag', name='outputs')
for r_xml in self.output_resources:
res_type = r_xml.get('type', None) or r_xml.get(
'resource_type', None) or r_xml.tag
if res_type == 'detected shapes':
# r_xml is a set of gobjects => append to output inside image tag
image_resource = self.bqSession.fetchxml(
self.options.InputFile)
image_elem = etree.SubElement(outputTag,
'tag',
name=image_resource.get('name'),
type='image',
value=image_resource.get('uri'))
image_elem.append(r_xml)
else:
# r_xml is some other resource (e.g., image or table) => append reference to output
etree.SubElement(outputTag,
'tag',
name='output_table'
if res_type == 'table' else 'output_image',
type=res_type,
value=r_xml.get('uri', ''))
self.bqSession.finish_mex(tags=[outputTag])
def collect_outputs(self):
"""
Perform reduce phase (i.e., examine final (top) mex and create any additional outputs based on submexes)
THIS IS JUST AN EXAMPLE.
"""
# collect submex output hdf urls and add them to top mex outputs section
top_mex = self.bqSession.fetchxml(self.options.mexURL, view='deep')
outputTag = top_mex.xpath('/mex/tag[@name="outputs"]')
if not outputTag:
# no "outputs" tag in mex => add it now
etree.SubElement(top_mex, 'tag', name='outputs')
top_mex = self.bqSession.postxml(url=top_mex.get('uri'),
xml=top_mex,
view='deep')
outputTag = top_mex.xpath('/mex/tag[@name="outputs"]')
outputTag = outputTag[0]
output_hdfs = top_mex.xpath(
'/mex/mex/tag[@name="outputs"]/tag[@name="output_hdf"]/@value')
etree.SubElement(outputTag,
'tag',
name='all_outputs',
value=';'.join(
[ohdf.split('/')[-1] for ohdf in output_hdfs]))
self.bqSession.postxml(url=outputTag.get('uri'), xml=outputTag)
def main(self):
"""
The main function that runs everything
"""
log.info('sysargv : %s' % sys.argv)
parser = OptionParser()
parser.add_option('--mex_url', dest="mexURL")
parser.add_option('--module_dir', dest="modulePath")
parser.add_option('--staging_path', dest="stagingPath")
parser.add_option('--bisque_token', dest="token")
parser.add_option('--user', dest="user")
parser.add_option('--pwd', dest="pwd")
parser.add_option('--root', dest="root")
# for the reduce phase: create output dataset of output HDFs (in this case, mexURL is top mex)
parser.add_option('--entrypoint', dest="entrypoint")
(options, args) = parser.parse_args()
fh = logging.FileHandler('phase_%s.log' %
(options.entrypoint or 'main'),
mode='a')
fh.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'[%(asctime)s] %(levelname)8s --- %(message)s ' +
'(%(filename)s:%(lineno)s)',
datefmt='%Y-%m-%d %H:%M:%S')
fh.setFormatter(formatter)
log.addHandler(fh)
try: #pull out the mex
if not options.mexURL:
options.mexURL = sys.argv[1]
if not options.token:
options.token = sys.argv[2]
except IndexError: #no argv were set
pass
if not options.stagingPath:
options.stagingPath = ''
log.info('\n\nPARAMS : %s \n\n Options: %s' % (args, options))
self.options = options
if self.validate_input():
#initalizes if user and password are provided
if (self.options.user and self.options.pwd and self.options.root):
self.bqSession = BQSession().init_local(
self.options.user,
self.options.pwd,
bisque_root=self.options.root)
self.options.mexURL = self.bqSession.mex.uri
#initalizes if mex and mex token is provided
elif (self.options.mexURL and self.options.token):
self.bqSession = BQSession().init_mex(self.options.mexURL,
self.options.token)
else:
raise Dream3DError(
'Insufficient options or arguments to start this module')
if not self.options.entrypoint:
# NOT a reduce phase => perform regular run processing
try:
self.setup()
except Exception as e:
log.exception("Exception during setup")
self.bqSession.fail_mex(msg="Exception during setup: %s" %
str(e))
return
try:
self.run()
except (Exception, Dream3DError) as e:
log.exception("Exception during run")
self.bqSession.fail_mex(msg="Exception during run: %s" %
str(e))
return
try:
self.teardown()
except (Exception, Dream3DError) as e:
log.exception("Exception during teardown")
self.bqSession.fail_mex(
msg="Exception during teardown: %s" % str(e))
return
else:
# in a reduce phase => run reduce code
if self.options.entrypoint != 'collect_outputs':
self.bqSession.fail_mex(
msg="Unknown Dream3D entrypoint: %s" %
self.options.entrypoint)
return
try:
self.collect_outputs()
except (Exception, Dream3DError) as e:
log.exception("Exception during collect_outputs")
self.bqSession.fail_mex(
msg="Exception during collect_outputs: %s" % str(e))
return
self.bqSession.close()
class WorkflowRunner(object):
def mex_parameter_parser(self, mex_xml):
"""
Parses input of the xml and add it to options attribute (unless already set)
@param: mex_xml
"""
# inputs are all non-"pipeline params" under "inputs" and all params under "pipeline_params"
mex_inputs = mex_xml.xpath(
'tag[@name="inputs"]/tag[@name!="workflow_parameters"] | tag[@name="inputs"]/tag[@name="workflow_parameters"]/tag'
)
if mex_inputs:
for tag in mex_inputs:
if tag.tag == 'tag' and tag.get(
'type',
'') != 'system-input': #skip system input values
if not getattr(self.options, tag.get('name', ''), None):
log.debug('Set options with %s as %s' %
(tag.get('name', ''), tag.get('value', '')))
setattr(self.options, tag.get('name', ''),
tag.get('value', ''))
else:
log.debug('WorkflowRunner: No Inputs Found on MEX!')
def validate_input(self):
"""
Check to see if a mex with token or user with password was provided.
@return True is returned if validation credention was provided else
False is returned
"""
if (self.options.mexURL
and self.options.token): #run module through engine service
return True
if (self.options.user and self.options.pwd and
self.options.root): #run module locally (note: to test module)
return True
log.debug(
'WorkflowRunner: Insufficient options or arguments to start this module'
)
return False
def setup(self):
"""
Fetches the mex and appends input_configurations to the option
attribute
"""
self.bqSession.update_mex('Initializing...')
self.mex_parameter_parser(self.bqSession.mex.xmltree)
self.output_resources = []
def run(self):
# retrieve tags
self.bqSession.update_mex('Extracting properties')
# set up initial parameters
pipeline_params = self.bqSession.mex.xmltree.xpath(
'tag[@name="inputs"]/tag[@name="workflow_parameters"]/tag')
self.global_vars = {'server_url': self.bqSession.c.root}
for tag in pipeline_params:
self.global_vars[tag.get('name',
'')] = getattr(self.options,
tag.get('name', ''))
pipeline = self._read_pipeline(pipeline_url=self.options.pipeline_url)
if not pipeline:
raise WFError("trying to run incompatible workflow")
# want error notification?
error_mail = pipeline['__Header__'].get('Error_mail')
try:
# Run the workflow
self.output_resources = [
ET.Element('tag',
name='initial state',
value=cgi.escape(str(self.global_vars)))
]
for step_id in xrange(len(pipeline) - 1):
curr_step = pipeline.get(str(step_id))
if curr_step is None:
raise WFError("workflow step %s missing" % step_id)
step_label = curr_step['__Label__']
service_name = self._prepare_param(
curr_step['__Meta__']['Service'])
method = self._prepare_param(curr_step['__Meta__']['Method'])
path = self._prepare_param(curr_step['__Meta__']['Path'])
extras = {}
for meta_param in curr_step['__Meta__']:
if meta_param.lower() not in ['service', 'method', 'path']:
extras[meta_param.lower(
)] = curr_step['__Meta__'][meta_param]
input_map = {}
output_map = {}
for param in curr_step['Parameters']:
if 'Inputs' in param:
input_map = param['Inputs']
if 'Outputs' in param:
output_map = param['Outputs']
res = self.run_single_step(step_id, step_label, service_name,
method, path, input_map, output_map,
**extras)
self.output_resources.append(
ET.Element('tag',
name='state after step %s' % step_id,
value=cgi.escape(str(self.global_vars))))
if isinstance(res, ET._Element):
self.output_resources.append(
ET.Element('tag',
name='reply from step %s' % step_id,
value=cgi.escape(ET.tostring(res))))
except Exception as exc:
if error_mail is not None:
input_map = {
"recipients": error_mail,
"subject": "Workflow failed",
"__xmlbody__": str(exc)
}
self.run_single_step_direct(step_id='FAIL HANDLER',
service_name='notify',
method='POSTXML',
path='email',
input_map=input_map,
output_map={})
raise
def _read_pipeline(self, pipeline_url):
"""
read workflow json doc
"""
pipeline_path = urlparse.urlsplit(pipeline_url).path.split('/')
pipeline_uid = pipeline_path[1] if is_uniq_code(
pipeline_path[1]) else pipeline_path[2]
url = self.bqSession.service_url('pipeline',
path=pipeline_uid,
query={'format': 'json'})
pipeline = self.bqSession.c.fetch(url)
if not pipeline:
# bad pipeline
return None, None
try:
res_pipeline = json.loads(pipeline)
except ValueError:
# not json
return None, None
return res_pipeline
def run_single_step(self, step_id, step_label, service_name, method, path,
input_map, output_map, **kw):
# update status
self.bqSession.update_mex("Executing step %s: %s" %
(str(step_id), step_label))
while True:
res = self.run_single_step_direct(step_id, service_name, method,
path, input_map, output_map)
poll_cond = kw.get('poll_cond')
if poll_cond is None:
break
else:
# need to check output and maybe repeat
cond_res = self._prepare_param(poll_cond, res)
if cond_res in [True, 'True', 1, '1']:
log.debug("poll condition returned %s, exit polling" %
cond_res)
break
else:
log.debug("poll condition returned %s, continue polling" %
cond_res)
poll_sleep = float(kw.get('poll_interval', 10))
time.sleep(poll_sleep)
return res
def run_single_step_direct(self, step_id, service_name, method, path,
input_map, output_map):
# prepare inputs
try:
prep_input_map = self._prepare_params(input_map)
except Exception as exc:
msg = "Step %s (%s %s/%s) failed. Reason: %s" % (
step_id, method, service_name, path, exc)
log.exception(msg)
raise WFError(msg)
# request service
try:
if method.upper() == 'GETXML':
params = prep_input_map
url = urlparse.urljoin(
self.bqSession.service_map[service_name], path)
res = self.bqSession.fetchxml(url=url, **params)
elif method.upper() == 'POSTXML':
params = {
key: val
for (key, val) in prep_input_map.iteritems()
if key not in ['__xmlbody__']
}
url = urlparse.urljoin(
self.bqSession.service_map[service_name], path)
xml = prep_input_map.get('__xmlbody__')
res = self.bqSession.postxml(url=url, xml=xml, **params)
elif method.upper() == 'GETBLOB':
params = prep_input_map
url = urlparse.urljoin(
self.bqSession.service_map[service_name], path)
res = self.bqSession.fetchblob(url=url, **params)
elif method.upper() == 'POSTBLOB':
params = {
key: val
for (key, val) in prep_input_map.iteritems()
if key not in ['__xmlbody__']
}
if service_name != 'import':
raise WFError(
"POSTBLOB method used for service other than import")
xml = prep_input_map.get('__xmlbody__')
res = self.bqSession.postblob(xml=xml, **params)
else:
raise WFError("Unknown method %s" % method)
except Exception as exc:
msg = "Step %s (%s %s/%s) failed. Reason: %s" % (
step_id, method, service_name, path, exc)
log.exception(msg)
raise WFError(msg)
# prepare outputs
try:
prep_output_map = self._prepare_params(output_map, res)
except Exception as exc:
msg = "Step %s (%s %s/%s) failed. Reason: %s" % (
step_id, method, service_name, path, exc)
log.exception(msg)
raise WFError(msg)
# store outputs
self.global_vars.update(prep_output_map)
return res
def _prepare_params(self, param_map, doc=None):
res = {}
for single_input in param_map:
val = param_map[single_input]
val = self._prepare_param(val, doc)
res[single_input] = val
return res
def _prepare_param(self, val, doc=None):
run_xpath = False
if val.startswith('xpath:'):
run_xpath = True
val = val[len('xpath:'):]
# expand variables etc
log.debug("Mako expand %s with %s" % (val, str(self.global_vars)))
template = Template(val)
val = template.render(**self.global_vars)
# run xpath if requested
if run_xpath:
if doc is None or not isinstance(doc, ET._Element):
raise WFError("no result XML")
valx = doc.xpath(val)
if len(valx) < 1:
msg = "xpath '%s' result empty for %s" % (val,
ET.tostring(doc))
log.error(msg)
raise WFError(msg)
val = valx[0]
return val
def teardown(self):
"""
Post the results to the mex xml.
"""
self.bqSession.update_mex('Returning results')
outputTagOuter = ET.Element('tag', name='outputs')
outputTag = ET.SubElement(outputTagOuter, 'tag', name='summary')
for r_xml in self.output_resources:
res_type = r_xml.get('type', None) or r_xml.get(
'resource_type', None) or r_xml.tag
if res_type == 'tag':
# simple tag => append to output as is
r_xml.tag = 'tag'
outputTag.append(r_xml)
else:
# r_xml is some other resource (e.g., image or table) => append reference to output
ET.SubElement(outputTag,
'tag',
name='output_table'
if res_type == 'table' else 'output_image',
type=res_type,
value=r_xml.get('uri', ''))
self.bqSession.finish_mex(tags=[outputTagOuter])
def main(self):
"""
The main function that runs everything
"""
log.info('sysargv : %s' % sys.argv)
parser = OptionParser()
parser.add_option('--mex_url', dest="mexURL")
parser.add_option('--module_dir', dest="modulePath")
parser.add_option('--staging_path', dest="stagingPath")
parser.add_option('--bisque_token', dest="token")
parser.add_option('--user', dest="user")
parser.add_option('--pwd', dest="pwd")
parser.add_option('--root', dest="root")
parser.add_option('--entrypoint', dest="entrypoint")
(options, args) = parser.parse_args()
fh = logging.FileHandler('phase_%s.log' %
(options.entrypoint or 'main'),
mode='a')
fh.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'[%(asctime)s] %(levelname)8s --- %(message)s ' +
'(%(filename)s:%(lineno)s)',
datefmt='%Y-%m-%d %H:%M:%S')
fh.setFormatter(formatter)
log.addHandler(fh)
try: #pull out the mex
if not options.mexURL:
options.mexURL = sys.argv[1]
if not options.token:
options.token = sys.argv[2]
except IndexError: #no argv were set
pass
if not options.stagingPath:
options.stagingPath = '/module/workdir'
log.info('\n\nPARAMS : %s \n\n Options: %s' % (args, options))
self.options = options
if self.validate_input():
#initalizes if user and password are provided
if (self.options.user and self.options.pwd and self.options.root):
self.bqSession = BQSession().init_local(
self.options.user,
self.options.pwd,
bisque_root=self.options.root)
self.options.mexURL = self.bqSession.mex.uri
#initalizes if mex and mex token is provided
elif (self.options.mexURL and self.options.token):
self.bqSession = BQSession().init_mex(self.options.mexURL,
self.options.token)
else:
raise WFError(
'Insufficient options or arguments to start this workflow')
try:
self.setup()
except Exception as e:
log.exception("Exception during setup")
self.bqSession.fail_mex(msg="Exception during setup: %s" %
str(e))
return
try:
self.run()
except (Exception, WFError) as e:
log.exception("Exception during run")
self.bqSession.fail_mex(msg="Exception during run: %s" %
str(e))
return
try:
self.teardown()
except (Exception, WFError) as e:
log.exception("Exception during teardown")
self.bqSession.fail_mex(msg="Exception during teardown: %s" %
str(e))
return
self.bqSession.close()
class Dream3D(object):
"""
Dream3D Module
"""
def mex_parameter_parser(self, mex_xml):
"""
Parses input of the xml and add it to Dream3D's options attribute (unless already set)
@param: mex_xml
"""
# inputs are all non-"pipeline params" under "inputs" and all params under "pipeline_params"
mex_inputs = mex_xml.xpath(
'tag[@name="inputs"]/tag[@name!="pipeline_params"] | tag[@name="inputs"]/tag[@name="pipeline_params"]/tag'
)
if mex_inputs:
for tag in mex_inputs:
if tag.tag == 'tag' and tag.get(
'type',
'') != 'system-input': #skip system input values
if not getattr(self.options, tag.get('name', ''), None):
log.debug('Set options with %s as %s' %
(tag.get('name', ''), tag.get('value', '')))
setattr(self.options, tag.get('name', ''),
tag.get('value', ''))
else:
log.debug('Dream3D: No Inputs Found on MEX!')
def validate_input(self):
"""
Check to see if a mex with token or user with password was provided.
@return True is returned if validation credention was provided else
False is returned
"""
if (self.options.mexURL
and self.options.token): #run module through engine service
return True
if (self.options.user and self.options.pwd and
self.options.root): #run module locally (note: to test module)
return True
log.debug(
'Dream3D: Insufficient options or arguments to start this module')
return False
def setup(self):
"""
Fetches the mex and appends input_configurations to the option
attribute of Dream3D
"""
self.bqSession.update_mex('Initializing...')
self.mex_parameter_parser(self.bqSession.mex.xmltree)
self.output_file = None
def run(self):
"""
The core of the Dream3D runner
"""
#retrieve tags
self.bqSession.update_mex('Extracting properties')
#type check
hdf_resource = self.bqSession.fetchxml(self.options.InputFile,
view='deep,clean')
if (hdf_resource.tag != 'resource'
or hdf_resource.get('resource_type', '') != 'table'
) and hdf_resource.tag != 'table':
raise Dream3DError("trying to run Dream3D on non-table resource")
hdf_url = self.bqSession.service_url(
'blob_service', path=hdf_resource.get('resource_uniq'))
self.bqSession.fetchblob(hdf_url,
path=os.path.join(self.options.stagingPath,
'input.h5'))
hdf_input_file = os.path.join(self.options.stagingPath, 'input.h5')
hdf_output_file = os.path.join(self.options.stagingPath, 'output.h5')
# create pipeline with correct parameters
pipeline_params = self.bqSession.mex.xmltree.xpath(
'tag[@name="inputs"]/tag[@name="pipeline_params"]/tag')
params = {}
for tag in pipeline_params:
params[tag.get('name', '')] = getattr(self.options,
tag.get('name', ''))
pipeline_file, err_file = self._instantiate_pipeline(
pipeline_url=self.options.pipeline_url,
input_file=hdf_input_file,
output_file=hdf_output_file,
params=params)
# run Dream3D on the pipeline
self.bqSession.update_mex('Running Dream3D')
log.debug('run Dream3D on %s', pipeline_file)
res = 1
with open(err_file, 'w') as fo:
# res = 0 #!!! TESTING
# open(hdf_output_file, 'a').close()
res = subprocess.call(
['/build/Bin/PipelineRunner', '-p', pipeline_file],
stderr=fo,
stdout=fo)
log.debug("Dream3D returned: %s", str(res))
if res > 0:
err_msg = 'pipeline execution failed\n'
with open(err_file, 'r') as fo:
err_msg += ''.join(fo.readlines())
raise Dream3DError(err_msg)
self.output_file = hdf_output_file
def _instantiate_pipeline(self, pipeline_url, input_file, output_file,
params):
"""instantiate pipeline json file with provided parameters"""
pipeline_resource = self.bqSession.fetchxml(pipeline_url, view='short')
out_pipeline_file = os.path.join(self.options.stagingPath,
'pipeline.json')
out_error_file = os.path.join(self.options.stagingPath,
'dream3d_error.txt')
pipeline_url = self.bqSession.service_url(
'blob_service', path=pipeline_resource.get('resource_uniq'))
self.bqSession.fetchblob(pipeline_url,
path=os.path.join(self.options.stagingPath,
'pipeline_uninit.json'))
pipeline_file = os.path.join(self.options.stagingPath,
'pipeline_uninit.json')
with open(pipeline_file, 'r') as fi:
pipeline = json.load(fi)
# replace all placeholders in pipeline template
_replace_placeholders(pipeline, input_file, output_file, params)
# write out pipeline to provided file
with open(out_pipeline_file, 'w') as fo:
json.dump(pipeline, fo)
return out_pipeline_file, out_error_file
def teardown(self):
"""
Post the results to the mex xml.
"""
#save the HDF output and upload it to the data service with all the meta data
self.bqSession.update_mex('Returning results')
log.debug('Storing HDF output')
#constructing and storing HDF file
mex_id = self.bqSession.mex.uri.split('/')[-1]
dt = datetime.now().strftime('%Y%m%dT%H%M%S')
final_output_file = "ModuleExecutions/Dream3D/%s_%s_%s.h5" % (
self.options.OutputPrefix, dt, mex_id)
#does not accept no name on the resource
cl_model = etree.Element('resource',
resource_type='table',
name=final_output_file)
#module identifier (a descriptor to be found by the Dream3D model)
etree.SubElement(cl_model,
'tag',
name='module_identifier',
value='Dream3D')
#hdf filename
etree.SubElement(cl_model,
'tag',
name='OutputFile',
value=final_output_file)
#pipeline param
#etree.SubElement(cl_model, 'tag', name='RefFrameZDir', value=self.options.RefFrameZDir)
#input hdf url
#etree.SubElement(cl_model, 'tag', name='InputFile', type='link', value=self.options.InputFile)
#input pipeline
#etree.SubElement(cl_model, 'tag', name='pipeline_url', type='link', value=self.options.pipeline_url)
#description
etree.SubElement(cl_model,
'tag',
name='description',
value='HDF output file from Dream3D Module')
#storing the HDF file in blobservice
log.debug('before postblob') #!!!
r = self.bqSession.postblob(self.output_file, xml=cl_model)
r_xml = etree.fromstring(r)
outputTag = etree.Element('tag', name='outputs')
etree.SubElement(outputTag,
'tag',
name='output_hdf',
type='table',
value=r_xml[0].get('uri', ''))
self.bqSession.finish_mex(tags=[outputTag])
def collect_outputs(self):
"""
Perform reduce phase (i.e., examine final (top) mex and create any additional outputs based on submexes)
THIS IS JUST AN EXAMPLE.
"""
# collect submex output hdf urls and add them to top mex outputs section
top_mex = self.bqSession.fetchxml(self.options.mexURL, view='deep')
outputTag = top_mex.xpath('/mex/tag[@name="outputs"]')[0]
output_hdfs = top_mex.xpath(
'/mex/mex/tag[@name="outputs"]/tag[@name="output_hdf"]/@value')
etree.SubElement(outputTag,
'tag',
name='all_outputs',
value=';'.join(
[id.split('/')[-1] for id in output_hdfs]))
self.bqSession.postxml(url=outputTag.get('uri'), xml=outputTag)
def main(self):
"""
The main function that runs everything
"""
log.info('sysargv : %s' % sys.argv)
parser = OptionParser()
parser.add_option('--mex_url', dest="mexURL")
parser.add_option('--module_dir', dest="modulePath")
parser.add_option('--staging_path', dest="stagingPath")
parser.add_option('--bisque_token', dest="token")
parser.add_option('--user', dest="user")
parser.add_option('--pwd', dest="pwd")
parser.add_option('--root', dest="root")
# for the reduce phase: create output dataset of output HDFs (in this case, mexURL is top mex)
parser.add_option('--entrypoint', dest="entrypoint")
(options, args) = parser.parse_args()
fh = logging.FileHandler('phase_%s.log' %
(options.entrypoint or 'main'),
mode='a')
fh.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'[%(asctime)s] %(levelname)8s --- %(message)s ' +
'(%(filename)s:%(lineno)s)',
datefmt='%Y-%m-%d %H:%M:%S')
fh.setFormatter(formatter)
log.addHandler(fh)
try: #pull out the mex
if not options.mexURL:
options.mexURL = sys.argv[1]
if not options.token:
options.token = sys.argv[2]
except IndexError: #no argv were set
pass
if not options.stagingPath:
options.stagingPath = ''
log.info('\n\nPARAMS : %s \n\n Options: %s' % (args, options))
self.options = options
if self.validate_input():
#initalizes if user and password are provided
if (self.options.user and self.options.pwd and self.options.root):
self.bqSession = BQSession().init_local(
self.options.user,
self.options.pwd,
bisque_root=self.options.root)
self.options.mexURL = self.bqSession.mex.uri
#initalizes if mex and mex token is provided
elif (self.options.mexURL and self.options.token):
self.bqSession = BQSession().init_mex(self.options.mexURL,
self.options.token)
else:
raise Dream3DError(
'Insufficient options or arguments to start this module')
if not self.options.entrypoint:
# NOT a reduce phase => perform regular run processing
try:
self.setup()
except Exception as e:
log.exception("Exception during setup")
self.bqSession.fail_mex(msg="Exception during setup: %s" %
str(e))
return
try:
self.run()
except (Exception, Dream3DError) as e:
log.exception("Exception during run")
self.bqSession.fail_mex(msg="Exception during run: %s" %
str(e))
return
try:
self.teardown()
except (Exception, Dream3DError) as e:
log.exception("Exception during teardown")
self.bqSession.fail_mex(
msg="Exception during teardown: %s" % str(e))
return
else:
# in a reduce phase => run reduce code
if self.options.entrypoint != 'collect_outputs':
self.bqSession.fail_mex(
msg="Unknown Dream3D entrypoint: %s" %
self.options.entrypoint)
return
try:
self.collect_outputs()
except (Exception, Dream3DError) as e:
log.exception("Exception during collect_outputs")
self.bqSession.fail_mex(
msg="Exception during collect_outputs: %s" % str(e))
return
self.bqSession.close()
class CellProfiler(object):
"""
CellProfiler Module
"""
def mex_parameter_parser(self, mex_xml):
"""
Parses input of the xml and add it to CellProfiler's options attribute (unless already set)
@param: mex_xml
"""
# inputs are all non-"pipeline params" under "inputs" and all params under "pipeline_params"
mex_inputs = mex_xml.xpath(
'tag[@name="inputs"]/tag[@name!="pipeline_params"] | tag[@name="inputs"]/tag[@name="pipeline_params"]/tag'
)
if mex_inputs:
for tag in mex_inputs:
if tag.tag == 'tag' and tag.get(
'type',
'') != 'system-input': #skip system input values
if not getattr(self.options, tag.get('name', ''), None):
log.debug('Set options with %s as %s' %
(tag.get('name', ''), tag.get('value', '')))
setattr(self.options, tag.get('name', ''),
tag.get('value', ''))
else:
log.debug('CellProfiler: No Inputs Found on MEX!')
def validate_input(self):
"""
Check to see if a mex with token or user with password was provided.
@return True is returned if validation credention was provided else
False is returned
"""
if (self.options.mexURL
and self.options.token): #run module through engine service
return True
if (self.options.user and self.options.pwd and
self.options.root): #run module locally (note: to test module)
return True
log.debug(
'CellProfiler: Insufficient options or arguments to start this module'
)
return False
def setup(self):
"""
Fetches the mex and appends input_configurations to the option
attribute of CellProfiler
"""
self.bqSession.update_mex('Initializing...')
self.mex_parameter_parser(self.bqSession.mex.xmltree)
self.output_resources = []
self.ppops = None
self.ppops_url = None
def run(self):
"""
The core of the CellProfiler runner
"""
module_time = datetime.now()
#retrieve tags
self.bqSession.update_mex('Extracting properties')
#type check
image_resource = self.bqSession.fetchxml(self.options.InputFile)
if image_resource.tag != 'image':
raise CPError("trying to run CellProfiler on non-image resource")
# run prerun operations
filelist_file = self._run_prerun_ops(
module_time,
pipeline_url=self.options.pipeline_url,
input_xml=image_resource)
# create pipeline with correct parameters
pipeline_params = self.bqSession.mex.xmltree.xpath(
'tag[@name="inputs"]/tag[@name="pipeline_params"]/tag')
params = {}
for tag in pipeline_params:
params[tag.get('name', '')] = getattr(self.options,
tag.get('name', ''))
pipeline_file, err_file = self._instantiate_pipeline(
pipeline_url=self.options.pipeline_url, params=params)
if not pipeline_file:
raise CPError("trying to run incompatible CellProfiler pipeline")
# run CellProfiler on the pipeline
self.bqSession.update_mex('Running CellProfiler')
log.debug('run CellProfiler on %s', pipeline_file)
res = 1
with open(err_file, 'w') as fo:
res = subprocess.call([
'python', '/module/CellProfiler/CellProfiler.py', '-c', '-r',
'-i', self.options.stagingPath, '-o', self.options.stagingPath,
'-p', pipeline_file, '--file-list', filelist_file
],
stderr=fo,
stdout=fo)
log.debug("CellProfiler returned: %s", str(res))
if res > 0:
err_msg = 'pipeline execution failed\n'
with open(err_file, 'r') as fo:
err_msg += ''.join(fo.readlines())
if len(err_msg) > 1024:
err_msg = err_msg[:512] + '...' + err_msg[-512:]
raise CPError(err_msg)
# run postrun operations
self.output_resources = self._run_postrun_ops(
module_time, pipeline_url=self.options.pipeline_url)
def _cache_ppops(self, pipeline_url):
if not self.ppops or self.ppops_url != pipeline_url:
pipeline_path = urlparse.urlsplit(pipeline_url).path.split('/')
pipeline_uid = pipeline_path[1] if is_uniq_code(
pipeline_path[1]) else pipeline_path[2]
url = self.bqSession.service_url(
'pipeline',
path='/'.join([pipeline_uid] + ['ppops:cellprofiler']))
self.ppops = json.loads(self.bqSession.c.fetch(url))
self.ppops_url = pipeline_url
def _run_prerun_ops(self, module_time, pipeline_url, input_xml):
"""
Perform any operations necessary before the pipeline runs (e.g., extract image channels) and return filelist file
"""
self._cache_ppops(pipeline_url)
post_ops = self.ppops['PreOps']
input_files = []
for op in post_ops:
input_files += self._run_single_op(module_time, op, input_xml)
filelist_file = os.path.join(self.options.stagingPath, 'filelist.txt')
with open(filelist_file, 'w') as fo:
for input_file in input_files:
fo.write(input_file + '\n')
return filelist_file
def _run_postrun_ops(self, module_time, pipeline_url):
"""
Perform any operations necessary after the pipeline finished (e.g., upload result tables) and return created resources
"""
self._cache_ppops(pipeline_url)
post_ops = self.ppops['PostOps']
created_resources = []
for op in post_ops:
created_resources += self._run_single_op(module_time, op)
return created_resources
def _run_single_op(self, module_time, op, input_xml=None):
"""
Perform single pre/post operation and return list of files or resources generated
"""
# replace special placeholders
if 'id' in op and op['id'] == '@INPUT':
op['id'] = input_xml.get('resource_uniq')
res = []
if op['service'] == 'image_service':
# perform image_service operation
log.debug("RUNOP %s" % str(op))
url = self.bqSession.service_url('image_service',
path=op['id'] + op['ops'])
# TODO: don't read image into memory!!!
image_data = self.bqSession.c.fetch(url)
image_file = os.path.join(self.options.stagingPath, op['filename'])
with open(image_file, 'w') as fo:
fo.write(image_data)
res += [image_file]
elif op['service'] == 'postblob':
# upload image or table (check op['type'])
dt = module_time.strftime('%Y%m%dT%H%M%S')
final_output_file = "ModuleExecutions/CellProfiler/%s/%s" % (
dt, op['name'])
cl_model = etree.Element('resource',
resource_type=op['type'],
name=final_output_file)
# module identifier (a descriptor to be found by the CellProfiler model)
etree.SubElement(cl_model,
'tag',
name='module_identifier',
value='CellProfiler')
# hdf filename
etree.SubElement(cl_model,
'tag',
name='OutputFile',
value=final_output_file)
#description
etree.SubElement(cl_model,
'tag',
name='description',
value='output from CellProfiler Module')
# post blob
output_file = os.path.join(self.options.stagingPath,
op['filename'])
resource = self.bqSession.postblob(output_file, xml=cl_model)
resource_xml = etree.fromstring(resource)
res += [resource_xml[0]]
elif op['service'] == 'postellipse':
# add ellipse gobject to mex
# Read object measurements from csv and write to gobjects
(header, records) = self._readCSV(
os.path.join(self.options.stagingPath, op['filename']))
if header is not None:
parentGObject = etree.Element('gobject',
type='detected shapes',
name='detected shapes')
for i in range(len(records)):
shape = self._get_ellipse_elem(
name=str(i),
header=header,
record=records[i],
x=op['x_coord'],
y=op['y_coord'],
label=op['label'],
color=op['color'],
orientation=op['orientation'],
major_axis=op['major_axis'],
minor_axis=op['minor_axis'])
if shape:
parentGObject.append(shape)
res += [parentGObject]
return res
def _instantiate_pipeline(self, pipeline_url, params):
"""
instantiate cellprofiler pipeline file with provided parameters
"""
pipeline_path = urlparse.urlsplit(pipeline_url).path.split('/')
pipeline_uid = pipeline_path[1] if is_uniq_code(
pipeline_path[1]) else pipeline_path[2]
url = self.bqSession.service_url(
'pipeline',
path='/'.join(
[pipeline_uid] +
["setvar:%s|%s" % (tag, params[tag])
for tag in params] + ['exbsteps:cellprofiler']),
query={'format': 'cellprofiler'})
pipeline = self.bqSession.c.fetch(url)
if not pipeline:
# bad pipeline
return None, None
out_pipeline_file = os.path.join(self.options.stagingPath,
'pipeline.cp')
out_error_file = os.path.join(self.options.stagingPath, 'cp_error.txt')
with open(out_pipeline_file, 'w') as fo:
fo.write(pipeline)
return out_pipeline_file, out_error_file
def teardown(self):
"""
Post the results to the mex xml.
"""
self.bqSession.update_mex('Returning results')
outputTag = etree.Element('tag', name='outputs')
for r_xml in self.output_resources:
res_type = r_xml.get('type', None) or r_xml.get(
'resource_type', None) or r_xml.tag
if res_type == 'detected shapes':
# r_xml is a set of gobjects => append to output inside image tag
image_resource = self.bqSession.fetchxml(
self.options.InputFile)
image_elem = etree.SubElement(outputTag,
'tag',
name=image_resource.get('name'),
type='image',
value=image_resource.get('uri'))
image_elem.append(r_xml)
else:
# r_xml is some other resource (e.g., image or table) => append reference to output
etree.SubElement(outputTag,
'tag',
name='output_table'
if res_type == 'table' else 'output_image',
type=res_type,
value=r_xml.get('uri', ''))
self.bqSession.finish_mex(tags=[outputTag])
def _get_ellipse_elem(self, name, **params):
header = params.get('header')
record = params.get('record')
getValue = lambda x: float(record[header.index(x)])
shape = etree.Element('gobject', name=name, type=params.get('label'))
res = etree.SubElement(shape, 'ellipse')
try:
# centroid
x = getValue(params.get('x'))
y = getValue(params.get('y'))
theta = math.radians(getValue(params.get('orientation')))
etree.SubElement(res, 'vertex', x=str(x), y=str(y))
# major axis/minor axis endpoint coordinates
a = 0.5 * getValue(params.get('major_axis'))
b = 0.5 * getValue(params.get('minor_axis'))
bX = round(x - b * math.sin(theta))
bY = round(y + b * math.cos(theta))
etree.SubElement(res, 'vertex', x=str(bX), y=str(bY))
aX = round(x + a * math.cos(theta))
aY = round(y + a * math.sin(theta))
etree.SubElement(res, 'vertex', x=str(aX), y=str(aY))
# other statistics
#etree.SubElement(res, 'tag', name="Compactness", value=str(getValue('AreaShape_Compactness')))
#etree.SubElement(res, 'tag', name="Eccentricity", value=str(getValue('AreaShape_Eccentricity')))
#etree.SubElement(res, 'tag', name="FormFactor", value=str(getValue('AreaShape_FormFactor')))
#etree.SubElement(res, 'tag', name="Solidity", value=str(getValue('AreaShape_Solidity')))
etree.SubElement(res,
'tag',
name='color',
value=params.get('color', '#FF0000'),
type='color')
except KeyError:
return None
except ValueError:
return None
return shape
def _readCSV(self, fileName):
if os.path.exists(fileName) == False:
return (None, None)
records = []
handle = open(fileName, 'rb')
csvHandle = csv.reader(handle)
header = csvHandle.next()
for row in csvHandle:
records.append(row)
handle.close()
return (header, records)
def main(self):
"""
The main function that runs everything
"""
log.info('sysargv : %s' % sys.argv)
parser = OptionParser()
parser.add_option('--mex_url', dest="mexURL")
parser.add_option('--module_dir', dest="modulePath")
parser.add_option('--staging_path', dest="stagingPath")
parser.add_option('--bisque_token', dest="token")
parser.add_option('--user', dest="user")
parser.add_option('--pwd', dest="pwd")
parser.add_option('--root', dest="root")
parser.add_option('--entrypoint', dest="entrypoint")
(options, args) = parser.parse_args()
fh = logging.FileHandler('phase_%s.log' %
(options.entrypoint or 'main'),
mode='a')
fh.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'[%(asctime)s] %(levelname)8s --- %(message)s ' +
'(%(filename)s:%(lineno)s)',
datefmt='%Y-%m-%d %H:%M:%S')
fh.setFormatter(formatter)
log.addHandler(fh)
try: #pull out the mex
if not options.mexURL:
options.mexURL = sys.argv[1]
if not options.token:
options.token = sys.argv[2]
except IndexError: #no argv were set
pass
if not options.stagingPath:
options.stagingPath = '/module/CellProfiler/workdir'
log.info('\n\nPARAMS : %s \n\n Options: %s' % (args, options))
self.options = options
if self.validate_input():
#initalizes if user and password are provided
if (self.options.user and self.options.pwd and self.options.root):
self.bqSession = BQSession().init_local(
self.options.user,
self.options.pwd,
bisque_root=self.options.root)
self.options.mexURL = self.bqSession.mex.uri
#initalizes if mex and mex token is provided
elif (self.options.mexURL and self.options.token):
self.bqSession = BQSession().init_mex(self.options.mexURL,
self.options.token)
else:
raise CPError(
'Insufficient options or arguments to start this module')
if not self.options.entrypoint:
# NOT a special phase => perform regular run processing
try:
self.setup()
except Exception as e:
log.exception("Exception during setup")
self.bqSession.fail_mex(msg="Exception during setup: %s" %
str(e))
return
try:
self.run()
except (Exception, CPError) as e:
log.exception("Exception during run")
self.bqSession.fail_mex(msg="Exception during run: %s" %
str(e))
return
try:
self.teardown()
except (Exception, CPError) as e:
log.exception("Exception during teardown")
self.bqSession.fail_mex(
msg="Exception during teardown: %s" % str(e))
return
else:
# in a special phase => run special code
self.bqSession.fail_mex(
msg="Unknown CellProfiler entrypoint: %s" %
self.options.entrypoint)
return
self.bqSession.close()
class SkeletonPython(object):
"""
SkeletonPython Model
"""
def mex_parameter_parser(self, mex_xml):
"""
Parses input of the xml and add it to SkeletonPython Trainer's options attribute
@param: mex_xml
"""
mex_inputs = mex_xml.xpath('tag[@name="inputs"]')
if mex_inputs:
for tag in mex_inputs[0]:
if tag.tag == 'tag' and tag.attrib['type'] != 'system-input':
log.debug('Set options with %s as %s'%(tag.attrib['name'],tag.attrib['value']))
setattr(self.options,tag.attrib['name'],tag.attrib['value'])
else:
log.debug('SkeletonPythonFS: No Inputs Found on MEX!')
def validateInput(self):
"""
Parses input of the xml and add it to SkeletonPython's options attribute
@param: mex_xml
"""
if (self.options.mexURL and self.options.token): #run module through engine service
return True
if (self.options.user and self.options.pwd and self.options.root): #run module locally (note: to test module)
return True
log.debug('SkeletonPython: Insufficient options or arguments to start this module')
return False
def setup(self):
"""
Fetches the mex, appends input_configurations to the option
attribute of SkeletonPython and looks up the model on bisque to
classify the provided resource.
"""
if (self.options.user and self.options.pwd and self.options.root):
self.bqSession = BQSession().init_local( self.options.user, self.options.pwd, bisque_root=self.options.root)
self.options.mexURL = self.bqSession.mex.uri
# This is when the module actually runs on the server with a mexURL and an access token
elif (self.options.mexURL and self.options.token):
self.bqSession = BQSession().init_mex(self.options.mexURL, self.options.token)
else:
return
# Parse the xml and construct the tree, also set options to proper values after parsing it (like image url)
self.mex_parameter_parser(self.bqSession.mex.xmltree)
log.debug('SkeletonPython: image URL: %s, mexURL: %s, stagingPath: %s, token: %s' % (self.options.image_url, self.options.mexURL, self.options.stagingPath, self.options.token))
def construct_vertices(self, child):
annotation_type = 'bg'
if 'foreground' in child.values():
annotation_type = 'fg'
roi = []
log.debug("This is the child")
vertices = child.getchildren()[0].getchildren()
for vertex in vertices:
values = vertex.values()
roi.append({'x':int(float(values[2])), 'y':int(float(values[3]))})
self.rois[annotation_type].append(roi)
log.debug(vertices)
log.debug(len(vertices))
def show_structure(self, r_xml):
for i, child in enumerate(r_xml.getchildren()):
if "background" in child.values() or 'foreground' in child.values():
log.debug('Background/Foreground annotation')
self.construct_vertices(child)
else:
self.show_structure(child)
def run(self):
"""
The core of the SkeletonPython Module
Requests features on the image provided. Classifies each tile
and picks a majority among the tiles.
"""
self.rois = {'fg':[],'bg':[]}
r_xml = self.bqSession.fetchxml(self.options.mexURL, view='deep')
log.debug("Shols structura")
self.show_structure(r_xml)
log.debug(self.rois)
image = self.bqSession.load(self.options.image_url)
ip = image.pixels().format('tiff')
pixels = ip.fetch()
f = open('./temp.tif','wb')
f.write(pixels)
f.close()
pickle.dump([self.rois,self.options.segmentImage,self.options.deepNetworkChoice,self.options.qualitySeg,self.options.deepSeg,self.options.mexURL,self.options.token], open('./data.p','wb'))
pathToScript = './DeepTools/deep_script.sh'
call([pathToScript])
def teardown(self):
"""
Posting results to the mex
"""
self.bqSession.update_mex('Returning results...')
log.debug('Returning results...')
prediction = "None-Module Failure"
with open("./results.txt","r") as f:
for line in f:
if "PREDICTION_C:" in line:
prediction_c = line
if "CONFIDENCE_C:" in line:
confidence_c = line[14:-1]
classes = ["leaf","fruit","flower","stem","entire"]
if self.options.deepNetworkChoice != 'None':
for i,class_tag in enumerate(classes):
prediction_c = prediction_c.replace(str(i),class_tag)
outputTag = etree.Element('tag', name='outputs')
outputSubTagImage = etree.SubElement(outputTag, 'tag', name='Final Image', value=self.options.image_url)
print "Module will output image, {}".format(self.options.image_url)
if not os.path.isfile("./contours.pkl"):
print "Module will not segment image, (were foreground and background polyline annotations provided?)"
if self.options.segmentImage != "False" and os.path.isfile("./contours.pkl"):
[contours, t_scale] = pickle.load(open("./contours.pkl","rb"))
gob = etree.SubElement (outputSubTagImage, 'gobject', name='Annotations', type='Annotations')
polyseg = etree.SubElement(gob, 'polygon', name='SEG')
etree.SubElement( polyseg, 'tag', name='color', value="#0000FF")
opd = 0
output_sampling = 1+int(len(contours)/100)
for j in range(len(contours)):
if j % (output_sampling) == 0:
opd += 1
etree.SubElement (polyseg, 'vertex', x=str(1+int(t_scale[1]*contours[j][1])), y=str(1+int(t_scale[0]*contours[j][0])))
log.debug(opd)
if self.options.deepNetworkChoice != 'None':
outputSubTagSummary = etree.SubElement(outputTag, 'tag', name='summary')
etree.SubElement(outputSubTagSummary, 'tag',name='Model File', value=self.options.deepNetworkChoice)
etree.SubElement(outputSubTagSummary, 'tag',name='Segment Image', value=self.options.segmentImage)
etree.SubElement(outputSubTagSummary, 'tag',name='Class', value=str(prediction_c))
etree.SubElement(outputSubTagSummary, 'tag', name='Class Confidence', value=str(confidence_c))
self.bqSession.finish_mex(tags = [outputTag])
log.debug('FINISHED')
self.bqSession.close()
def main(self):
"""
The main function that runs everything
"""
print("DEBUG_INIT")
log.debug('SkeletonPython is called with the following arguments')
log.debug('sysargv : %s\n\n' % sys.argv )
parser = OptionParser()
parser.add_option( '--image_url' , dest="image_url")
parser.add_option( '--mex_url' , dest="mexURL")
parser.add_option( '--module_dir' , dest="modulePath")
parser.add_option( '--staging_path', dest="stagingPath")
parser.add_option( '--bisque_token', dest="token")
parser.add_option( '--user' , dest="user")
parser.add_option( '--pwd' , dest="pwd")
parser.add_option( '--root' , dest="root")
(options, args) = parser.parse_args()
# Set up the mexURL and token based on the arguments passed to the script
try: #pull out the mex
log.debug("options %s" % options)
if not options.mexURL:
options.mexURL = sys.argv[1]
if not options.token:
options.token = sys.argv[2]
except IndexError: #no argv were set
pass
if not options.stagingPath:
options.stagingPath = ''
# Still don't have an imgurl, but it will be set up in self.setup()
log.debug('\n\nPARAMS : %s \n\n Options: %s'%(args, options))
self.options = options
if self.validateInput():
try: #run setup and retrieve mex variables
self.setup()
except Exception, e:
log.exception("Exception during setup")
self.bqSession.fail_mex(msg = "Exception during setup: %s" % str(e))
return
try: #run module operation
self.run()
except SkeletonPythonError, e:
log.exception("Exception during run")
self.bqSession.fail_mex(msg = "Exception during run: %s" % str(e.message))
return
except Exception, e:
log.exception("Exception during run")
self.bqSession.fail_mex(msg = "Exception during run: %s" % str(e))
return
class SkeletonPython(object):
"""
SkeletonPython Model
"""
def mex_parameter_parser(self, mex_xml):
"""
Parses input of the xml and add it to SkeletonPython Trainer's options attribute
@param: mex_xml
"""
mex_inputs = mex_xml.xpath('tag[@name="inputs"]')
if mex_inputs:
for tag in mex_inputs[0]:
if tag.tag == 'tag' and tag.attrib['type'] != 'system-input':
log.debug('Set options with %s as %s' %
(tag.attrib['name'], tag.attrib['value']))
setattr(self.options, tag.attrib['name'],
tag.attrib['value'])
else:
log.debug('SkeletonPythonFS: No Inputs Found on MEX!')
def validateInput(self):
"""
Parses input of the xml and add it to SkeletonPython's options attribute
@param: mex_xml
"""
if (self.options.mexURL
and self.options.token): #run module through engine service
return True
if (self.options.user and self.options.pwd and
self.options.root): #run module locally (note: to test module)
return True
log.debug(
'SkeletonPython: Insufficient options or arguments to start this module'
)
return False
def setup(self):
"""
Fetches the mex, appends input_configurations to the option
attribute of SkeletonPython and looks up the model on bisque to
classify the provided resource.
"""
if (self.options.user and self.options.pwd and self.options.root):
self.bqSession = BQSession().init_local(
self.options.user,
self.options.pwd,
bisque_root=self.options.root)
self.options.mexURL = self.bqSession.mex.uri
# This is when the module actually runs on the server with a mexURL and an access token
elif (self.options.mexURL and self.options.token):
self.bqSession = BQSession().init_mex(self.options.mexURL,
self.options.token)
else:
return
# Parse the xml and construct the tree, also set options to proper values after parsing it (like image url)
self.mex_parameter_parser(self.bqSession.mex.xmltree)
log.debug(
'SkeletonPython: image URL: %s, mexURL: %s, stagingPath: %s, token: %s'
% (self.options.image_url, self.options.mexURL,
self.options.stagingPath, self.options.token))
def run(self):
"""
The core of the SkeletonPython Module
Requests features on the image provided. Classifies each tile
and picks a majority among the tiles.
"""
r_xml = self.bqSession.fetchxml(self.options.mexURL, view='deep')
'''rectangles = r_xml.xpath('//tag[@name="inputs"]/tag[@name="image_url"]/gobject[@name="roi"]/rectangle')
rois = []
rois_rectangles = []
for i in range(len(rectangles)):
x1 = int(float(rectangles[i][0].attrib['x']))
y1 = int(float(rectangles[i][0].attrib['y']))
x2 = int(float(rectangles[i][1].attrib['x']))
y2 = int(float(rectangles[i][1].attrib['y']))
rois_rectangles.append([x1,y1,x2,y2])
polygons = r_xml.xpath('//tag[@name="inputs"]/tag[@name="image_url"]/gobject[@name="roi"]/polygon')
rois_polygons = []
for i in range(len(polygons)):
polygon = []
for j in range(len(polygons[i])):
x = int(float(polygons[i][j].attrib['x']))
y = int(float(polygons[i][j].attrib['y']))
polygon.append([x,y])
rois_polygons.append(polygon)
polylines = r_xml.xpath('//tag[@name="inputs"]/tag[@name="image_url"]/gobject[@name="roi"]/polyline')
rois_polylines = []
for i in range(len(polylines)):
polyline = []
for j in range(len(polylines[i])):
x = int(float(polylines[i][j].attrib['x']))
y = int(float(polylines[i][j].attrib['y']))
polyline.append([x,y])
rois_polylines.append(polyline)
rois.append(rois_polylines)
rois.append(rois_polygons)
rois.append(rois_rectangles)'''
def teardown(self):
"""
Posting results to the mex
"""
self.bqSession.update_mex('Returning results...')
'''log.debug('Returning results...')
prediction = "None-Module Failure"
with open("./results.txt","r") as f:
for line in f:
if "PREDICTION_C:" in line:
prediction_c = line
if "CONFIDENCE_C:" in line:
confidence_c = line[14:-1]
if "PREDICTION_T:" in line:
prediction_t = line
if "CONFIDENCE_T:" in line:
confidence_t = line[14:-1]
print line
classes = ["flower","stem","fruit","entire","leaf"]
classes_type = ["sheet","natural"]
for i,class_tag in enumerate(classes):
prediction_c = prediction_c.replace(str(i),class_tag)
for i,class_tag in enumerate(classes_type):
prediction_t = prediction_t.replace(str(i),class_tag)
'''
outputTag = etree.Element('tag', name='outputs')
#dda = etree.SubElement(outputTag, 'tag', name='mex_url', value=self.options.image_url)
outputSubTagImage = etree.SubElement(outputTag,
'tag',
name='OutputImage',
value=self.options.image_url)
'''gob = etree.SubElement (outputSubTagImage, 'gobject', name='Annotations', type='Annotations')
xc = [100, 300, 200]
yc = [100, 150, 200]
polyseg = etree.SubElement(gob, 'polygon', name='SEG')
etree.SubElement( polyseg, 'tag', name='color', value="#0000FF")
for i in range(len(xc)):
etree.SubElement (polyseg, 'vertex', x=str(1*int(yc[i])), y=str(1*int(xc[i])))
print outputTag'''
'''print "Module will output image, {}".format(self.options.image_url)
if not os.path.isfile("./contours.pkl"):
print "Module will not segment image, (were foreground and background polyline annotations provided?)"
if self.options.segmentImage != "False" and os.path.isfile("./contours.pkl"):
[contours, t_scale] = pickle.load(open("./contours.pkl","rb"))
xc = []
yc = []
for i, p in enumerate(contours):
if i < len(contours)/2:
xc.append(p)
else:
yc.append(p)
gob = etree.SubElement (outputSubTagImage, 'gobject', name='Annotations', type='Annotations')
polyseg = etree.SubElement(gob, 'polygon', name='SEG')
etree.SubElement( polyseg, 'tag', name='color', value="#0000FF")
print "TSCALE", t_scale
for i in range(len(xc)):
etree.SubElement (polyseg, 'vertex', x=str(t_scale[1]*int(yc[i])), y=str(t_scale[0]*int(xc[i])))
###outputSubTagSummary = etree.SubElement(outputTag, 'tag', name='summary')
#etree.SubElement(outputSubTagSummary, 'tag',name='Model File', value=self.options.deepNetworkChoice)
#etree.SubElement(outputSubTagSummary, 'tag',name='Segment Image', value=self.options.segmentImage)
###etree.SubElement(outputSubTagSummary, 'tag',name='Class', value=str(prediction_c))
###etree.SubElement(outputSubTagSummary, 'tag', name='Class Confidence', value=str(confidence_c))
#etree.SubElement(outputSubTagSummary, 'tag',name='Type', value=str(prediction_t))
#etree.SubElement(outputSubTagSummary, 'tag', name='Type Confidence', value=str(confidence_t))
'''
self.bqSession.finish_mex(tags=[outputTag])
log.debug('FINISHED')
self.bqSession.close()
def main(self):
"""
The main function that runs everything
"""
print("DEBUG_INIT")
log.debug('SkeletonPython is called with the following arguments')
log.debug('sysargv : %s\n\n' % sys.argv)
parser = OptionParser()
parser.add_option('--image_url', dest="image_url")
parser.add_option('--mex_url', dest="mexURL")
parser.add_option('--module_dir', dest="modulePath")
parser.add_option('--staging_path', dest="stagingPath")
parser.add_option('--bisque_token', dest="token")
parser.add_option('--user', dest="user")
parser.add_option('--pwd', dest="pwd")
parser.add_option('--root', dest="root")
(options, args) = parser.parse_args()
# Set up the mexURL and token based on the arguments passed to the script
try: #pull out the mex
log.debug("options %s" % options)
if not options.mexURL:
options.mexURL = sys.argv[1]
if not options.token:
options.token = sys.argv[2]
except IndexError: #no argv were set
pass
if not options.stagingPath:
options.stagingPath = ''
# Still don't have an imgurl, but it will be set up in self.setup()
log.debug('\n\nPARAMS : %s \n\n Options: %s' % (args, options))
self.options = options
if self.validateInput():
try: #run setup and retrieve mex variables
self.setup()
except Exception, e:
log.exception("Exception during setup")
self.bqSession.fail_mex(msg="Exception during setup: %s" %
str(e))
return
try: #run module operation
self.run()
except SkeletonPythonError, e:
log.exception("Exception during run")
self.bqSession.fail_mex(msg="Exception during run: %s" %
str(e.message))
return
except Exception, e:
log.exception("Exception during run")
self.bqSession.fail_mex(msg="Exception during run: %s" %
str(e))
return
class PythonScriptWrapper(object):
def preprocess(self, bq):
"""
Pre-process the images
"""
log.info('Pre-Process Options: %s' % (self.options))
"""
1. Get the resource image
2. call hist.py with bq, log, resource_url, seeds, threshold ( bq, log, **self.options.__dict__ )
"""
image = bq.load(self.options.resource_url)
dt = self.getstrtime()
self.tiff_file = os.path.join(self.options.stagingPath,
TIFF_IMAGE_PATH, dt + '-' + image.name)
log.info("process image as %s" % (self.tiff_file))
log.info("image meta: %s" % (image))
ip = image.pixels().format('tiff')
meta = image.pixels().meta().fetch()
#meta = ET.XML(meta)
meta = bq.factory.string2etree(meta)
z = meta.findall('.//tag[@name="image_num_z"]')
z = len(z) and z[0].get('value')
zplanes = int(z)
#if int(self.options.seed_count) >= zplanes:
#raise Exception("Seed out of bounds. Please input a valid seed less than %s" % (zplanes))
# log.info('INVALID SEED BRUH %s' % (zplanes))
with open(self.tiff_file, 'wb') as f:
f.write(ip.fetch())
log.info('Executing Histogram match')
hist_match.main(testing_data_dir, hist_data_dir)
log.info('Completed Histogram match')
return
def predict(self, bq):
"""
Infer the probability map for the image
"""
log.info('Executing Inference')
"""
1. call predict.py with bq, log, resource_url, seeds, threshold ( bq, log, **self.options.__dict__ )
main(model_path, cell_hist_datadir, prob_map_datadir)
"""
predict.main(model_path, hist_data_dir, prob_map_datadir)
log.info('Completed Inference')
return
def postprocess(self, bq):
"""
Post-Process for the image.
This will select seed slice index and provide a black threshold for mask creation
"""
log.info('Executing Post-process: %s' % (self.options))
#seeds_slice_id=int(self.options.seed_count)
black_threshold = float(self.options.threshold)
min_distance = float(self.options.min_dist)
label_threshold = float(self.options.label_threshd)
#log.info('MINIMUM DISTANCE : %f, LABELS THRESHOLD: %f, BLACK THRESHOLD: %f' %(min_distance, label_threshold, black_threshold))
output_files, adj_table, points, cell_vol, coordinates, center = postprocessing.main(
bq, prob_map_datadir, results_outdir, testing_data_dir,
min_distance, label_threshold, black_threshold)
#outtable_xml_adj_table = table_service.store_array(adj_table, name='adj_table')
#outtable_xml_points = table_service.store_array(points, name='points')
#outtable_xml_cell_vol = table_service.store_array(cell_vol, name='cell_vol')
log.info('Output files: %s' % str(output_files))
return output_files, adj_table, points, cell_vol, coordinates, center
def getstrtime(self):
# format timestamp
ts = time.gmtime()
ts_str = time.strftime("%Y-%m-%dT%H-%M-%S", ts)
return ts_str
def uploadimgservice(self, bq, files):
"""
Upload mask to image_service upon post process
"""
mex_id = bq.mex.uri.split('/')[-1]
filename = os.path.basename(files[0])
log.info('Up Mex: %s' % (mex_id))
log.info('Up File: %s' % (filename))
resource = etree.Element('image',
name='ModuleExecutions/CellSegment3D/' +
filename)
t = etree.SubElement(resource,
'tag',
name="datetime",
value=self.getstrtime())
log.info('Creating upload xml data: %s ' %
str(etree.tostring(resource, pretty_print=True)))
filepath = files[
0] # os.path.join("ModuleExecutions","CellSegment3D", filename)
# use import service to /import/transfer activating import service
r = etree.XML(bq.postblob(filepath, xml=resource)).find('./')
if r is None or r.get('uri') is None:
bq.fail_mex(msg="Exception during upload results")
else:
log.info('Uploaded ID: %s, URL: %s' %
(r.get('resource_uniq'), r.get('uri')))
bq.update_mex('Uploaded ID: %s, URL: %s' %
(r.get('resource_uniq'), r.get('uri')))
self.furl = r.get('uri')
self.fname = r.get('name')
resource.set('value', self.furl)
return resource
def uploadtableservice(self, bq, files):
"""
Upload mask to image_service upon post process
"""
mex_id = bq.mex.uri.split('/')[-1]
filename = os.path.basename(files)
log.info('Up Mex: %s' % (mex_id))
log.info('Up File: %s' % (filename))
resource = etree.Element('table',
name='ModuleExecutions/CellSegment3D/' +
filename)
t = etree.SubElement(resource,
'tag',
name="datetime",
value=self.getstrtime())
log.info('Creating upload xml data: %s ' %
str(etree.tostring(resource, pretty_print=True)))
filepath = files # os.path.join("ModuleExecutions","CellSegment3D", filename)
# use import service to /import/transfer activating import service
r = etree.XML(bq.postblob(filepath, xml=resource)).find('./')
if r is None or r.get('uri') is None:
bq.fail_mex(msg="Exception during upload results")
else:
log.info('Uploaded ID: %s, URL: %s' %
(r.get('resource_uniq'), r.get('uri')))
bq.update_mex('Uploaded ID: %s, URL: %s' %
(r.get('resource_uniq'), r.get('uri')))
self.furl = r.get('uri')
self.fname = r.get('name')
resource.set('value', self.furl)
return resource
def run(self):
"""
Run Python script
"""
bq = self.bqSession
# table_service = bq.service ('table')
# call scripts
try:
bq.update_mex('Pre-process the images')
self.preprocess(bq)
except (Exception, ScriptError) as e:
log.exception("Exception during preprocess")
bq.fail_mex(msg="Exception during pre-process: %s" % str(e))
return
try:
bq.update_mex('Infer the images')
self.predict(bq)
except (Exception, ScriptError) as e:
log.exception("Exception during inference")
bq.fail_mex(msg="Exception during inference: %s" % str(e))
return
try:
bq.update_mex('Post process the images')
self.outfiles, self.outtable_xml_adj_table, self.outtable_xml_points, self.outtable_xml_cell_vol, self.outtable_xml_coordinates, self.outtable_xml_center = self.postprocess(
bq)
except (Exception, ScriptError) as e:
log.exception("Exception during post-process")
bq.fail_mex(msg="Exception during post-process: %s" % str(e))
return
try:
bq.update_mex('Uploading Mask result')
self.resimage = self.uploadimgservice(bq, self.outfiles)
bq.update_mex('Uploading Table result')
self.restable = self.uploadtableservice(
bq, 'source/hdf/PlantCellSegmentation.h5')
except (Exception, ScriptError) as e:
log.exception("Exception during upload result")
bq.fail_mex(msg="Exception during upload result: %s" % str(e))
return
log.info('Completed the workflow: %s' % (self.resimage.get('value')))
out_imgxml = """<tag name="Segmentation" type="image" value="%s">
<template>
<tag name="label" value="Output image" />
</template>
</tag>""" % (str(self.resimage.get('value')))
# format timestamp
ts = time.gmtime()
ts_str = time.strftime("%Y-%m-%d %H:%M:%S", ts)
# outputs = predict( bq, log, **self.options.__dict__ )
#outtable_xml = table_service.store_array(maxMisorient, name='maxMisorientData')
out_xml = """<tag name="Metadata">
<tag name="Adjacency Table" type="string" value="%s"/>
<tag name="Three-way Conjuction Points" type="string" value="%s"/>
<tag name="Cell Volume" type="string" value="%s"/>
<tag name="Surface Coordinates" type="string" value="%s"/>
<tag name="Cell Center" type="string" value="%s"/>
<tag name="Output Table" type="resource" value="%s"/>
</tag>""" % (
str(self.outtable_xml_adj_table), str(self.outtable_xml_points),
str(self.outtable_xml_cell_vol), str(
self.outtable_xml_coordinates), str(
self.outtable_xml_center), self.restable.get('value'))
outputs = [out_imgxml, out_xml]
log.debug(outputs)
# save output back to BisQue
for output in outputs:
self.output_resources.append(output)
def setup(self):
"""
Pre-run initialization
"""
self.output_resources.append(output)
def setup(self):
"""
Pre-run initialization
"""
self.bqSession.update_mex('Initializing...')
self.mex_parameter_parser(self.bqSession.mex.xmltree)
self.output_resources = []
def teardown(self):
"""
Post the results to the mex xml
"""
self.bqSession.update_mex('Returning results')
outputTag = etree.Element('tag', name='outputs')
for r_xml in self.output_resources:
if isinstance(r_xml, basestring):
r_xml = etree.fromstring(r_xml)
res_type = r_xml.get('type', None) or r_xml.get(
'resource_type', None) or r_xml.tag
# append reference to output
if res_type in ['table', 'image']:
outputTag.append(r_xml)
#etree.SubElement(outputTag, 'tag', name='output_table' if res_type=='table' else 'output_image', type=res_type, value=r_xml.get('uri',''))
else:
outputTag.append(r_xml)
#etree.SubElement(outputTag, r_xml.tag, name=r_xml.get('name', '_'), type=r_xml.get('type', 'string'), value=r_xml.get('value', ''))
log.debug('Output Mex results: %s' %
(etree.tostring(outputTag, pretty_print=True)))
self.bqSession.finish_mex(tags=[outputTag])
def mex_parameter_parser(self, mex_xml):
"""
Parses input of the xml and add it to options attribute (unless already set)
@param: mex_xml
"""
# inputs are all non-"script_params" under "inputs" and all params under "script_params"
mex_inputs = mex_xml.xpath(
'tag[@name="inputs"]/tag[@name!="script_params"] | tag[@name="inputs"]/tag[@name="script_params"]/tag'
)
if mex_inputs:
for tag in mex_inputs:
if tag.tag == 'tag' and tag.get(
'type',
'') != 'system-input': #skip system input values
if not getattr(self.options, tag.get('name', ''), None):
log.debug('Set options with %s as %s' %
(tag.get('name', ''), tag.get('value', '')))
setattr(self.options, tag.get('name', ''),
tag.get('value', ''))
else:
log.debug('No Inputs Found on MEX!')
def validate_input(self):
"""
Check to see if a mex with token or user with password was provided.
@return True is returned if validation credention was provided else
False is returned
"""
if (self.options.mexURL
and self.options.token): #run module through engine service
return True
if (self.options.user and self.options.pwd and
self.options.root): #run module locally (note: to test module)
return True
log.debug('Insufficient options or arguments to start this module')
return False
def main(self):
parser = optparse.OptionParser()
parser.add_option('--mex_url', dest="mexURL")
parser.add_option('--module_dir', dest="modulePath")
parser.add_option('--staging_path', dest="stagingPath")
parser.add_option('--bisque_token', dest="token")
parser.add_option('--user', dest="user")
parser.add_option('--pwd', dest="pwd")
parser.add_option('--root', dest="root")
(options, args) = parser.parse_args()
# Logging initializations
fh = logging.FileHandler('scriptrun.log', mode='a')
fh.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'[%(asctime)s] %(levelname)8s --- %(message)s ' +
'(%(filename)s:%(lineno)s)',
datefmt='%Y-%m-%d %H:%M:%S')
fh.setFormatter(formatter)
log.addHandler(fh)
try: #pull out the mex
if not options.mexURL:
options.mexURL = sys.argv[-2]
if not options.token:
options.token = sys.argv[-1]
except IndexError: #no argv were set
pass
if not options.stagingPath:
options.stagingPath = ''
# Options configuration
log.debug('PARAMS : %s Options: %s' % (args, options))
self.options = options
if self.validate_input():
#initalizes if user and password are provided
if (self.options.user and self.options.pwd and self.options.root):
self.bqSession = BQSession().init_local(
self.options.user,
self.options.pwd,
bisque_root=self.options.root)
self.options.mexURL = self.bqSession.mex.uri
#initalizes if mex and mex token is provided
elif (self.options.mexURL and self.options.token):
self.bqSession = BQSession().init_mex(self.options.mexURL,
self.options.token)
else:
raise ScriptError(
'Insufficient options or arguments to start this module')
# Setup the mex and sessions
try:
self.setup()
except Exception as e:
log.exception("Exception during setup")
self.bqSession.fail_mex(msg="Exception during setup: %s" %
str(e))
return
# Execute the module functionality
try:
self.run()
except (Exception, ScriptError) as e:
log.exception("Exception during run")
self.bqSession.fail_mex(msg="Exception during run: %s" %
str(e))
return
try:
self.teardown()
except (Exception, ScriptError) as e:
log.exception("Exception during teardown")
self.bqSession.fail_mex(msg="Exception during teardown: %s" %
str(e))
return
self.bqSession.close()
log.debug('Session Close')
class PythonScriptWrapper(object):
def run(self):
"""
Run Python script
"""
bq = self.bqSession
# call script
outputs = predict(bq, log, **self.options.__dict__)
# save output back to BisQue
for output in outputs:
self.output_resources.append(output)
def setup(self):
"""
Pre-run initialization
"""
self.bqSession.update_mex('Initializing...')
self.mex_parameter_parser(self.bqSession.mex.xmltree)
self.output_resources = []
def teardown(self):
"""
Post the results to the mex xml
"""
self.bqSession.update_mex('Returning results')
outputTag = etree.Element('tag', name='outputs')
for r_xml in self.output_resources:
if isinstance(r_xml, basestring):
r_xml = etree.fromstring(r_xml)
res_type = r_xml.get('type', None) or r_xml.get(
'resource_type', None) or r_xml.tag
# append reference to output
if res_type in ['table', 'image']:
outputTag.append(r_xml)
#etree.SubElement(outputTag, 'tag', name='output_table' if res_type=='table' else 'output_image', type=res_type, value=r_xml.get('uri',''))
else:
outputTag.append(r_xml)
#etree.SubElement(outputTag, r_xml.tag, name=r_xml.get('name', '_'), type=r_xml.get('type', 'string'), value=r_xml.get('value', ''))
self.bqSession.finish_mex(tags=[outputTag])
def mex_parameter_parser(self, mex_xml):
"""
Parses input of the xml and add it to options attribute (unless already set)
@param: mex_xml
"""
# inputs are all non-"script_params" under "inputs" and all params under "script_params"
mex_inputs = mex_xml.xpath(
'tag[@name="inputs"]/tag[@name!="script_params"] | tag[@name="inputs"]/tag[@name="script_params"]/tag'
)
if mex_inputs:
for tag in mex_inputs:
if tag.tag == 'tag' and tag.get(
'type',
'') != 'system-input': #skip system input values
if not getattr(self.options, tag.get('name', ''), None):
log.debug('Set options with %s as %s' %
(tag.get('name', ''), tag.get('value', '')))
setattr(self.options, tag.get('name', ''),
tag.get('value', ''))
else:
log.debug('No Inputs Found on MEX!')
def validate_input(self):
"""
Check to see if a mex with token or user with password was provided.
@return True is returned if validation credention was provided else
False is returned
"""
if (self.options.mexURL
and self.options.token): #run module through engine service
return True
if (self.options.user and self.options.pwd and
self.options.root): #run module locally (note: to test module)
return True
log.debug('Insufficient options or arguments to start this module')
return False
def main(self):
parser = optparse.OptionParser()
parser.add_option('--mex_url', dest="mexURL")
parser.add_option('--module_dir', dest="modulePath")
parser.add_option('--staging_path', dest="stagingPath")
parser.add_option('--bisque_token', dest="token")
parser.add_option('--user', dest="user")
parser.add_option('--pwd', dest="pwd")
parser.add_option('--root', dest="root")
(options, args) = parser.parse_args()
fh = logging.FileHandler('scriptrun.log', mode='a')
fh.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'[%(asctime)s] %(levelname)8s --- %(message)s ' +
'(%(filename)s:%(lineno)s)',
datefmt='%Y-%m-%d %H:%M:%S')
fh.setFormatter(formatter)
log.addHandler(fh)
try: #pull out the mex
if not options.mexURL:
options.mexURL = sys.argv[-2]
if not options.token:
options.token = sys.argv[-1]
except IndexError: #no argv were set
pass
if not options.stagingPath:
options.stagingPath = ''
log.info('\n\nPARAMS : %s \n\n Options: %s' % (args, options))
self.options = options
if self.validate_input():
#initalizes if user and password are provided
if (self.options.user and self.options.pwd and self.options.root):
self.bqSession = BQSession().init_local(
self.options.user,
self.options.pwd,
bisque_root=self.options.root)
self.options.mexURL = self.bqSession.mex.uri
#initalizes if mex and mex token is provided
elif (self.options.mexURL and self.options.token):
self.bqSession = BQSession().init_mex(self.options.mexURL,
self.options.token)
else:
raise ScriptError(
'Insufficient options or arguments to start this module')
try:
self.setup()
except Exception as e:
log.exception("Exception during setup")
self.bqSession.fail_mex(msg="Exception during setup: %s" %
str(e))
return
try:
self.run()
except (Exception, ScriptError) as e:
log.exception("Exception during run")
self.bqSession.fail_mex(msg="Exception during run: %s" %
str(e))
return
try:
self.teardown()
except (Exception, ScriptError) as e:
log.exception("Exception during teardown")
self.bqSession.fail_mex(msg="Exception during teardown: %s" %
str(e))
return
self.bqSession.close()
class PythonScriptWrapper(object):
def preprocess(self, bq):
log.info('Options: %s' % (self.options))
"""
1. Get the resource image
"""
image = bq.load(self.options.resource_url)
dt = self.getstrtime()
self.img_name = dt + '-' + image.name
self.in_file = os.path.join(self.options.stagingPath, IMAGE_PATH,
self.img_name)
log.info("process image as %s" % (self.in_file))
log.info("image meta: %s" % (image))
ip = image.pixels() #.format('jpg')
with open(self.in_file, 'wb') as f:
f.write(ip.fetch())
return
def detect(self, bq):
"""
2. Infer the mask for the image & write file
"""
log.info('Executing Inference')
image, r = detect(self.in_file)
log.info('Completed Inference')
self.out_file = os.path.join(self.options.stagingPath,
"mask_" + self.img_name)
fsave(image, r, outfile=self.out_file)
return
def getstrtime(self):
# format timestamp
ts = time.gmtime()
ts_str = time.strftime("%Y-%m-%dT%H-%M-%S", ts)
return ts_str
def uploadimgservice(self, bq, out_file):
"""
3. Upload mask to image_service upon post process
"""
mex_id = bq.mex.uri.split('/')[-1]
filename = os.path.basename(out_file)
log.info('Up Mex: %s' % (mex_id))
log.info('Up File: %s' % (filename))
resource = etree.Element('image',
name='ModuleExecutions/MaskRCNN/' + filename)
t = etree.SubElement(resource,
'tag',
name="datetime",
value=self.getstrtime())
log.info('Creating upload xml data: %s ' %
str(etree.tostring(resource, pretty_print=True)))
filepath = out_file
r = etree.XML(bq.postblob(filepath, xml=resource)).find('./')
if r is None or r.get('uri') is None:
bq.fail_mex(msg="Exception during upload results")
else:
log.info('Uploaded ID: %s, URL: %s' %
(r.get('resource_uniq'), r.get('uri')))
bq.update_mex('Uploaded ID: %s, URL: %s' %
(r.get('resource_uniq'), r.get('uri')))
self.furl = r.get('uri')
self.fname = r.get('name')
resource.set('value', self.furl)
return resource
def run(self):
"""
Run Python script
"""
bq = self.bqSession
# call scripts
try:
bq.update_mex('Pre-process the images')
self.preprocess(bq)
except (Exception, ScriptError) as e:
log.exception("Exception during preprocess")
bq.fail_mex(msg="Exception during pre-process: %s" % str(e))
return
try:
bq.update_mex('Infer the images')
self.detect(bq)
except (Exception, ScriptError) as e:
log.exception("Exception during inference")
bq.fail_mex(msg="Exception during inference: %s" % str(e))
return
try:
bq.update_mex('Uploading mask result')
self.resimage = self.uploadimgservice(bq, self.out_file)
except (Exception, ScriptError) as e:
log.exception("Exception during upload result")
bq.fail_mex(msg="Exception during upload result: %s" % str(e))
return
log.info('Completed the workflow: %s' % (self.resimage.get('value')))
out_imgxml = """<tag name="Segmentation" type="image" value="%s">
<template>
<tag name="label" value="Output image" />
</template>
</tag>""" % (str(self.resimage.get('value')))
# format timestamp
ts = time.gmtime()
ts_str = time.strftime("%Y-%m-%d %H:%M:%S", ts)
# outputs = predict( bq, log, **self.options.__dict__ )
out_xml = """<tag name="Metadata">
<tag name="mask_name" type="string" value="%s"/>
<tag name="mask_url" type="string" value="%s"/>
</tag>""" % (str(
self.resimage.get('name')), str(self.resimage.get('value')))
outputs = [out_imgxml, out_xml]
log.debug(outputs)
# save output back to BisQue
for output in outputs:
self.output_resources.append(output)
def setup(self):
"""
Pre-run initialization
"""
self.output_resources.append(output)
def setup(self):
"""
Pre-run initialization
"""
self.bqSession.update_mex('Initializing...')
self.mex_parameter_parser(self.bqSession.mex.xmltree)
self.output_resources = []
def teardown(self):
"""
Post the results to the mex xml
"""
self.bqSession.update_mex('Returning results')
outputTag = etree.Element('tag', name='outputs')
for r_xml in self.output_resources:
if isinstance(r_xml, str):
r_xml = etree.fromstring(r_xml)
res_type = r_xml.get('type', None) or r_xml.get(
'resource_type', None) or r_xml.tag
# append reference to output
if res_type in ['table', 'image']:
outputTag.append(r_xml)
else:
outputTag.append(r_xml)
log.debug('Output Mex results: %s' %
(etree.tostring(outputTag, pretty_print=True)))
self.bqSession.finish_mex(tags=[outputTag])
def mex_parameter_parser(self, mex_xml):
"""
Parses input of the xml and add it to options attribute (unless already set)
@param: mex_xml
"""
# inputs are all non-"script_params" under "inputs" and all params under "script_params"
mex_inputs = mex_xml.xpath(
'tag[@name="inputs"]/tag[@name!="script_params"] | tag[@name="inputs"]/tag[@name="script_params"]/tag'
)
if mex_inputs:
for tag in mex_inputs:
if tag.tag == 'tag' and tag.get(
'type',
'') != 'system-input': #skip system input values
if not getattr(self.options, tag.get('name', ''), None):
log.debug('Set options with %s as %s' %
(tag.get('name', ''), tag.get('value', '')))
setattr(self.options, tag.get('name', ''),
tag.get('value', ''))
else:
log.debug('No Inputs Found on MEX!')
def validate_input(self):
"""
Check to see if a mex with token or user with password was provided.
@return True is returned if validation credention was provided else
False is returned
"""
if (self.options.mexURL
and self.options.token): #run module through engine service
return True
if (self.options.user and self.options.pwd and
self.options.root): #run module locally (note: to test module)
return True
log.debug('Insufficient options or arguments to start this module')
return False
def main(self):
parser = optparse.OptionParser()
parser.add_option('--mex_url', dest="mexURL")
parser.add_option('--module_dir', dest="modulePath")
parser.add_option('--staging_path', dest="stagingPath")
parser.add_option('--bisque_token', dest="token")
parser.add_option('--user', dest="user")
parser.add_option('--pwd', dest="pwd")
parser.add_option('--root', dest="root")
(options, args) = parser.parse_args()
# Logging initializations
fh = logging.FileHandler('scriptrun.log', mode='a')
fh.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'[%(asctime)s] %(levelname)8s --- %(message)s ' +
'(%(filename)s:%(lineno)s)',
datefmt='%Y-%m-%d %H:%M:%S')
fh.setFormatter(formatter)
log.addHandler(fh)
try: #pull out the mex
if not options.mexURL:
options.mexURL = sys.argv[-2]
if not options.token:
options.token = sys.argv[-1]
except IndexError: #no argv were set
pass
if not options.stagingPath:
options.stagingPath = ''
# Options configuration
log.debug('PARAMS : %s Options: %s' % (args, options))
self.options = options
if self.validate_input():
#initalizes if user and password are provided
if (self.options.user and self.options.pwd and self.options.root):
self.bqSession = BQSession().init_local(
self.options.user,
self.options.pwd,
bisque_root=self.options.root)
self.options.mexURL = self.bqSession.mex.uri
#initalizes if mex and mex token is provided
elif (self.options.mexURL and self.options.token):
self.bqSession = BQSession().init_mex(self.options.mexURL,
self.options.token)
else:
raise ScriptError(
'Insufficient options or arguments to start this module')
# Setup the mex and sessions
try:
self.setup()
except Exception as e:
log.exception("Exception during setup")
self.bqSession.fail_mex(msg="Exception during setup: %s" %
str(e))
return
# Execute the module functionality
try:
self.run()
except (Exception, ScriptError) as e:
log.exception("Exception during run")
self.bqSession.fail_mex(msg="Exception during run: %s" %
str(e))
return
try:
self.teardown()
except (Exception, ScriptError) as e:
log.exception("Exception during teardown")
self.bqSession.fail_mex(msg="Exception during teardown: %s" %
str(e))
return
self.bqSession.close()
log.debug('Session Close')
class Botanicam(object):
"""
Botanicam Model
"""
def mex_parameter_parser(self, mex_xml):
"""
Parses input of the xml and add it to BotanicamTrainer's options attribute
@param: mex_xml
"""
mex_inputs = mex_xml.xpath('tag[@name="inputs"]')
if mex_inputs:
for tag in mex_inputs[0]:
if tag.tag == 'tag' and tag.attrib['type'] != 'system-input':
log.debug('Set options with %s as %s' %
(tag.attrib['name'], tag.attrib['value']))
setattr(self.options, tag.attrib['name'],
tag.attrib['value'])
else:
log.debug('BotanicamFS: No Inputs Found on MEX!')
def validateInput(self):
"""
Parses input of the xml and add it to Botanicam's options attribute
@param: mex_xml
"""
if (self.options.mexURL
and self.options.token): #run module through engine service
return True
if (self.options.user and self.options.pwd and
self.options.root): #run module locally (note: to test module)
return True
log.debug(
'Botanicam: Insufficient options or arguments to start this module'
)
return False
def setup(self):
"""
Fetches the mex, appends input_configurations to the option
attribute of Botanicam and looks up the model on bisque to
classify the provided resource.
"""
log.debug('Initializing Mex...')
if (self.options.user and self.options.pwd and self.options.root):
self.bqSession = BQSession().init_local(
self.options.user,
self.options.pwd,
bisque_root=self.options.root)
self.options.mexURL = self.bqSession.mex.uri
elif (self.options.mexURL and self.options.token):
self.bqSession = BQSession().init_mex(self.options.mexURL,
self.options.token)
else:
return
self.mex_parameter_parser(self.bqSession.mex.xmltree)
#finds and opens model file
self.bqSession.update_mex('Initializing Classification Model...')
log.debug('Forming Feature Requests...')
#no options currently
#combo = mex_xml.xpath('tag[@name="plant_part"]/@value')[0]
combo = 'bush'
if combo:
if combo == 'bush':
MODEL_QUERY[
'tag_query'] = '"module_identifier":"Botanicam" AND "Classification Method":"Bush Descriptor"'
elif combo == 'leaf':
MODEL_QUERY[
'tag_query'] = '"module_identifier":"Botanicam" AND "Classification Method":"Leaf Descriptor"'
else:
raise BotanicamError(
'The incorrect model type was found -> Model Type: %s' %
combo)
else:
raise BotanicamError('No model type was choosen')
query_xml = self.bqSession.fetchxml('/data_service/file',
**MODEL_QUERY)
self.options.model_url = None
if len(query_xml) > 0:
try:
model_url = query_xml[0].attrib['uri']
self.options.model_url = model_url
log.debug('Fetching Model @ %s' % model_url)
self.model_xml = self.bqSession.fetchxml(model_url,
view='deep')
self.model_path = os.path.join(self.options.stagingPath,
'model')
model = self.bqSession.load(model_url)
model_url = self.bqSession.service_url(
'blob_service', path=model.resource_uniq)
self.bqSession.fetchblob(model_url,
path=self.model_path + '.zip')
with zipfile.ZipFile(self.model_path + '.zip') as dirzip:
dirzip.extractall(self.model_path)
except BQCommError:
raise BotanicamError(
'Model file was not found! Ask admin to set the correct model file'
)
else: #run demo classifier model store in the module
raise BotanicamError(
'No model file was found. Ask your admin to train a new model with \
the Botanicam Trainer.')
self.bqSession.update_mex('Initialized...')
log.debug(
'Botanicam: image URL: %s, mexURL: %s, stagingPath: %s, token: %s'
% (self.options.image_url, self.options.mexURL,
self.options.stagingPath, self.options.token))
def run(self):
"""
The core of the Botanicam Module
Requests features on the image provided. Classifies each tile
and picks a majority among the tiles.
"""
#parse requests
self.bqSession.update_mex('Calculating Features...')
log.debug('Forming Feature Requests...')
#get rectanle gobjects for roi
r_xml = self.bqSession.fetchxml(self.options.mexURL, view='deep')
rectangles = r_xml.xpath(
'//tag[@name="inputs"]/tag[@name="image_url"]/gobject[@name="roi"]/rectangle'
)
image_xml = self.bqSession.fetchxml(self.options.image_url)
image_url = self.bqSession.service_url(
'image_service', path=image_xml.attrib['resource_uniq'])
if rectangles: #On chooses the first rectangle
#construct operation node
x1 = int(float(rectangles[0][0].attrib['x']))
y1 = int(float(rectangles[0][0].attrib['y']))
x2 = int(float(rectangles[0][1].attrib['x']))
y2 = int(float(rectangles[0][1].attrib['y']))
log.debug('Adding Crop: roi=%s,%s,%s,%s' % (x1, y1, x2, y2))
image_url = self.bqSession.c.prepare_url(image_url,
roi='%s,%s,%s,%s' %
(x1, y1, x2, y2))
try:
feature_vectors = extract_bush_feature(self.bqSession, image_url)
except FeatureError as e:
raise BotanicamError(str(e))
#parse features
self.bqSession.update_mex('Classifying Results...')
log.debug('Classifying Results...')
results = []
pca = joblib.load(os.path.join(self.model_path, 'pca_model'))
clf = joblib.load(os.path.join(self.model_path, 'svm_model'))
for f in feature_vectors:
f_norm = pca.transform(f)
results.append(int(clf.predict(f_norm)))
class_count = np.bincount(np.array(results))
self.class_number = np.argmax(class_count)
self.confidence = float(
class_count[self.class_number]) / np.sum(class_count)
log.debug('Found Class %s' % str(self.class_number))
def teardown(self):
"""
Posting results to the mex
"""
self.bqSession.update_mex('Returning results...')
log.debug('Returning results...')
tag_list = self.model_xml.xpath(
'tag[@name="Classes"]/tag[@value="%s"]' %
str(self.class_number))[0]
outputTag = etree.Element('tag', name='outputs')
outputSubTag = etree.SubElement(outputTag, 'tag', name='summary')
if self.options.model_url:
etree.SubElement(outputSubTag,
'tag',
name='Model File',
value=self.options.model_url,
type='url')
else:
etree.SubElement(outputSubTag,
'tag',
name='Model File',
value='Internal Model File')
etree.SubElement(outputSubTag,
'tag',
name='Class',
value=str(self.class_number))
query = []
for t in tag_list:
etree.SubElement(outputSubTag,
'tag',
name=t.attrib['name'],
value=t.attrib['value'])
query.append('"%s":"%s"' % (t.attrib['name'], t.attrib['value']))
query = ' & '.join(query)
etree.SubElement(outputSubTag,
'tag',
name='confidence',
value=str(self.confidence))
etree.SubElement(outputTag,
'tag',
name='similar_images',
value=query,
type='browser')
self.bqSession.finish_mex(tags=[outputTag])
log.debug('FINISHED')
self.bqSession.close()
def main(self):
"""
The main function that runs everything
"""
log.debug('sysargv : %s' % sys.argv)
parser = OptionParser()
parser.add_option('--image_url', dest="image_url")
parser.add_option('--mex_url', dest="mexURL")
parser.add_option('--module_dir', dest="modulePath")
parser.add_option('--staging_path', dest="stagingPath")
parser.add_option('--bisque_token', dest="token")
parser.add_option('--user', dest="user")
parser.add_option('--pwd', dest="pwd")
parser.add_option('--root', dest="root")
(options, args) = parser.parse_args()
try: #pull out the mex
if not options.mexURL:
options.mexURL = sys.argv[1]
if not options.token:
options.token = sys.argv[2]
except IndexError: #no argv were set
pass
if not options.stagingPath:
options.stagingPath = ''
log.debug('\n\nPARAMS : %s \n\n Options: %s' % (args, options))
self.options = options
if self.validateInput():
try: #run setup and retrieve mex variables
self.setup()
except Exception, e:
log.exception("Exception during setup")
self.bqSession.fail_mex(msg="Exception during setup: %s" %
str(e))
return
try: #run module operation
self.run()
except BotanicamError, e:
log.exception("Exception during run")
self.bqSession.fail_mex(msg="Exception during run: %s" %
str(e.message))
return
except Exception, e:
log.exception("Exception during run")
self.bqSession.fail_mex(msg="Exception during run: %s" %
str(e))
return
class ImageJ(object):
"""
ImageJ Module
"""
def mex_parameter_parser(self, mex_xml):
"""
Parses input of the xml and add it to ImageJ's options attribute (unless already set)
@param: mex_xml
"""
# inputs are all non-"pipeline params" under "inputs" and all params under "pipeline_params"
mex_inputs = mex_xml.xpath(
'tag[@name="inputs"]/tag[@name!="pipeline_params"] | tag[@name="inputs"]/tag[@name="pipeline_params"]/tag'
)
if mex_inputs:
for tag in mex_inputs:
if tag.tag == 'tag' and tag.get(
'type',
'') != 'system-input': #skip system input values
if not getattr(self.options, tag.get('name', ''), None):
log.debug('Set options with %s as %s' %
(tag.get('name', ''), tag.get('value', '')))
setattr(self.options, tag.get('name', ''),
tag.get('value', ''))
else:
log.debug('ImageJ: No Inputs Found on MEX!')
def validate_input(self):
"""
Check to see if a mex with token or user with password was provided.
@return True is returned if validation credention was provided else
False is returned
"""
if (self.options.mexURL
and self.options.token): #run module through engine service
return True
if (self.options.user and self.options.pwd and
self.options.root): #run module locally (note: to test module)
return True
log.debug(
'ImageJ: Insufficient options or arguments to start this module')
return False
def setup(self):
"""
Fetches the mex and appends input_configurations to the option
attribute of ImageJ
"""
self.bqSession.update_mex('Initializing...')
self.mex_parameter_parser(self.bqSession.mex.xmltree)
self.output_resources = []
self.ppops = None
self.ppops_url = None
def run(self):
"""
The core of the ImageJ runner
"""
module_time = datetime.now()
#retrieve tags
self.bqSession.update_mex('Extracting properties')
#type check
image_resource = self.bqSession.fetchxml(self.options.InputFile)
if image_resource.tag != 'image':
raise IJError("trying to run ImageJ on non-image resource")
# run prerun operations
filelist = self._run_prerun_ops(module_time,
pipeline_url=self.options.pipeline_url,
input_xml=image_resource)
# create pipeline with correct parameters
pipeline_params = self.bqSession.mex.xmltree.xpath(
'tag[@name="inputs"]/tag[@name="pipeline_params"]/tag')
params = {}
for tag in pipeline_params:
params[tag.get('name', '')] = getattr(self.options,
tag.get('name', ''))
pipeline_file, err_file = self._instantiate_pipeline(
pipeline_url=self.options.pipeline_url, params=params)
if not pipeline_file:
raise IJError("trying to run incompatible ImageJ pipeline")
# run ImageJ on the pipeline
self.bqSession.update_mex('Running ImageJ')
log.debug('run: ImageJ-linux64 -batch %s on image %s', pipeline_file,
filelist)
res = 1
with open(err_file, 'w') as fo:
# start virtual X server
vfb = subprocess.Popen(['Xvfb', ':0'])
# TODO: wait for Xvfb to show up?
time.sleep(1)
res = subprocess.call(
[
'/module/Fiji.app/ImageJ-linux64', # '--headless',
'-macro',
pipeline_file
],
env={'DISPLAY': ':0.0'}, # use virtual X server
stderr=fo,
stdout=fo)
# stop virtual X server
vfb.terminate()
log.debug("ImageJ returned: %s", str(res))
# TODO: detect error somehow!!!
# if res > 0:
# err_msg = 'pipeline execution failed\n'
# with open(err_file, 'r') as fo:
# err_msg += ''.join(fo.readlines())
# if len(err_msg) > 1024:
# err_msg = err_msg[:512] + '...' + err_msg[-512:]
# raise IJError(err_msg)
# run postrun operations
self.output_resources = self._run_postrun_ops(
module_time,
pipeline_url=self.options.pipeline_url,
output_name='output_image')
def _cache_ppops(self, pipeline_url):
if not self.ppops or self.ppops_url != pipeline_url:
pipeline_path = urlparse.urlsplit(pipeline_url).path.split('/')
pipeline_uid = pipeline_path[1] if is_uniq_code(
pipeline_path[1]) else pipeline_path[2]
url = self.bqSession.service_url('pipeline',
path='/'.join([pipeline_uid] +
['ppops:imagej']))
self.ppops = json.loads(self.bqSession.c.fetch(url))
self.ppops_url = pipeline_url
def _run_prerun_ops(self, module_time, pipeline_url, input_xml):
"""
Perform any operations necessary before the pipeline runs (e.g., extract image channels) and return filelist
"""
self._cache_ppops(pipeline_url)
pre_ops = self.ppops['PreOps']
input_files = []
for op in pre_ops:
input_files += self._run_single_op(module_time, op, input_xml)
filelist_file = os.path.join(self.options.stagingPath, 'filelist.txt')
with open(filelist_file, 'w') as fo:
for input_file in input_files:
fo.write(input_file + '\n')
return input_files
def _run_postrun_ops(self, module_time, pipeline_url, output_name):
"""
Perform any operations necessary after the pipeline finished (e.g., upload result tables) and return created resources
"""
self._cache_ppops(pipeline_url)
post_ops = self.ppops['PostOps']
created_resources = []
for op in post_ops:
created_resources += self._run_single_op(module_time, op,
output_name)
return created_resources
def _run_single_op(self,
module_time,
op,
input_xml=None,
output_name=None):
"""
Perform single pre/post operation and return list of files or resources generated
"""
# replace special placeholders
if 'id' in op and op['id'] == '@INPUT':
op['id'] = input_xml.get('resource_uniq')
if 'name' in op and op['name'] == '@OUTPUT':
op['name'] = output_name
if 'filename' in op and op['filename'] == '@OUTPUT':
op['filename'] = output_name + '.tif'
res = []
if op['service'] == 'image_service':
# perform image_service operation
log.debug("RUNOP %s" % str(op))
url = self.bqSession.service_url('image_service',
path=op['id'] + op['ops'])
# TODO: don't read image into memory!!!
image_data = self.bqSession.c.fetch(url)
if image_data:
image_file = os.path.join(self.options.stagingPath,
op['filename'])
with open(image_file, 'w') as fo:
fo.write(image_data)
res += [image_file]
elif op['service'] == 'data_service':
# perform data_service operation
doc = self.bqSession.fetchxml(url=op['id'], view='deep,clean')
csv_file = os.path.join(self.options.stagingPath, op['filename'])
matches = doc.xpath(op['ops'])
if len(matches) > 0:
with open(csv_file, 'w') as fo:
for match_id in xrange(0, len(matches)):
match = matches[match_id]
line = '\t'.join([
match.get(attr_name, '')
for attr_name in op['attrs']
]) + '\n'
fo.write(line)
res += [csv_file]
elif op['service'] == 'postblob':
filename = op['filename']
with open(op['name']) as namef:
resname = namef.readlines()[0].rstrip('\n')
# upload image or table (check op['type'])
dt = module_time.strftime('%Y%m%dT%H%M%S')
final_output_file = "ModuleExecutions/ImageJ/%s/%s" % (dt, resname)
cl_model = etree.Element('resource',
resource_type=op['type'],
name=final_output_file)
# module identifier (a descriptor to be found by the ImageJ model)
etree.SubElement(cl_model,
'tag',
name='module_identifier',
value='ImageJ')
# hdf filename
etree.SubElement(cl_model,
'tag',
name='OutputFile',
value=final_output_file)
#description
etree.SubElement(cl_model,
'tag',
name='description',
value='output from ImageJ Module')
# post blob
output_file = os.path.join(self.options.stagingPath, filename)
if os.path.isfile(output_file):
resource = self.bqSession.postblob(output_file, xml=cl_model)
resource_xml = etree.fromstring(resource)
res += [resource_xml[0]]
elif op['service'] == 'posttag':
with open(op['name']) as namef:
lines = namef.readlines()
resname = lines[0].rstrip('\n')
resval = lines[1].rstrip('\n')
res += [etree.Element('tag', name=resname, value=resval)]
elif op['service'] == 'postpolygon':
# add polygon gobject to mex
# Read object measurements from csv and write to gobjects
(header, records) = self._readCSV(
os.path.join(self.options.stagingPath, op['filename']))
if header is not None:
parentGObject = etree.Element('gobject',
type='detected shapes',
name='detected shapes')
xcoords = []
ycoords = []
for i in range(len(records)):
curr_id = int(records[i][header.index(op['id_col'])])
xcoords.append(
float(records[i][header.index(op['x_coord'])]))
ycoords.append(
float(records[i][header.index(op['y_coord'])]))
if i == len(records) - 1 or curr_id != int(
records[i + 1][header.index(op['id_col'])]):
# new polygon starts => save current one
xcoords, ycoords = _simplify_polygon(xcoords, ycoords)
shape = self._get_polygon_elem(name=str(i),
xcoords=xcoords,
ycoords=ycoords,
label=op['label'],
color=op['color'])
if shape:
parentGObject.append(shape)
xcoords = []
ycoords = []
res += [parentGObject]
return res
def _instantiate_pipeline(self, pipeline_url, params):
"""
instantiate ImageJ pipeline file with provided parameters
"""
pipeline_path = urlparse.urlsplit(pipeline_url).path.split('/')
pipeline_uid = pipeline_path[1] if is_uniq_code(
pipeline_path[1]) else pipeline_path[2]
url = self.bqSession.service_url(
'pipeline',
path='/'.join(
[pipeline_uid] +
["setvar:%s|%s" % (tag, params[tag])
for tag in params] + ['exbsteps:imagej']),
query={'format': 'imagej'})
pipeline = self.bqSession.c.fetch(url)
if not pipeline:
# bad pipeline
return None, None
out_pipeline_file = os.path.join(self.options.stagingPath, 'macro.ijm')
out_error_file = os.path.join(self.options.stagingPath, 'ij_error.txt')
with open(out_pipeline_file, 'w') as fo:
fo.write(pipeline)
return out_pipeline_file, out_error_file
def teardown(self):
"""
Post the results to the mex xml.
"""
self.bqSession.update_mex('Returning results')
outputTag = etree.Element('tag', name='outputs')
for r_xml in self.output_resources:
res_type = r_xml.get('type', None) or r_xml.get(
'resource_type', None) or r_xml.tag
if res_type == 'detected shapes':
# r_xml is a set of gobjects => append to output inside image tag
image_resource = self.bqSession.fetchxml(
self.options.InputFile)
image_elem = etree.SubElement(outputTag,
'tag',
name=image_resource.get('name'),
type='image',
value=image_resource.get('uri'))
image_elem.append(r_xml)
elif res_type == 'tag':
# simple tag => append to output as is
etree.SubElement(outputTag,
'tag',
name=r_xml.get('name'),
value=r_xml.get('value'))
else:
# r_xml is some other resource (e.g., image or table) => append reference to output
etree.SubElement(outputTag,
'tag',
name='output_table'
if res_type == 'table' else 'output_image',
type=res_type,
value=r_xml.get('uri', ''))
self.bqSession.finish_mex(tags=[outputTag])
def _get_polygon_elem(self, name, **params):
shape = etree.Element('gobject', name=name, type=params.get('label'))
res = etree.SubElement(shape, 'polygon')
xcoords = params.get('xcoords')
ycoords = params.get('ycoords')
try:
for i in range(len(xcoords)):
x = xcoords[i]
y = ycoords[i]
etree.SubElement(res, 'vertex', x=str(x), y=str(y))
etree.SubElement(res,
'tag',
name='color',
value=params.get('color', '#FF0000'),
type='color')
except KeyError:
return None
except ValueError:
return None
return shape
def _readCSV(self, fileName):
if os.path.exists(fileName) == False:
return (None, None)
records = []
handle = open(fileName, 'rb')
csvHandle = csv.reader(handle)
header = csvHandle.next()
for row in csvHandle:
records.append(row)
handle.close()
return (header, records)
def main(self):
"""
The main function that runs everything
"""
log.info('sysargv : %s' % sys.argv)
parser = OptionParser()
parser.add_option('--mex_url', dest="mexURL")
parser.add_option('--module_dir', dest="modulePath")
parser.add_option('--staging_path', dest="stagingPath")
parser.add_option('--bisque_token', dest="token")
parser.add_option('--user', dest="user")
parser.add_option('--pwd', dest="pwd")
parser.add_option('--root', dest="root")
parser.add_option('--entrypoint', dest="entrypoint")
(options, args) = parser.parse_args()
fh = logging.FileHandler('phase_%s.log' %
(options.entrypoint or 'main'),
mode='a')
fh.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'[%(asctime)s] %(levelname)8s --- %(message)s ' +
'(%(filename)s:%(lineno)s)',
datefmt='%Y-%m-%d %H:%M:%S')
fh.setFormatter(formatter)
log.addHandler(fh)
try: #pull out the mex
if not options.mexURL:
options.mexURL = sys.argv[1]
if not options.token:
options.token = sys.argv[2]
except IndexError: #no argv were set
pass
if not options.stagingPath:
options.stagingPath = '/module/workdir'
log.info('\n\nPARAMS : %s \n\n Options: %s' % (args, options))
self.options = options
if self.validate_input():
#initalizes if user and password are provided
if (self.options.user and self.options.pwd and self.options.root):
self.bqSession = BQSession().init_local(
self.options.user,
self.options.pwd,
bisque_root=self.options.root)
self.options.mexURL = self.bqSession.mex.uri
#initalizes if mex and mex token is provided
elif (self.options.mexURL and self.options.token):
self.bqSession = BQSession().init_mex(self.options.mexURL,
self.options.token)
else:
raise IJError(
'Insufficient options or arguments to start this module')
if not self.options.entrypoint:
# NOT a special phase => perform regular run processing
try:
self.setup()
except Exception as e:
log.exception("Exception during setup")
self.bqSession.fail_mex(msg="Exception during setup: %s" %
str(e))
return
try:
self.run()
except (Exception, IJError) as e:
log.exception("Exception during run")
self.bqSession.fail_mex(msg="Exception during run: %s" %
str(e))
return
try:
self.teardown()
except (Exception, IJError) as e:
log.exception("Exception during teardown")
self.bqSession.fail_mex(
msg="Exception during teardown: %s" % str(e))
return
else:
# in a special phase => run special code
self.bqSession.fail_mex(msg="Unknown ImageJ entrypoint: %s" %
self.options.entrypoint)
return
self.bqSession.close()
class PlanteomeDeepSegment(object):
# +
# __init__()
# -
def __init__(self):
# entry message
log.debug('{}.__init__()> message on entry'.format(MODULE_NAME))
# declare some variables and initialize them
self.options = None
self.bqSession = None
self.rois = None
self.message = None
# get full path(s) to file(s)
self.contours_file = os.path.abspath(
os.path.expanduser(PICKLE_CONTOURS_FILE))
self.data_file = os.path.abspath(os.path.expanduser(PICKLE_DATA_FILE))
self.results_file = os.path.abspath(
os.path.expanduser(TEXT_RESULTS_FILE))
self.tiff_file = os.path.abspath(os.path.expanduser(TIFF_IMAGE_FILE))
self.csv_leaf_file = os.path.abspath(os.path.expanduser(CSV_LEAF_FILE))
log.debug('{}.__init()> self.contours_file={}'.format(
MODULE_NAME, self.contours_file))
log.debug('{}.__init()> self.data_file={}'.format(
MODULE_NAME, self.data_file))
log.debug('{}.__init()> self.results_file={}'.format(
MODULE_NAME, self.results_file))
log.debug('{}.__init()> self.tiff_file={}'.format(
MODULE_NAME, self.tiff_file))
log.debug('{}.__init()> self.csv_leaf_file={}'.format(
MODULE_NAME, self.csv_leaf_file))
# exit message
log.debug('{}.__init__()> message on exit'.format(MODULE_NAME))
# +
# hidden method: _mex_parameter_parser()
# -
def _mex_parameter_parser(self, mex_xml=None):
# entry message
log.debug(
'{}._mex_parameter_parser()> message on entry, mex_xml={}'.format(
MODULE_NAME, str(mex_xml)))
if mex_xml is not None:
mex_inputs = mex_xml.xpath('tag[@name="inputs"]')
if mex_inputs:
for tag in mex_inputs[0]:
if tag.tag == 'tag' and tag.attrib[
'type'] != 'system-input':
_name = tag.attrib['name'].strip()
_value = tag.attrib['value'].strip()
log.debug(
'{}._mex_parameter_parser()> setting self.options.{}={}'
.format(MODULE_NAME, _name, _value))
setattr(self.options, _name, _value)
log.debug(
"{}._mex_parameter_parser()> set self.options.{}={}"
.format(MODULE_NAME, _name,
getattr(self.options, _name)))
else:
log.error('{}.mex_parameter_parser()> no inputs found on mex!'.
format(MODULE_NAME))
else:
self.message = '{}.mex_parameter_parser()> mex_xml is None'.format(
MODULE_NAME)
log.error(self.message)
# exit message
log.debug('{}.main()> message on exit, options={}'.format(
MODULE_NAME, self.options))
# +
# hidden method: _validate_input()
# -
def _validate_input(self):
# entry message
retval = False
log.debug('{}._validate_input()> message on entry, retval={}'.format(
MODULE_NAME, retval))
# run module through engine_service (default)
if self.options.mexURL and self.options.token:
retval = True
# run module locally
elif self.options.user and self.options.pwd and self.options.root:
retval = True
else:
retval = False
log.error(
'{}.validate_input()> insufficient options or arguments to start this module'
.format(MODULE_NAME))
# exit message
log.debug('{}._validate_input()> message on exit, retval={}'.format(
MODULE_NAME, retval))
return retval
# +
# hidden method: _construct_vertices()
# -
def _construct_vertices(self, child=None):
# entry message
vertices = None
roi = []
log.debug(
'{}._construct_vertices()> message on entry, child={}'.format(
MODULE_NAME, str(child)))
# get annotation type
if child is not None:
annotation_type = 'fg' if 'foreground' in child.values() else 'bg'
# get vertices
vertices = child.getchildren()[0].getchildren()
for _vertex in vertices:
_values = _vertex.values()
roi.append({
'x': int(float(_values[2])),
'y': int(float(_values[3]))
})
log.debug(
'{}._construct_vertices()> ROI: appending {} value with {}'.
format(MODULE_NAME, annotation_type, str(roi)))
self.rois[annotation_type].append(roi)
# exit message
log.debug(
'{}._construct_vertices()> message on exit, vertices={}, length={}, rois={}'
.format(MODULE_NAME, str(vertices), len(vertices), str(self.rois)))
# +
# hidden method: _show_structure()
# -
def _show_structure(self, r_xml=None):
# entry message
log.debug('{}._show_structure()> message on entry, r_xml={}'.format(
MODULE_NAME, str(r_xml)))
if r_xml is not None:
for _i, _child in enumerate(r_xml.getchildren()):
log.debug('{}._show_structure()> index={}, child={}'.format(
MODULE_NAME, _i, str(_child)))
log.debug('{}._show_structure()> index={}, values={}'.format(
MODULE_NAME, _i, str(_child.values())))
if 'background' in _child.values(
) or 'foreground' in _child.values():
self._construct_vertices(_child)
else:
self._show_structure(_child)
# exit message
log.debug('{}._show_structure()> message on exit'.format(MODULE_NAME))
# +
# method: setup()
# -
def setup(self):
# entry message
log.debug('{}.setup()> message on entry, options={}'.format(
MODULE_NAME, self.options))
# run locally
if self.options.user and self.options.pwd and self.options.root:
log.debug(
'{}.setup()> running locally with user={}, pwd={}, root={}'.
format(MODULE_NAME, self.options.user, self.options.pwd,
self.options.root))
self.bqSession = BQSession().init_local(
self.options.user,
self.options.pwd,
bisque_root=self.options.root)
self.options.mexURL = self.bqSession.mex.uri
# run on the server with a mexURL and an access token
elif self.options.mexURL and self.options.token:
log.debug('{}.setup()> running on server with mexURL={}, token={}'.
format(MODULE_NAME, self.options.mexURL,
self.options.token))
self.bqSession = BQSession().init_mex(self.options.mexURL,
self.options.token)
# failed to connect to bisque
else:
self.bqSession = None
self.message(
'{}.setup()> failed to connect to bisque'.format(MODULE_NAME))
log.error(self.message)
raise PlanteomeDeepSegmentError(self.message)
# parse the xml and construct the tree, also set options to proper values after parsing it
if self.bqSession is not None:
self._mex_parameter_parser(self.bqSession.mex.xmltree)
log.debug(
'{}.setup()> image URL={}, mexURL={}, stagingPath={}, token={}'
.format(MODULE_NAME, self.options.image_url,
self.options.mexURL, self.options.stagingPath,
self.options.token))
# exit message
log.debug('{}.setup()> message on exit, options={}'.format(
MODULE_NAME, self.options))
# +
# method: run()
# The core of the PlanteomeDeepSegment module. It requests features on the provided image, classifies each tile
# and selects a majority amongst the tiles.
# -
def run(self):
# entry message
log.debug('{}.run()> message on entry, options={}'.format(
MODULE_NAME, self.options))
self.rois = {'fg': [], 'bg': []}
r_xml = self.bqSession.fetchxml(self.options.mexURL, view='deep')
log.debug('{}.run()> Shols structura'.format(MODULE_NAME))
self._show_structure(r_xml)
log.debug(self.rois)
# dump image as .tiff
image = self.bqSession.load(self.options.image_url)
ip = image.pixels().format('tiff')
with open(self.tiff_file, 'wb') as f:
f.write(ip.fetch())
# pickle the data
try:
if self.rois and getattr(self.options, 'segmentImage') != '' and \
getattr(self.options, 'deepNetworkChoice') != '' and getattr(self.options, 'qualitySeg') != '' and \
getattr(self.options, 'deepSeg') != '' and getattr(self.options, 'mexURL') != '' and \
getattr(self.options, 'token') != '':
log.debug('{}.run()> pickling data to {}'.format(
MODULE_NAME, self.data_file))
pickle.dump([
self.rois, self.options.segmentImage,
self.options.deepNetworkChoice, self.options.qualitySeg,
self.options.deepSeg, self.options.mexURL,
self.options.token
], open(self.data_file, 'wb'))
except AttributeError as e:
self.message('{}.run()> failed to pickle data, e={}'.format(
MODULE_NAME, str(e)))
log.error(self.message)
# do something
x = PlanteomeDeepSegmentLearning(self.contours_file, self.data_file,
self.tiff_file, self.results_file)
x.main()
# exit message
log.debug('{}.run()> message on exit, options={}'.format(
MODULE_NAME, self.options))
# +
# method: teardown()
# -
def teardown(self):
# entry message
log.debug('{}.teardown()> message on entry, options={}'.format(
MODULE_NAME, self.options))
# set up tag(s)
self.bqSession.update_mex('Returning results...')
output_tag = eTree.Element('tag', name='outputs')
output_sub_tag_image = eTree.SubElement(output_tag,
'tag',
name='Final Image',
value=self.options.image_url)
output_sub_tag_summary = eTree.SubElement(output_tag,
'tag',
name='summary')
log.info('Module will output image {}'.format(self.options.image_url))
# segment the image (if required)
if getattr(self.options, 'segmentImage', '') != '' and self.options.segmentImage.lower() == 'true' and \
os.path.isfile(self.contours_file):
log.debug(
'{}.teardown()> module will segment image from file {}'.format(
MODULE_NAME, self.contours_file))
eTree.SubElement(output_sub_tag_summary,
'tag',
name='Segment Image',
value=self.options.segmentImage)
[_contours, _t_scale] = pickle.load(open(self.contours_file, 'rb'))
_gob = eTree.SubElement(output_sub_tag_image,
'gobject',
name='Annotations',
type='Annotations')
_polyseg = eTree.SubElement(_gob, 'polygon', name='SEG')
eTree.SubElement(_polyseg, 'tag', name='color', value="#0000FF")
_opd = 0
_output_sampling = 1 + int(len(_contours) / 100)
for _j in range(len(_contours)):
if _j % _output_sampling == 0:
_opd += 1
_x = str(1 + int(_t_scale[1] * _contours[_j][1]))
_y = str(1 + int(_t_scale[0] * _contours[_j][0]))
eTree.SubElement(_polyseg, 'vertex', x=_x, y=_y)
log.debug('{}.teardown()> _opd={}'.format(MODULE_NAME, _opd))
else:
log.info(
'Module will not segment image, (were foreground and background polyline annotations provided?)'
)
# select deepNetworkChoice
opts = getattr(self.options, 'deepNetworkChoice', '')
eTree.SubElement(output_sub_tag_summary,
'tag',
name='Model File',
value=opts)
if opts == '':
log.error('{}.teardown()> deepNetworkChoice={}'.format(
MODULE_NAME, self.options.deepNetworkChoice))
else:
# simple classification
if opts.split()[0].lower() == 'simple':
# get prediction
prediction_c = -1
confidence_c = 0.0
prediction_t = -1
confidence_t = 0.0
try:
with open(self.results_file, 'r') as f:
for _line in f:
if _line.strip() != '':
log.debug('{}.teardown()> _line={}'.format(
MODULE_NAME, _line))
if 'PREDICTION_C:' in _line:
prediction_c = int(
_line.split(':')[1].strip())
if 'CONFIDENCE_C:' in _line:
confidence_c = float(
_line.split(':')[1].strip())
if 'PREDICTION_T:' in _line:
prediction_t = int(
_line.split(':')[1].strip())
if 'CONFIDENCE_T:' in _line:
confidence_t = float(
_line.split(':')[1].strip())
except IOError as e:
self.message = '{}.teardown()> io error reading results, e={}'.format(
MODULE_NAME, str(e))
log.error(self.message)
finally:
log.debug('{}.teardown()> prediction_c={}'.format(
MODULE_NAME, prediction_c))
log.debug('{}.teardown()> confidence_c={}'.format(
MODULE_NAME, confidence_c))
log.debug('{}.teardown()> prediction_t={}'.format(
MODULE_NAME, prediction_t))
log.debug('{}.teardown()> confidence_t={}'.format(
MODULE_NAME, confidence_t))
# annotate with prediction
classes = [
'Leaf (PO:0025034): http://browser.planteome.org/amigo/term/PO:0025034',
'Fruit (PO:0009001): http://browser.planteome.org/amigo/term/PO:0009001',
'Flower (PO:0009046): http://browser.planteome.org/amigo/term/PO:0009046',
'Stem (PO:0009047): http://browser.planteome.org/amigo/term/PO:0009047',
'Whole plant (PO:0000003): http://browser.planteome.org/amigo/term/PO:0000003 '
]
prediction_c = classes[prediction_c] if (
0 <= prediction_c <= len(classes)) else 'unknown'
eTree.SubElement(output_sub_tag_summary,
'tag',
name='Class',
value=prediction_c)
if prediction_c.lower() != 'unknown':
eTree.SubElement(
output_sub_tag_summary,
'tag',
type='link',
name='Class link',
value=prediction_c.split('):')[-1].strip())
eTree.SubElement(output_sub_tag_summary,
'tag',
name='Class Confidence',
value=str(confidence_c))
# leaf classification
elif opts.split()[0].lower() == 'leaf':
log.debug('{}.teardown()> leaf_targets{}'.format(
MODULE_NAME, leaf_targets))
log.debug('{}.teardown()> leaf_targets_links{}'.format(
MODULE_NAME, leaf_targets_links))
# map each leaf target to the corresponding PO term
with open(self.csv_leaf_file) as cf:
reader = csv.reader(cf, delimiter=',', quotechar='|')
_cn = ''
for _row in reader:
_n = _row[0] if _row[0] != '' else 'undefined'
_m = _row[1] if _row[1] != '' else 'undefined'
# _c = _row[2] if _row[2] != '' else 'undefined'
_t = _n.replace(' ', '').lower()
# get the current name
for _lc in leaf_keys_nospaces:
if _t == _lc:
_cn = leaf_keys_spaces[
leaf_keys_nospaces.index(_lc)]
break
# replace dictionary entry is mapping exists
if _cn in leaf_targets:
for _l in leaf_targets[_cn]:
if _n == _l.replace(' ', ''):
_i = leaf_targets[_cn].index(_l)
leaf_targets_links[_cn][_i] = _m
break
# read result(s)
with open(self.results_file, 'r') as f:
class_list = []
for _i, _line in enumerate(f):
# remove after introduction of the leaf classifier (below start with appends)
if int(_line) == len(leaf_targets[leaf_keys[_i]]) - 1:
_line = '0'
class_list.append(_line)
eTree.SubElement(output_sub_tag_summary,
'tag',
name='{}-Name'.format(
leaf_keys_proper_names[_i]),
value=leaf_targets[leaf_keys[_i]][int(
class_list[_i])])
if leaf_targets_links[leaf_keys[_i]][int(
class_list[_i])] != 'undefined':
eTree.SubElement(
output_sub_tag_summary,
'tag',
type='link',
name='{}-Accession'.format(
leaf_keys_proper_names[_i]),
value=
'http://browser.planteome.org/amigo/term/{}'.
format(leaf_targets_links[leaf_keys[_i]][int(
class_list[_i])]))
else:
eTree.SubElement(
output_sub_tag_summary,
'tag',
name='{}-Accession'.format(
leaf_keys_proper_names[_i]),
value=leaf_targets_links[leaf_keys[_i]][int(
class_list[_i])])
# update mex
self.bqSession.finish_mex(tags=[output_tag])
self.bqSession.close()
# exit message
log.debug('{}.teardown()> message on exit, options={}'.format(
MODULE_NAME, self.options))
# +
# method: main()
# -
def main(self):
# entry message
log.debug('{}.main()> message on entry, args={}'.format(
MODULE_NAME, sys.argv))
parser = OptionParser()
parser.add_option('--image_url', dest="image_url")
parser.add_option('--mex_url', dest="mexURL")
parser.add_option('--module_dir', dest="modulePath")
parser.add_option('--staging_path', dest="stagingPath")
parser.add_option('--bisque_token', dest="token")
parser.add_option('--user', dest="user")
parser.add_option('--pwd', dest="pwd")
parser.add_option('--root', dest="root")
(options, args) = parser.parse_args()
log.debug('{}.main()> options={}'.format(MODULE_NAME, options))
log.debug('{}.main()> args={}'.format(MODULE_NAME, args))
# set up the mexURL and token based on the arguments passed to the script
try:
if not options.mexURL:
options.mexURL = sys.argv[1]
if not options.token:
options.token = sys.argv[2]
if not options.stagingPath:
options.stagingPath = ''
except IndexError:
pass
finally:
self.options = options
log.debug('{}.main()> self.options={}'.format(
MODULE_NAME, self.options))
# check input(s)
if self._validate_input():
# noinspection PyBroadException
try:
# set up the module
self.setup()
except PlanteomeDeepSegmentError as e:
self.message = '{}.main()> specific exception after setup(), e={}'.format(
MODULE_NAME, str(e.errstr))
log.exception(self.message)
self.bqSession.fail_mex(msg=self.message)
raise PlanteomeDeepSegmentError(self.message)
except Exception as e:
self.message = '{}.main()> exception after setup(), e={}'.format(
MODULE_NAME, str(e))
log.exception(self.message)
self.bqSession.fail_mex(msg=self.message)
raise PlanteomeDeepSegmentError(self.message)
except:
self.message = '{}.main()> error after setup()'.format(
MODULE_NAME)
log.exception(self.message)
self.bqSession.fail_mex(msg=self.message)
raise PlanteomeDeepSegmentError(self.message)
# noinspection PyBroadException
try:
# run the module
self.run()
except PlanteomeDeepSegmentError as e:
self.message = '{}.main()> specific exception after run(), e={}'.format(
MODULE_NAME, str(e.errstr))
log.exception(self.message)
self.bqSession.fail_mex(msg=self.message)
raise PlanteomeDeepSegmentError(self.message)
except Exception as e:
self.message = '{}.main()> exception after run(), e={}'.format(
MODULE_NAME, str(e))
log.exception(self.message)
self.bqSession.fail_mex(msg=self.message)
raise PlanteomeDeepSegmentError(self.message)
except:
self.message = '{}.main()> error after run()'.format(
MODULE_NAME)
log.exception(self.message)
self.bqSession.fail_mex(msg=self.message)
raise PlanteomeDeepSegmentError(self.message)
# noinspection PyBroadException
try:
# tear down the module
self.teardown()
except PlanteomeDeepSegmentError as e:
self.message = '{}.main()> specific exception after teardown(), e={}'.format(
MODULE_NAME, str(e.errstr))
log.exception(self.message)
self.bqSession.fail_mex(msg=self.message)
raise PlanteomeDeepSegmentError(self.message)
except Exception as e:
self.message = '{}.main()> exception after teardown(), e={}'.format(
MODULE_NAME, str(e))
log.exception(self.message)
self.bqSession.fail_mex(msg=self.message)
raise PlanteomeDeepSegmentError(self.message)
except:
self.message = '{}.main()> error after teardown()'.format(
MODULE_NAME)
log.exception(self.message)
self.bqSession.fail_mex(msg=self.message)
raise PlanteomeDeepSegmentError(self.message)
else:
self.message = '{}.main()> failed to validate instance'.format(
MODULE_NAME)
log.error(self.message)
self.bqSession.fail_mex(msg=self.message)
raise PlanteomeDeepSegmentError(self.message)
# exit message
log.debug('{}.main()> message on exit, args={}'.format(
MODULE_NAME, sys.argv))