class BotanicamTrainer(object):
"""
Botanicam Trainer Module
"""
def mex_parameter_parser(self, mex_xml):
"""
Parses input of the xml and add it to BotanicamTrainer's options attribute
@param: mex_xml
"""
mex_inputs = mex_xml.xpath('tag[@name="inputs"]')
if mex_inputs:
for tag in mex_inputs[0]:
if tag.tag == 'tag' and tag.attrib[
'type'] != 'system-input': #skip system input values
log.debug('Set options with %s as %s' %
(tag.attrib['name'], tag.attrib['value']))
setattr(self.options, tag.attrib['name'],
tag.attrib['value'])
else:
log.debug('BotanicamFS: No Inputs Found on MEX!')
def validate_input(self):
"""
Check to see if a mex with token or user with password was provided.
@return True is returned if validation credention was provided else
False is returned
"""
if (self.options.mexURL
and self.options.token): #run module through engine service
return True
if (self.options.user and self.options.pwd and
self.options.root): #run module locally (note: to test module)
return True
log.debug(
'BotanicamTrainer: Insufficient options or arguments to start this module'
)
return False
def setup(self):
"""
Fetches the mex and appends input_configurations to the option
attribute of BotanicamTrainer
"""
#initalizes if user and password are provided
if (self.options.user and self.options.pwd and self.options.root):
self.bqSession = BQSession().init_local(
self.options.user,
self.options.pwd,
bisque_root=self.options.root)
self.options.mexURL = self.bqSession.mex.uri
#initalizes if mex and mex token is provided
elif (self.options.mexURL and self.options.token):
self.bqSession = BQSession().init_mex(self.options.mexURL,
self.options.token)
else:
raise BotanicamTrainerError(
'BotanicamTrainer: Insufficient options or arguments to start this module'
)
self.bqSession.update_mex('Initializing...')
self.mex_parameter_parser(self.bqSession.mex.xmltree)
return
def run(self):
"""
The core of the Botanicam Trainer
Parses the tags and find all the corresponding values to from classes.
Classes that have no images in them are removed. Features are requested
on all the images and then trained using the tag value classes. The
resulting model file is stored on bisque as a zip file.
"""
#retieve tags
self.bqSession.update_mex('Parse Tags...')
if not self.options.tag_names:
raise BotanicamTrainerError('Tags are a required input!')
self.tag_names = self.options.tag_names.split(',')
#type check
resource_short = self.bqSession.fetchxml(self.options.resource_url,
view='short')
if resource_short.tag == 'dataset':
resource_url_values = '%s/value' % self.options.resource_url
else:
resource_url_values = self.options.resource_url
all_images = self.bqSession.fetchxml(resource_url_values,
view='full,clean')
tag_query_list = []
for name in self.tag_names:
name_list = []
for u in np.unique(
all_images.xpath('image/tag[@name="%s"]/@value' % name)):
name_list.append('"%s":"%s"' % (name, u))
tag_query_list.append(name_list)
#need to find the unique values to create lists of images
#hopefully the tag names and lists are not too complicated
tag_query_list = [
list(element) for element in itertools.product(*tag_query_list)
] #cartesian product
self.complete_tag_list = []
tag_query_url_list = []
#search for classes with images
#query all the values to see if images return
#removing query_tag from the resource_url and adding it back in later
resource_url_wo_query = resource_url_values
resource_query = None
from urlparse import urlsplit, urlunsplit, parse_qs
from urllib import urlencode
o = urlsplit(resource_url_values)
q = parse_qs(o.query)
if q.get('tag_query'):
resource_query = q['tag_query']
del q['tag_query']
query = urlencode(q)
resource_url_wo_query = urlunsplit(
(o.scheme, o.netloc, o.path, query, o.fragment))
log.debug(tag_query_list)
for tag_query in tag_query_list:
encoded_tag_query = tag_query
if resource_query:
encoded_tag_query += resource_query #adding back in the query_tag from the resource_url
encoded_tag_query = map(urllib.quote, tag_query)
encoded_tag_query = '%s' % ' AND '.join(encoded_tag_query)
query_xml = self.bqSession.fetchxml(resource_url_wo_query,
tag_query=encoded_tag_query,
view='full,clean')
if len(query_xml.xpath('image')) > 0:
name_value_pairs = {}
for t in tag_query: #create dictionary of clases with list of name value pairs
m = re.match('"(?P<name>[^"]+)":"(?P<value>[^"]+)"', t)
name_value_pairs[m.group('name')] = m.group('value')
self.complete_tag_list.append(name_value_pairs)
tag_query_url_list.append(query_xml.attrib['uri'])
feature_length = Feature().length(self.bqSession, FEATURE_NAME)
if len(tag_query_url_list) < 2:
raise BotanicamTrainerError(
'Requires atleast 2 classes to train found %s' %
len(tag_query_url_list))
#extracts all the features and then appends it to a larger table
_mkdir(os.path.join(self.options.stagingPath, FEATURE_TABLE_DIR))
main_table = os.path.join(self.options.stagingPath, FEATURE_TABLE_DIR,
'feature_table.h5')
with tables.open_file(main_table, 'w') as h5file:
columns = {
'label': tables.Int64Col(),
'feature': tables.Float32Col(shape=(feature_length))
}
table = h5file.create_table('/', 'values', columns)
table.flush()
self.bqSession.update_mex('Calculated features on (0/%s) spieces...' %
(len(tag_query_url_list)))
for i, tag_url_query in enumerate(tag_query_url_list):
try:
vectors = extract_bush_feature(
self.bqSession,
tag_url_query) #may need to be moved into local temp
feature_table = vectors.root.values
with tables.open_file(main_table, 'a') as h5file:
table = h5file.root.values
r = table.row
for fr in feature_table:
r['feature'] = fr['feature']
r['label'] = i
r.append()
table.flush()
vectors.close()
os.remove(vectors.filename)
except FeatureError as e:
raise BotanicamTrainerError(str(e))
self.bqSession.update_mex(
'Calculated features on (%s/%s) spieces...' %
(i + 1, len(tag_query_url_list)))
self.bqSession.update_mex('Classifying')
log.debug('Training model')
#classify the features
pca = RandomizedPCA(whiten=True)
clf = svm.SVC()
with tables.open_file(main_table, 'r') as h5file:
table = h5file.root.values
pca.fit(table[:]['feature'])
clf.fit(pca.transform(table[:]['feature']), table[:]['label'])
self.bqSession.update_mex('Posting model to bisque...')
log.debug('Storing and Zipping model')
self.model_dir = os.path.join(self.options.stagingPath, MODEL_DIR)
self.svm_model_file = os.path.join(self.options.stagingPath, MODEL_DIR,
'svm_model')
self.pca_model_file = os.path.join(self.options.stagingPath, MODEL_DIR,
'pca_model')
_mkdir(self.model_dir)
svm_files = joblib.dump(clf, self.svm_model_file)
pca_files = joblib.dump(pca, self.pca_model_file)
#zip file
import zipfile
with zipfile.ZipFile('%s.zip' % self.model_dir, 'w') as fzip:
for f in svm_files:
fzip.write(f, os.path.basename(f))
for f in pca_files:
fzip.write(f, os.path.basename(f))
def teardown(self):
"""
Post the results to the mex xml.
"""
#save the model and upload it to the data service with all the meta data
self.bqSession.update_mex('Returning results...')
#self.bqSession.update_mex('Returning home after a long day...')
#constructing and storing model file
#does not accept no name on the resource
cl_model = etree.Element('resource',
name=os.path.basename('%s.zip' % MODEL_DIR))
#classes
tag_classes = etree.SubElement(cl_model, 'tag', name='Classes')
for i, name_value_pairs in enumerate(self.complete_tag_list):
tag_labels = etree.SubElement(tag_classes,
'tag',
name='Class_%s' % str(i),
value=str(i))
for n in name_value_pairs.keys():
etree.SubElement(tag_labels,
'tag',
name=n,
value=name_value_pairs[n])
etree.SubElement(cl_model,
'tag',
name='Number of Classes',
value=str(len(self.complete_tag_list)))
#module identifier (a descriptor to be found by the botanicam model)
etree.SubElement(cl_model,
'tag',
name='module_identifier',
value='Botanicam')
#model filename
etree.SubElement(cl_model,
'tag',
name='filename',
value=os.path.basename('%s.zip' % MODEL_DIR))
#classification method
etree.SubElement(cl_model,
'tag',
name='Classification Method',
value='Bush Descriptor')
#Resource url
etree.SubElement(cl_model,
'tag',
name='Resource',
value=self.options.resource_url)
#description
etree.SubElement(cl_model,
'tag',
name='description',
value='Model File for the Botanicam Module')
#storing the model file in blobservice
r = self.bqSession.postblob('%s.zip' % self.model_dir, xml=cl_model)
r_xml = etree.fromstring(r)
outputTag = etree.Element('tag', name='outputs')
etree.SubElement(outputTag,
'tag',
name='classification_file',
value=r_xml[0].attrib['uri'],
type='model_viewer')
self.bqSession.finish_mex(tags=[outputTag])
self.bqSession.close()
def main(self):
"""
The main function that runs everything
"""
log.debug('sysargv : %s' % sys.argv)
parser = OptionParser()
parser.add_option('--resource_url',
dest="resource_url") #for now only datasets
parser.add_option('--Tags', dest="tag_names") #labels on the images
parser.add_option('--ClassifierMethod', dest="classifier_methods")
parser.add_option('--FeatureName', dest="feature_name")
parser.add_option('--mex_url', dest="mexURL")
parser.add_option('--module_dir', dest="modulePath")
parser.add_option('--staging_path', dest="stagingPath")
parser.add_option('--bisque_token', dest="token")
parser.add_option('--user', dest="user")
parser.add_option('--pwd', dest="pwd")
parser.add_option('--root', dest="root")
(options, args) = parser.parse_args()
try: #pull out the mex
if not options.mexURL:
options.mexURL = sys.argv[1]
if not options.token:
options.token = sys.argv[2]
except IndexError: #no argv were set
pass
if not options.stagingPath:
options.stagingPath = ''
log.debug('\n\nPARAMS : %s \n\n Options: %s' % (args, options))
self.options = options
if self.validate_input():
try:
self.setup()
except Exception as e:
log.exception("Exception during setup")
self.bqSession.fail_mex(msg="Exception during setup: %s" %
str(e))
return
try:
self.run()
except (Exception, BotanicamTrainerError) as e:
log.exception("Exception during run")
self.bqSession.fail_mex(msg="Exception during run: %s" %
str(e))
return
try:
self.teardown()
except (Exception, BotanicamTrainerError) as e:
log.exception("Exception during teardown %s")
self.bqSession.fail_mex(msg="Exception during teardown: %s" %
str(e))
return
class NuclearDetector3DTests(ImageServiceTestBase):
# setups
@classmethod
def setUpClass(self):
config = ConfigParser.ConfigParser()
config.read('config.cfg')
self.root = config.get('Host', 'root') or 'localhost:8080'
self.user = config.get('Host', 'user') or 'test'
self.pswd = config.get('Host', 'password') or 'test'
self.session = BQSession().init_local(self.user,
self.pswd,
bisque_root=self.root,
create_mex=False)
# download and upload test images ang get their IDs
self.resource_2k = self.ensure_bisque_file(image_2k)
@classmethod
def tearDownClass(self):
self.delete_resource(self.resource_2k)
self.cleanup_tests_dir()
pass
# tests
def test_nd3d_run(self):
resource = self.resource_2k
self.assertIsNotNone(resource, 'Resource was not uploaded')
url = urlparse.urljoin(self.root,
posixpath.join(url_module, 'execute'))
request = request_xml.replace('{IMAGE_URL}', resource.get('uri', ''))
request = request.replace('{NUCLEAR_CHANNEL}', '1')
request = request.replace('{NUCLEAR_SIZE}', '5')
r = self.session.postxml(url, etree.fromstring(request), method='POST')
self.assertIsNotNone(r, 'Module execution failed')
url = r.get('uri', None)
status = r.get('value', None)
self.assertIsNotNone(url, 'Module MEX is incorrect')
self.assertIsNotNone(status, 'Module MEX is incorrect')
self.assertTrue(status != 'FAILED',
msg='Module failed to run correctly')
while status != 'FINISHED':
time.sleep(2)
mex = self.session.fetchxml(url)
self.assertIsNotNone(mex, 'Fetching MEX update failed')
status = mex.get('value', None)
self.assertTrue(status != 'FAILED',
msg='Module failed to run correctly')
# module finished, compare results
mex = self.session.fetchxml(url, view='deep')
l = mex.xpath('//gobject[@type="nucleus"]/point')
self.assertTrue(
len(l) == 444,
msg=
'Module returned incorrect number of points, expected 444 got %s' %
len(l))
class SkeletonPython(object):
"""
SkeletonPython Model
"""
def mex_parameter_parser(self, mex_xml):
"""
Parses input of the xml and add it to SkeletonPython Trainer's options attribute
@param: mex_xml
"""
mex_inputs = mex_xml.xpath('tag[@name="inputs"]')
if mex_inputs:
for tag in mex_inputs[0]:
if tag.tag == 'tag' and tag.attrib['type'] != 'system-input':
log.debug('Set options with %s as %s'%(tag.attrib['name'],tag.attrib['value']))
setattr(self.options,tag.attrib['name'],tag.attrib['value'])
else:
log.debug('SkeletonPythonFS: No Inputs Found on MEX!')
def validateInput(self):
"""
Parses input of the xml and add it to SkeletonPython's options attribute
@param: mex_xml
"""
if (self.options.mexURL and self.options.token): #run module through engine service
return True
if (self.options.user and self.options.pwd and self.options.root): #run module locally (note: to test module)
return True
log.debug('SkeletonPython: Insufficient options or arguments to start this module')
return False
def setup(self):
"""
Fetches the mex, appends input_configurations to the option
attribute of SkeletonPython and looks up the model on bisque to
classify the provided resource.
"""
if (self.options.user and self.options.pwd and self.options.root):
self.bqSession = BQSession().init_local( self.options.user, self.options.pwd, bisque_root=self.options.root)
self.options.mexURL = self.bqSession.mex.uri
# This is when the module actually runs on the server with a mexURL and an access token
elif (self.options.mexURL and self.options.token):
self.bqSession = BQSession().init_mex(self.options.mexURL, self.options.token)
else:
return
# Parse the xml and construct the tree, also set options to proper values after parsing it (like image url)
self.mex_parameter_parser(self.bqSession.mex.xmltree)
log.debug('SkeletonPython: image URL: %s, mexURL: %s, stagingPath: %s, token: %s' % (self.options.image_url, self.options.mexURL, self.options.stagingPath, self.options.token))
def construct_vertices(self, child):
annotation_type = 'bg'
if 'foreground' in child.values():
annotation_type = 'fg'
roi = []
log.debug("This is the child")
vertices = child.getchildren()[0].getchildren()
for vertex in vertices:
values = vertex.values()
roi.append({'x':int(float(values[2])), 'y':int(float(values[3]))})
self.rois[annotation_type].append(roi)
log.debug(vertices)
log.debug(len(vertices))
def show_structure(self, r_xml):
for i, child in enumerate(r_xml.getchildren()):
if "background" in child.values() or 'foreground' in child.values():
log.debug('Background/Foreground annotation')
self.construct_vertices(child)
else:
self.show_structure(child)
def run(self):
"""
The core of the SkeletonPython Module
Requests features on the image provided. Classifies each tile
and picks a majority among the tiles.
"""
self.rois = {'fg':[],'bg':[]}
r_xml = self.bqSession.fetchxml(self.options.mexURL, view='deep')
log.debug("Shols structura")
self.show_structure(r_xml)
log.debug(self.rois)
image = self.bqSession.load(self.options.image_url)
ip = image.pixels().format('tiff')
pixels = ip.fetch()
f = open('./temp.tif','wb')
f.write(pixels)
f.close()
pickle.dump([self.rois,self.options.segmentImage,self.options.deepNetworkChoice,self.options.qualitySeg,self.options.deepSeg,self.options.mexURL,self.options.token], open('./data.p','wb'))
pathToScript = './DeepTools/deep_script.sh'
call([pathToScript])
def teardown(self):
"""
Posting results to the mex
"""
self.bqSession.update_mex('Returning results...')
log.debug('Returning results...')
prediction = "None-Module Failure"
with open("./results.txt","r") as f:
for line in f:
if "PREDICTION_C:" in line:
prediction_c = line
if "CONFIDENCE_C:" in line:
confidence_c = line[14:-1]
classes = ["leaf","fruit","flower","stem","entire"]
if self.options.deepNetworkChoice != 'None':
for i,class_tag in enumerate(classes):
prediction_c = prediction_c.replace(str(i),class_tag)
outputTag = etree.Element('tag', name='outputs')
outputSubTagImage = etree.SubElement(outputTag, 'tag', name='Final Image', value=self.options.image_url)
print "Module will output image, {}".format(self.options.image_url)
if not os.path.isfile("./contours.pkl"):
print "Module will not segment image, (were foreground and background polyline annotations provided?)"
if self.options.segmentImage != "False" and os.path.isfile("./contours.pkl"):
[contours, t_scale] = pickle.load(open("./contours.pkl","rb"))
gob = etree.SubElement (outputSubTagImage, 'gobject', name='Annotations', type='Annotations')
polyseg = etree.SubElement(gob, 'polygon', name='SEG')
etree.SubElement( polyseg, 'tag', name='color', value="#0000FF")
opd = 0
output_sampling = 1+int(len(contours)/100)
for j in range(len(contours)):
if j % (output_sampling) == 0:
opd += 1
etree.SubElement (polyseg, 'vertex', x=str(1+int(t_scale[1]*contours[j][1])), y=str(1+int(t_scale[0]*contours[j][0])))
log.debug(opd)
if self.options.deepNetworkChoice != 'None':
outputSubTagSummary = etree.SubElement(outputTag, 'tag', name='summary')
etree.SubElement(outputSubTagSummary, 'tag',name='Model File', value=self.options.deepNetworkChoice)
etree.SubElement(outputSubTagSummary, 'tag',name='Segment Image', value=self.options.segmentImage)
etree.SubElement(outputSubTagSummary, 'tag',name='Class', value=str(prediction_c))
etree.SubElement(outputSubTagSummary, 'tag', name='Class Confidence', value=str(confidence_c))
self.bqSession.finish_mex(tags = [outputTag])
log.debug('FINISHED')
self.bqSession.close()
def main(self):
"""
The main function that runs everything
"""
print("DEBUG_INIT")
log.debug('SkeletonPython is called with the following arguments')
log.debug('sysargv : %s\n\n' % sys.argv )
parser = OptionParser()
parser.add_option( '--image_url' , dest="image_url")
parser.add_option( '--mex_url' , dest="mexURL")
parser.add_option( '--module_dir' , dest="modulePath")
parser.add_option( '--staging_path', dest="stagingPath")
parser.add_option( '--bisque_token', dest="token")
parser.add_option( '--user' , dest="user")
parser.add_option( '--pwd' , dest="pwd")
parser.add_option( '--root' , dest="root")
(options, args) = parser.parse_args()
# Set up the mexURL and token based on the arguments passed to the script
try: #pull out the mex
log.debug("options %s" % options)
if not options.mexURL:
options.mexURL = sys.argv[1]
if not options.token:
options.token = sys.argv[2]
except IndexError: #no argv were set
pass
if not options.stagingPath:
options.stagingPath = ''
# Still don't have an imgurl, but it will be set up in self.setup()
log.debug('\n\nPARAMS : %s \n\n Options: %s'%(args, options))
self.options = options
if self.validateInput():
try: #run setup and retrieve mex variables
self.setup()
except Exception, e:
log.exception("Exception during setup")
self.bqSession.fail_mex(msg = "Exception during setup: %s" % str(e))
return
try: #run module operation
self.run()
except SkeletonPythonError, e:
log.exception("Exception during run")
self.bqSession.fail_mex(msg = "Exception during run: %s" % str(e.message))
return
except Exception, e:
log.exception("Exception during run")
self.bqSession.fail_mex(msg = "Exception during run: %s" % str(e))
return
class SkeletonPython(object):
"""
SkeletonPython Model
"""
def mex_parameter_parser(self, mex_xml):
"""
Parses input of the xml and add it to SkeletonPython Trainer's options attribute
@param: mex_xml
"""
mex_inputs = mex_xml.xpath('tag[@name="inputs"]')
if mex_inputs:
for tag in mex_inputs[0]:
if tag.tag == 'tag' and tag.attrib['type'] != 'system-input':
log.debug('Set options with %s as %s' %
(tag.attrib['name'], tag.attrib['value']))
setattr(self.options, tag.attrib['name'],
tag.attrib['value'])
else:
log.debug('SkeletonPythonFS: No Inputs Found on MEX!')
def validateInput(self):
"""
Parses input of the xml and add it to SkeletonPython's options attribute
@param: mex_xml
"""
if (self.options.mexURL
and self.options.token): #run module through engine service
return True
if (self.options.user and self.options.pwd and
self.options.root): #run module locally (note: to test module)
return True
log.debug(
'SkeletonPython: Insufficient options or arguments to start this module'
)
return False
def setup(self):
"""
Fetches the mex, appends input_configurations to the option
attribute of SkeletonPython and looks up the model on bisque to
classify the provided resource.
"""
if (self.options.user and self.options.pwd and self.options.root):
self.bqSession = BQSession().init_local(
self.options.user,
self.options.pwd,
bisque_root=self.options.root)
self.options.mexURL = self.bqSession.mex.uri
# This is when the module actually runs on the server with a mexURL and an access token
elif (self.options.mexURL and self.options.token):
self.bqSession = BQSession().init_mex(self.options.mexURL,
self.options.token)
else:
return
# Parse the xml and construct the tree, also set options to proper values after parsing it (like image url)
self.mex_parameter_parser(self.bqSession.mex.xmltree)
log.debug(
'SkeletonPython: image URL: %s, mexURL: %s, stagingPath: %s, token: %s'
% (self.options.image_url, self.options.mexURL,
self.options.stagingPath, self.options.token))
def run(self):
"""
The core of the SkeletonPython Module
Requests features on the image provided. Classifies each tile
and picks a majority among the tiles.
"""
r_xml = self.bqSession.fetchxml(self.options.mexURL, view='deep')
'''rectangles = r_xml.xpath('//tag[@name="inputs"]/tag[@name="image_url"]/gobject[@name="roi"]/rectangle')
rois = []
rois_rectangles = []
for i in range(len(rectangles)):
x1 = int(float(rectangles[i][0].attrib['x']))
y1 = int(float(rectangles[i][0].attrib['y']))
x2 = int(float(rectangles[i][1].attrib['x']))
y2 = int(float(rectangles[i][1].attrib['y']))
rois_rectangles.append([x1,y1,x2,y2])
polygons = r_xml.xpath('//tag[@name="inputs"]/tag[@name="image_url"]/gobject[@name="roi"]/polygon')
rois_polygons = []
for i in range(len(polygons)):
polygon = []
for j in range(len(polygons[i])):
x = int(float(polygons[i][j].attrib['x']))
y = int(float(polygons[i][j].attrib['y']))
polygon.append([x,y])
rois_polygons.append(polygon)
polylines = r_xml.xpath('//tag[@name="inputs"]/tag[@name="image_url"]/gobject[@name="roi"]/polyline')
rois_polylines = []
for i in range(len(polylines)):
polyline = []
for j in range(len(polylines[i])):
x = int(float(polylines[i][j].attrib['x']))
y = int(float(polylines[i][j].attrib['y']))
polyline.append([x,y])
rois_polylines.append(polyline)
rois.append(rois_polylines)
rois.append(rois_polygons)
rois.append(rois_rectangles)'''
def teardown(self):
"""
Posting results to the mex
"""
self.bqSession.update_mex('Returning results...')
'''log.debug('Returning results...')
prediction = "None-Module Failure"
with open("./results.txt","r") as f:
for line in f:
if "PREDICTION_C:" in line:
prediction_c = line
if "CONFIDENCE_C:" in line:
confidence_c = line[14:-1]
if "PREDICTION_T:" in line:
prediction_t = line
if "CONFIDENCE_T:" in line:
confidence_t = line[14:-1]
print line
classes = ["flower","stem","fruit","entire","leaf"]
classes_type = ["sheet","natural"]
for i,class_tag in enumerate(classes):
prediction_c = prediction_c.replace(str(i),class_tag)
for i,class_tag in enumerate(classes_type):
prediction_t = prediction_t.replace(str(i),class_tag)
'''
outputTag = etree.Element('tag', name='outputs')
#dda = etree.SubElement(outputTag, 'tag', name='mex_url', value=self.options.image_url)
outputSubTagImage = etree.SubElement(outputTag,
'tag',
name='OutputImage',
value=self.options.image_url)
'''gob = etree.SubElement (outputSubTagImage, 'gobject', name='Annotations', type='Annotations')
xc = [100, 300, 200]
yc = [100, 150, 200]
polyseg = etree.SubElement(gob, 'polygon', name='SEG')
etree.SubElement( polyseg, 'tag', name='color', value="#0000FF")
for i in range(len(xc)):
etree.SubElement (polyseg, 'vertex', x=str(1*int(yc[i])), y=str(1*int(xc[i])))
print outputTag'''
'''print "Module will output image, {}".format(self.options.image_url)
if not os.path.isfile("./contours.pkl"):
print "Module will not segment image, (were foreground and background polyline annotations provided?)"
if self.options.segmentImage != "False" and os.path.isfile("./contours.pkl"):
[contours, t_scale] = pickle.load(open("./contours.pkl","rb"))
xc = []
yc = []
for i, p in enumerate(contours):
if i < len(contours)/2:
xc.append(p)
else:
yc.append(p)
gob = etree.SubElement (outputSubTagImage, 'gobject', name='Annotations', type='Annotations')
polyseg = etree.SubElement(gob, 'polygon', name='SEG')
etree.SubElement( polyseg, 'tag', name='color', value="#0000FF")
print "TSCALE", t_scale
for i in range(len(xc)):
etree.SubElement (polyseg, 'vertex', x=str(t_scale[1]*int(yc[i])), y=str(t_scale[0]*int(xc[i])))
###outputSubTagSummary = etree.SubElement(outputTag, 'tag', name='summary')
#etree.SubElement(outputSubTagSummary, 'tag',name='Model File', value=self.options.deepNetworkChoice)
#etree.SubElement(outputSubTagSummary, 'tag',name='Segment Image', value=self.options.segmentImage)
###etree.SubElement(outputSubTagSummary, 'tag',name='Class', value=str(prediction_c))
###etree.SubElement(outputSubTagSummary, 'tag', name='Class Confidence', value=str(confidence_c))
#etree.SubElement(outputSubTagSummary, 'tag',name='Type', value=str(prediction_t))
#etree.SubElement(outputSubTagSummary, 'tag', name='Type Confidence', value=str(confidence_t))
'''
self.bqSession.finish_mex(tags=[outputTag])
log.debug('FINISHED')
self.bqSession.close()
def main(self):
"""
The main function that runs everything
"""
print("DEBUG_INIT")
log.debug('SkeletonPython is called with the following arguments')
log.debug('sysargv : %s\n\n' % sys.argv)
parser = OptionParser()
parser.add_option('--image_url', dest="image_url")
parser.add_option('--mex_url', dest="mexURL")
parser.add_option('--module_dir', dest="modulePath")
parser.add_option('--staging_path', dest="stagingPath")
parser.add_option('--bisque_token', dest="token")
parser.add_option('--user', dest="user")
parser.add_option('--pwd', dest="pwd")
parser.add_option('--root', dest="root")
(options, args) = parser.parse_args()
# Set up the mexURL and token based on the arguments passed to the script
try: #pull out the mex
log.debug("options %s" % options)
if not options.mexURL:
options.mexURL = sys.argv[1]
if not options.token:
options.token = sys.argv[2]
except IndexError: #no argv were set
pass
if not options.stagingPath:
options.stagingPath = ''
# Still don't have an imgurl, but it will be set up in self.setup()
log.debug('\n\nPARAMS : %s \n\n Options: %s' % (args, options))
self.options = options
if self.validateInput():
try: #run setup and retrieve mex variables
self.setup()
except Exception, e:
log.exception("Exception during setup")
self.bqSession.fail_mex(msg="Exception during setup: %s" %
str(e))
return
try: #run module operation
self.run()
except SkeletonPythonError, e:
log.exception("Exception during run")
self.bqSession.fail_mex(msg="Exception during run: %s" %
str(e.message))
return
except Exception, e:
log.exception("Exception during run")
self.bqSession.fail_mex(msg="Exception during run: %s" %
str(e))
return
class Botanicam(object):
"""
Botanicam Model
"""
def mex_parameter_parser(self, mex_xml):
"""
Parses input of the xml and add it to BotanicamTrainer's options attribute
@param: mex_xml
"""
mex_inputs = mex_xml.xpath('tag[@name="inputs"]')
if mex_inputs:
for tag in mex_inputs[0]:
if tag.tag == 'tag' and tag.attrib['type'] != 'system-input':
log.debug('Set options with %s as %s' %
(tag.attrib['name'], tag.attrib['value']))
setattr(self.options, tag.attrib['name'],
tag.attrib['value'])
else:
log.debug('BotanicamFS: No Inputs Found on MEX!')
def validateInput(self):
"""
Parses input of the xml and add it to Botanicam's options attribute
@param: mex_xml
"""
if (self.options.mexURL
and self.options.token): #run module through engine service
return True
if (self.options.user and self.options.pwd and
self.options.root): #run module locally (note: to test module)
return True
log.debug(
'Botanicam: Insufficient options or arguments to start this module'
)
return False
def setup(self):
"""
Fetches the mex, appends input_configurations to the option
attribute of Botanicam and looks up the model on bisque to
classify the provided resource.
"""
log.debug('Initializing Mex...')
if (self.options.user and self.options.pwd and self.options.root):
self.bqSession = BQSession().init_local(
self.options.user,
self.options.pwd,
bisque_root=self.options.root)
self.options.mexURL = self.bqSession.mex.uri
elif (self.options.mexURL and self.options.token):
self.bqSession = BQSession().init_mex(self.options.mexURL,
self.options.token)
else:
return
self.mex_parameter_parser(self.bqSession.mex.xmltree)
#finds and opens model file
self.bqSession.update_mex('Initializing Classification Model...')
log.debug('Forming Feature Requests...')
#no options currently
#combo = mex_xml.xpath('tag[@name="plant_part"]/@value')[0]
combo = 'bush'
if combo:
if combo == 'bush':
MODEL_QUERY[
'tag_query'] = '"module_identifier":"Botanicam" AND "Classification Method":"Bush Descriptor"'
elif combo == 'leaf':
MODEL_QUERY[
'tag_query'] = '"module_identifier":"Botanicam" AND "Classification Method":"Leaf Descriptor"'
else:
raise BotanicamError(
'The incorrect model type was found -> Model Type: %s' %
combo)
else:
raise BotanicamError('No model type was choosen')
query_xml = self.bqSession.fetchxml('/data_service/file',
**MODEL_QUERY)
self.options.model_url = None
if len(query_xml) > 0:
try:
model_url = query_xml[0].attrib['uri']
self.options.model_url = model_url
log.debug('Fetching Model @ %s' % model_url)
self.model_xml = self.bqSession.fetchxml(model_url,
view='deep')
self.model_path = os.path.join(self.options.stagingPath,
'model')
model = self.bqSession.load(model_url)
model_url = self.bqSession.service_url(
'blob_service', path=model.resource_uniq)
self.bqSession.fetchblob(model_url,
path=self.model_path + '.zip')
with zipfile.ZipFile(self.model_path + '.zip') as dirzip:
dirzip.extractall(self.model_path)
except BQCommError:
raise BotanicamError(
'Model file was not found! Ask admin to set the correct model file'
)
else: #run demo classifier model store in the module
raise BotanicamError(
'No model file was found. Ask your admin to train a new model with \
the Botanicam Trainer.')
self.bqSession.update_mex('Initialized...')
log.debug(
'Botanicam: image URL: %s, mexURL: %s, stagingPath: %s, token: %s'
% (self.options.image_url, self.options.mexURL,
self.options.stagingPath, self.options.token))
def run(self):
"""
The core of the Botanicam Module
Requests features on the image provided. Classifies each tile
and picks a majority among the tiles.
"""
#parse requests
self.bqSession.update_mex('Calculating Features...')
log.debug('Forming Feature Requests...')
#get rectanle gobjects for roi
r_xml = self.bqSession.fetchxml(self.options.mexURL, view='deep')
rectangles = r_xml.xpath(
'//tag[@name="inputs"]/tag[@name="image_url"]/gobject[@name="roi"]/rectangle'
)
image_xml = self.bqSession.fetchxml(self.options.image_url)
image_url = self.bqSession.service_url(
'image_service', path=image_xml.attrib['resource_uniq'])
if rectangles: #On chooses the first rectangle
#construct operation node
x1 = int(float(rectangles[0][0].attrib['x']))
y1 = int(float(rectangles[0][0].attrib['y']))
x2 = int(float(rectangles[0][1].attrib['x']))
y2 = int(float(rectangles[0][1].attrib['y']))
log.debug('Adding Crop: roi=%s,%s,%s,%s' % (x1, y1, x2, y2))
image_url = self.bqSession.c.prepare_url(image_url,
roi='%s,%s,%s,%s' %
(x1, y1, x2, y2))
try:
feature_vectors = extract_bush_feature(self.bqSession, image_url)
except FeatureError as e:
raise BotanicamError(str(e))
#parse features
self.bqSession.update_mex('Classifying Results...')
log.debug('Classifying Results...')
results = []
pca = joblib.load(os.path.join(self.model_path, 'pca_model'))
clf = joblib.load(os.path.join(self.model_path, 'svm_model'))
for f in feature_vectors:
f_norm = pca.transform(f)
results.append(int(clf.predict(f_norm)))
class_count = np.bincount(np.array(results))
self.class_number = np.argmax(class_count)
self.confidence = float(
class_count[self.class_number]) / np.sum(class_count)
log.debug('Found Class %s' % str(self.class_number))
def teardown(self):
"""
Posting results to the mex
"""
self.bqSession.update_mex('Returning results...')
log.debug('Returning results...')
tag_list = self.model_xml.xpath(
'tag[@name="Classes"]/tag[@value="%s"]' %
str(self.class_number))[0]
outputTag = etree.Element('tag', name='outputs')
outputSubTag = etree.SubElement(outputTag, 'tag', name='summary')
if self.options.model_url:
etree.SubElement(outputSubTag,
'tag',
name='Model File',
value=self.options.model_url,
type='url')
else:
etree.SubElement(outputSubTag,
'tag',
name='Model File',
value='Internal Model File')
etree.SubElement(outputSubTag,
'tag',
name='Class',
value=str(self.class_number))
query = []
for t in tag_list:
etree.SubElement(outputSubTag,
'tag',
name=t.attrib['name'],
value=t.attrib['value'])
query.append('"%s":"%s"' % (t.attrib['name'], t.attrib['value']))
query = ' & '.join(query)
etree.SubElement(outputSubTag,
'tag',
name='confidence',
value=str(self.confidence))
etree.SubElement(outputTag,
'tag',
name='similar_images',
value=query,
type='browser')
self.bqSession.finish_mex(tags=[outputTag])
log.debug('FINISHED')
self.bqSession.close()
def main(self):
"""
The main function that runs everything
"""
log.debug('sysargv : %s' % sys.argv)
parser = OptionParser()
parser.add_option('--image_url', dest="image_url")
parser.add_option('--mex_url', dest="mexURL")
parser.add_option('--module_dir', dest="modulePath")
parser.add_option('--staging_path', dest="stagingPath")
parser.add_option('--bisque_token', dest="token")
parser.add_option('--user', dest="user")
parser.add_option('--pwd', dest="pwd")
parser.add_option('--root', dest="root")
(options, args) = parser.parse_args()
try: #pull out the mex
if not options.mexURL:
options.mexURL = sys.argv[1]
if not options.token:
options.token = sys.argv[2]
except IndexError: #no argv were set
pass
if not options.stagingPath:
options.stagingPath = ''
log.debug('\n\nPARAMS : %s \n\n Options: %s' % (args, options))
self.options = options
if self.validateInput():
try: #run setup and retrieve mex variables
self.setup()
except Exception, e:
log.exception("Exception during setup")
self.bqSession.fail_mex(msg="Exception during setup: %s" %
str(e))
return
try: #run module operation
self.run()
except BotanicamError, e:
log.exception("Exception during run")
self.bqSession.fail_mex(msg="Exception during run: %s" %
str(e.message))
return
except Exception, e:
log.exception("Exception during run")
self.bqSession.fail_mex(msg="Exception during run: %s" %
str(e))
return
class PlanteomeDeepSegment(object):
# +
# __init__()
# -
def __init__(self):
# entry message
log.debug('{}.__init__()> message on entry'.format(MODULE_NAME))
# declare some variables and initialize them
self.options = None
self.bqSession = None
self.rois = None
self.message = None
# get full path(s) to file(s)
self.contours_file = os.path.abspath(
os.path.expanduser(PICKLE_CONTOURS_FILE))
self.data_file = os.path.abspath(os.path.expanduser(PICKLE_DATA_FILE))
self.results_file = os.path.abspath(
os.path.expanduser(TEXT_RESULTS_FILE))
self.tiff_file = os.path.abspath(os.path.expanduser(TIFF_IMAGE_FILE))
self.csv_leaf_file = os.path.abspath(os.path.expanduser(CSV_LEAF_FILE))
log.debug('{}.__init()> self.contours_file={}'.format(
MODULE_NAME, self.contours_file))
log.debug('{}.__init()> self.data_file={}'.format(
MODULE_NAME, self.data_file))
log.debug('{}.__init()> self.results_file={}'.format(
MODULE_NAME, self.results_file))
log.debug('{}.__init()> self.tiff_file={}'.format(
MODULE_NAME, self.tiff_file))
log.debug('{}.__init()> self.csv_leaf_file={}'.format(
MODULE_NAME, self.csv_leaf_file))
# exit message
log.debug('{}.__init__()> message on exit'.format(MODULE_NAME))
# +
# hidden method: _mex_parameter_parser()
# -
def _mex_parameter_parser(self, mex_xml=None):
# entry message
log.debug(
'{}._mex_parameter_parser()> message on entry, mex_xml={}'.format(
MODULE_NAME, str(mex_xml)))
if mex_xml is not None:
mex_inputs = mex_xml.xpath('tag[@name="inputs"]')
if mex_inputs:
for tag in mex_inputs[0]:
if tag.tag == 'tag' and tag.attrib[
'type'] != 'system-input':
_name = tag.attrib['name'].strip()
_value = tag.attrib['value'].strip()
log.debug(
'{}._mex_parameter_parser()> setting self.options.{}={}'
.format(MODULE_NAME, _name, _value))
setattr(self.options, _name, _value)
log.debug(
"{}._mex_parameter_parser()> set self.options.{}={}"
.format(MODULE_NAME, _name,
getattr(self.options, _name)))
else:
log.error('{}.mex_parameter_parser()> no inputs found on mex!'.
format(MODULE_NAME))
else:
self.message = '{}.mex_parameter_parser()> mex_xml is None'.format(
MODULE_NAME)
log.error(self.message)
# exit message
log.debug('{}.main()> message on exit, options={}'.format(
MODULE_NAME, self.options))
# +
# hidden method: _validate_input()
# -
def _validate_input(self):
# entry message
retval = False
log.debug('{}._validate_input()> message on entry, retval={}'.format(
MODULE_NAME, retval))
# run module through engine_service (default)
if self.options.mexURL and self.options.token:
retval = True
# run module locally
elif self.options.user and self.options.pwd and self.options.root:
retval = True
else:
retval = False
log.error(
'{}.validate_input()> insufficient options or arguments to start this module'
.format(MODULE_NAME))
# exit message
log.debug('{}._validate_input()> message on exit, retval={}'.format(
MODULE_NAME, retval))
return retval
# +
# hidden method: _construct_vertices()
# -
def _construct_vertices(self, child=None):
# entry message
vertices = None
roi = []
log.debug(
'{}._construct_vertices()> message on entry, child={}'.format(
MODULE_NAME, str(child)))
# get annotation type
if child is not None:
annotation_type = 'fg' if 'foreground' in child.values() else 'bg'
# get vertices
vertices = child.getchildren()[0].getchildren()
for _vertex in vertices:
_values = _vertex.values()
roi.append({
'x': int(float(_values[2])),
'y': int(float(_values[3]))
})
log.debug(
'{}._construct_vertices()> ROI: appending {} value with {}'.
format(MODULE_NAME, annotation_type, str(roi)))
self.rois[annotation_type].append(roi)
# exit message
log.debug(
'{}._construct_vertices()> message on exit, vertices={}, length={}, rois={}'
.format(MODULE_NAME, str(vertices), len(vertices), str(self.rois)))
# +
# hidden method: _show_structure()
# -
def _show_structure(self, r_xml=None):
# entry message
log.debug('{}._show_structure()> message on entry, r_xml={}'.format(
MODULE_NAME, str(r_xml)))
if r_xml is not None:
for _i, _child in enumerate(r_xml.getchildren()):
log.debug('{}._show_structure()> index={}, child={}'.format(
MODULE_NAME, _i, str(_child)))
log.debug('{}._show_structure()> index={}, values={}'.format(
MODULE_NAME, _i, str(_child.values())))
if 'background' in _child.values(
) or 'foreground' in _child.values():
self._construct_vertices(_child)
else:
self._show_structure(_child)
# exit message
log.debug('{}._show_structure()> message on exit'.format(MODULE_NAME))
# +
# method: setup()
# -
def setup(self):
# entry message
log.debug('{}.setup()> message on entry, options={}'.format(
MODULE_NAME, self.options))
# run locally
if self.options.user and self.options.pwd and self.options.root:
log.debug(
'{}.setup()> running locally with user={}, pwd={}, root={}'.
format(MODULE_NAME, self.options.user, self.options.pwd,
self.options.root))
self.bqSession = BQSession().init_local(
self.options.user,
self.options.pwd,
bisque_root=self.options.root)
self.options.mexURL = self.bqSession.mex.uri
# run on the server with a mexURL and an access token
elif self.options.mexURL and self.options.token:
log.debug('{}.setup()> running on server with mexURL={}, token={}'.
format(MODULE_NAME, self.options.mexURL,
self.options.token))
self.bqSession = BQSession().init_mex(self.options.mexURL,
self.options.token)
# failed to connect to bisque
else:
self.bqSession = None
self.message(
'{}.setup()> failed to connect to bisque'.format(MODULE_NAME))
log.error(self.message)
raise PlanteomeDeepSegmentError(self.message)
# parse the xml and construct the tree, also set options to proper values after parsing it
if self.bqSession is not None:
self._mex_parameter_parser(self.bqSession.mex.xmltree)
log.debug(
'{}.setup()> image URL={}, mexURL={}, stagingPath={}, token={}'
.format(MODULE_NAME, self.options.image_url,
self.options.mexURL, self.options.stagingPath,
self.options.token))
# exit message
log.debug('{}.setup()> message on exit, options={}'.format(
MODULE_NAME, self.options))
# +
# method: run()
# The core of the PlanteomeDeepSegment module. It requests features on the provided image, classifies each tile
# and selects a majority amongst the tiles.
# -
def run(self):
# entry message
log.debug('{}.run()> message on entry, options={}'.format(
MODULE_NAME, self.options))
self.rois = {'fg': [], 'bg': []}
r_xml = self.bqSession.fetchxml(self.options.mexURL, view='deep')
log.debug('{}.run()> Shols structura'.format(MODULE_NAME))
self._show_structure(r_xml)
log.debug(self.rois)
# dump image as .tiff
image = self.bqSession.load(self.options.image_url)
ip = image.pixels().format('tiff')
with open(self.tiff_file, 'wb') as f:
f.write(ip.fetch())
# pickle the data
try:
if self.rois and getattr(self.options, 'segmentImage') != '' and \
getattr(self.options, 'deepNetworkChoice') != '' and getattr(self.options, 'qualitySeg') != '' and \
getattr(self.options, 'deepSeg') != '' and getattr(self.options, 'mexURL') != '' and \
getattr(self.options, 'token') != '':
log.debug('{}.run()> pickling data to {}'.format(
MODULE_NAME, self.data_file))
pickle.dump([
self.rois, self.options.segmentImage,
self.options.deepNetworkChoice, self.options.qualitySeg,
self.options.deepSeg, self.options.mexURL,
self.options.token
], open(self.data_file, 'wb'))
except AttributeError as e:
self.message('{}.run()> failed to pickle data, e={}'.format(
MODULE_NAME, str(e)))
log.error(self.message)
# do something
x = PlanteomeDeepSegmentLearning(self.contours_file, self.data_file,
self.tiff_file, self.results_file)
x.main()
# exit message
log.debug('{}.run()> message on exit, options={}'.format(
MODULE_NAME, self.options))
# +
# method: teardown()
# -
def teardown(self):
# entry message
log.debug('{}.teardown()> message on entry, options={}'.format(
MODULE_NAME, self.options))
# set up tag(s)
self.bqSession.update_mex('Returning results...')
output_tag = eTree.Element('tag', name='outputs')
output_sub_tag_image = eTree.SubElement(output_tag,
'tag',
name='Final Image',
value=self.options.image_url)
output_sub_tag_summary = eTree.SubElement(output_tag,
'tag',
name='summary')
log.info('Module will output image {}'.format(self.options.image_url))
# segment the image (if required)
if getattr(self.options, 'segmentImage', '') != '' and self.options.segmentImage.lower() == 'true' and \
os.path.isfile(self.contours_file):
log.debug(
'{}.teardown()> module will segment image from file {}'.format(
MODULE_NAME, self.contours_file))
eTree.SubElement(output_sub_tag_summary,
'tag',
name='Segment Image',
value=self.options.segmentImage)
[_contours, _t_scale] = pickle.load(open(self.contours_file, 'rb'))
_gob = eTree.SubElement(output_sub_tag_image,
'gobject',
name='Annotations',
type='Annotations')
_polyseg = eTree.SubElement(_gob, 'polygon', name='SEG')
eTree.SubElement(_polyseg, 'tag', name='color', value="#0000FF")
_opd = 0
_output_sampling = 1 + int(len(_contours) / 100)
for _j in range(len(_contours)):
if _j % _output_sampling == 0:
_opd += 1
_x = str(1 + int(_t_scale[1] * _contours[_j][1]))
_y = str(1 + int(_t_scale[0] * _contours[_j][0]))
eTree.SubElement(_polyseg, 'vertex', x=_x, y=_y)
log.debug('{}.teardown()> _opd={}'.format(MODULE_NAME, _opd))
else:
log.info(
'Module will not segment image, (were foreground and background polyline annotations provided?)'
)
# select deepNetworkChoice
opts = getattr(self.options, 'deepNetworkChoice', '')
eTree.SubElement(output_sub_tag_summary,
'tag',
name='Model File',
value=opts)
if opts == '':
log.error('{}.teardown()> deepNetworkChoice={}'.format(
MODULE_NAME, self.options.deepNetworkChoice))
else:
# simple classification
if opts.split()[0].lower() == 'simple':
# get prediction
prediction_c = -1
confidence_c = 0.0
prediction_t = -1
confidence_t = 0.0
try:
with open(self.results_file, 'r') as f:
for _line in f:
if _line.strip() != '':
log.debug('{}.teardown()> _line={}'.format(
MODULE_NAME, _line))
if 'PREDICTION_C:' in _line:
prediction_c = int(
_line.split(':')[1].strip())
if 'CONFIDENCE_C:' in _line:
confidence_c = float(
_line.split(':')[1].strip())
if 'PREDICTION_T:' in _line:
prediction_t = int(
_line.split(':')[1].strip())
if 'CONFIDENCE_T:' in _line:
confidence_t = float(
_line.split(':')[1].strip())
except IOError as e:
self.message = '{}.teardown()> io error reading results, e={}'.format(
MODULE_NAME, str(e))
log.error(self.message)
finally:
log.debug('{}.teardown()> prediction_c={}'.format(
MODULE_NAME, prediction_c))
log.debug('{}.teardown()> confidence_c={}'.format(
MODULE_NAME, confidence_c))
log.debug('{}.teardown()> prediction_t={}'.format(
MODULE_NAME, prediction_t))
log.debug('{}.teardown()> confidence_t={}'.format(
MODULE_NAME, confidence_t))
# annotate with prediction
classes = [
'Leaf (PO:0025034): http://browser.planteome.org/amigo/term/PO:0025034',
'Fruit (PO:0009001): http://browser.planteome.org/amigo/term/PO:0009001',
'Flower (PO:0009046): http://browser.planteome.org/amigo/term/PO:0009046',
'Stem (PO:0009047): http://browser.planteome.org/amigo/term/PO:0009047',
'Whole plant (PO:0000003): http://browser.planteome.org/amigo/term/PO:0000003 '
]
prediction_c = classes[prediction_c] if (
0 <= prediction_c <= len(classes)) else 'unknown'
eTree.SubElement(output_sub_tag_summary,
'tag',
name='Class',
value=prediction_c)
if prediction_c.lower() != 'unknown':
eTree.SubElement(
output_sub_tag_summary,
'tag',
type='link',
name='Class link',
value=prediction_c.split('):')[-1].strip())
eTree.SubElement(output_sub_tag_summary,
'tag',
name='Class Confidence',
value=str(confidence_c))
# leaf classification
elif opts.split()[0].lower() == 'leaf':
log.debug('{}.teardown()> leaf_targets{}'.format(
MODULE_NAME, leaf_targets))
log.debug('{}.teardown()> leaf_targets_links{}'.format(
MODULE_NAME, leaf_targets_links))
# map each leaf target to the corresponding PO term
with open(self.csv_leaf_file) as cf:
reader = csv.reader(cf, delimiter=',', quotechar='|')
_cn = ''
for _row in reader:
_n = _row[0] if _row[0] != '' else 'undefined'
_m = _row[1] if _row[1] != '' else 'undefined'
# _c = _row[2] if _row[2] != '' else 'undefined'
_t = _n.replace(' ', '').lower()
# get the current name
for _lc in leaf_keys_nospaces:
if _t == _lc:
_cn = leaf_keys_spaces[
leaf_keys_nospaces.index(_lc)]
break
# replace dictionary entry is mapping exists
if _cn in leaf_targets:
for _l in leaf_targets[_cn]:
if _n == _l.replace(' ', ''):
_i = leaf_targets[_cn].index(_l)
leaf_targets_links[_cn][_i] = _m
break
# read result(s)
with open(self.results_file, 'r') as f:
class_list = []
for _i, _line in enumerate(f):
# remove after introduction of the leaf classifier (below start with appends)
if int(_line) == len(leaf_targets[leaf_keys[_i]]) - 1:
_line = '0'
class_list.append(_line)
eTree.SubElement(output_sub_tag_summary,
'tag',
name='{}-Name'.format(
leaf_keys_proper_names[_i]),
value=leaf_targets[leaf_keys[_i]][int(
class_list[_i])])
if leaf_targets_links[leaf_keys[_i]][int(
class_list[_i])] != 'undefined':
eTree.SubElement(
output_sub_tag_summary,
'tag',
type='link',
name='{}-Accession'.format(
leaf_keys_proper_names[_i]),
value=
'http://browser.planteome.org/amigo/term/{}'.
format(leaf_targets_links[leaf_keys[_i]][int(
class_list[_i])]))
else:
eTree.SubElement(
output_sub_tag_summary,
'tag',
name='{}-Accession'.format(
leaf_keys_proper_names[_i]),
value=leaf_targets_links[leaf_keys[_i]][int(
class_list[_i])])
# update mex
self.bqSession.finish_mex(tags=[output_tag])
self.bqSession.close()
# exit message
log.debug('{}.teardown()> message on exit, options={}'.format(
MODULE_NAME, self.options))
# +
# method: main()
# -
def main(self):
# entry message
log.debug('{}.main()> message on entry, args={}'.format(
MODULE_NAME, sys.argv))
parser = OptionParser()
parser.add_option('--image_url', dest="image_url")
parser.add_option('--mex_url', dest="mexURL")
parser.add_option('--module_dir', dest="modulePath")
parser.add_option('--staging_path', dest="stagingPath")
parser.add_option('--bisque_token', dest="token")
parser.add_option('--user', dest="user")
parser.add_option('--pwd', dest="pwd")
parser.add_option('--root', dest="root")
(options, args) = parser.parse_args()
log.debug('{}.main()> options={}'.format(MODULE_NAME, options))
log.debug('{}.main()> args={}'.format(MODULE_NAME, args))
# set up the mexURL and token based on the arguments passed to the script
try:
if not options.mexURL:
options.mexURL = sys.argv[1]
if not options.token:
options.token = sys.argv[2]
if not options.stagingPath:
options.stagingPath = ''
except IndexError:
pass
finally:
self.options = options
log.debug('{}.main()> self.options={}'.format(
MODULE_NAME, self.options))
# check input(s)
if self._validate_input():
# noinspection PyBroadException
try:
# set up the module
self.setup()
except PlanteomeDeepSegmentError as e:
self.message = '{}.main()> specific exception after setup(), e={}'.format(
MODULE_NAME, str(e.errstr))
log.exception(self.message)
self.bqSession.fail_mex(msg=self.message)
raise PlanteomeDeepSegmentError(self.message)
except Exception as e:
self.message = '{}.main()> exception after setup(), e={}'.format(
MODULE_NAME, str(e))
log.exception(self.message)
self.bqSession.fail_mex(msg=self.message)
raise PlanteomeDeepSegmentError(self.message)
except:
self.message = '{}.main()> error after setup()'.format(
MODULE_NAME)
log.exception(self.message)
self.bqSession.fail_mex(msg=self.message)
raise PlanteomeDeepSegmentError(self.message)
# noinspection PyBroadException
try:
# run the module
self.run()
except PlanteomeDeepSegmentError as e:
self.message = '{}.main()> specific exception after run(), e={}'.format(
MODULE_NAME, str(e.errstr))
log.exception(self.message)
self.bqSession.fail_mex(msg=self.message)
raise PlanteomeDeepSegmentError(self.message)
except Exception as e:
self.message = '{}.main()> exception after run(), e={}'.format(
MODULE_NAME, str(e))
log.exception(self.message)
self.bqSession.fail_mex(msg=self.message)
raise PlanteomeDeepSegmentError(self.message)
except:
self.message = '{}.main()> error after run()'.format(
MODULE_NAME)
log.exception(self.message)
self.bqSession.fail_mex(msg=self.message)
raise PlanteomeDeepSegmentError(self.message)
# noinspection PyBroadException
try:
# tear down the module
self.teardown()
except PlanteomeDeepSegmentError as e:
self.message = '{}.main()> specific exception after teardown(), e={}'.format(
MODULE_NAME, str(e.errstr))
log.exception(self.message)
self.bqSession.fail_mex(msg=self.message)
raise PlanteomeDeepSegmentError(self.message)
except Exception as e:
self.message = '{}.main()> exception after teardown(), e={}'.format(
MODULE_NAME, str(e))
log.exception(self.message)
self.bqSession.fail_mex(msg=self.message)
raise PlanteomeDeepSegmentError(self.message)
except:
self.message = '{}.main()> error after teardown()'.format(
MODULE_NAME)
log.exception(self.message)
self.bqSession.fail_mex(msg=self.message)
raise PlanteomeDeepSegmentError(self.message)
else:
self.message = '{}.main()> failed to validate instance'.format(
MODULE_NAME)
log.error(self.message)
self.bqSession.fail_mex(msg=self.message)
raise PlanteomeDeepSegmentError(self.message)
# exit message
log.debug('{}.main()> message on exit, args={}'.format(
MODULE_NAME, sys.argv))