def main(self, mex_url=None, bisque_token=None, bq=None):
# Allow for testing by passing an alreay initialized session
if bq is None:
bq = BQSession().init_mex(mex_url, bisque_token)
pars = bq.parameters()
image_url = pars.get('dataset_url', None)
annotation_type = pars.get('annotation_type', None)
annotation_attribute = pars.get('annotation_attribute', None)
value_old = pars.get('value_old', None)
bq.update_mex('Starting')
mex_id = mex_url.split('/')[-1]
if image_url is None or len(image_url) < 2:
datasets = bq.fetchxml('/data_service/dataset')
datasets = [d.get('uri') for d in datasets.xpath('dataset')]
else:
datasets = [image_url]
total = 0
changes = []
# delete annotations for each element in the dataset
for ds_url in datasets:
dataset = bq.fetchxml(ds_url, view='deep')
dataset_name = dataset.get('name')
bq.update_mex('processing "%s"' % dataset_name)
refs = dataset.xpath('value[@type="object"]')
for r in refs:
url = r.text
image = bq.fetchxml(url, view='deep')
uuid = image.get('resource_uniq')
modified = delete(image,
value_old,
ann_type=annotation_type,
ann_attr=annotation_attribute)
if len(modified) > 0:
bq.postxml(url, image, method='PUT')
total = total + len(modified)
changes.append({
'name':
'%s (%s)' %
(image.get('name'), image.get('resource_uniq')),
'value':
'%s' % len(modified)
})
changes.insert(0, {'name': 'Total', 'value': '%s' % total})
bq.finish_mex(tags=[{
'name': 'outputs',
'tag': [{
'name': 'deleted',
'tag': changes,
}]
}])
fn = data['ImageName'][i].strip(' ')
m = {}
for k in Ks:
v = data[k][i]
if isinstance(v, basestring):
m[k] = v.strip(' ')
else:
m[k] = v
meta[fn] = m
##################################################################
# load dataset
##################################################################
query_url = '/data_service/dataset?tag_query=@name:%s&offset=0&limit=10'%(dataset_name)
q = session.fetchxml (query_url)
ds = q.xpath('dataset')
if len(ds)>1:
print 'Stopping: found more than one dataset with a given name: %s'%dataset_name
sys.exit()
dataset_url = ds[0].get('uri')
dataset = session.fetchxml (dataset_url, view='deep')
images = dataset.xpath('value')
for m in images:
image_url = m.text
image = session.fetchxml (image_url, view='deep')
if image.tag != 'image':
continue
#print etree.tostring(image)
class ImageServiceTestBase(unittest.TestCase):
"""
Test image service operations
"""
@classmethod
def setUpClass(self):
config = ConfigParser.ConfigParser()
config.read('config.cfg')
self.root = config.get('Host', 'root') or 'localhost:8080'
self.user = config.get('Host', 'user') or 'test'
self.pswd = config.get('Host', 'password') or 'test'
self.session = BQSession().init_local(self.user,
self.pswd,
bisque_root=self.root,
create_mex=False)
# download and upload test images ang get their IDs
#self.uniq_2d_uint8 = self.ensure_bisque_file(image_rgb_uint8)
#self.uniq_3d_uint16 = self.ensure_bisque_file(image_zstack_uint16)
@classmethod
def tearDownClass(self):
#self.delete_resource(self.uniq_2d_uint8)
#self.delete_resource(self.uniq_3d_uint16)
self.cleanup_tests_dir()
pass
@classmethod
def fetch_file(self, filename):
_mkdir(local_store_images)
_mkdir(local_store_tests)
url = posixpath.join(url_image_store, filename).encode('utf-8')
path = os.path.join(local_store_images, filename)
if not os.path.exists(path):
urllib.urlretrieve(url, path)
return path
@classmethod
def upload_file(self, path, resource=None):
#if resource is not None:
# print etree.tostring(resource)
r = save_blob(self.session, path, resource=resource)
if r is None or r.get('uri') is None:
print 'Error uploading: %s' % path.encode('ascii', 'replace')
return None
print 'Uploaded id: %s url: %s' % (r.get('resource_uniq'),
r.get('uri'))
return r
@classmethod
def delete_resource(self, r):
if r is None:
return
url = r.get('uri')
print 'Deleting id: %s url: %s' % (r.get('resource_uniq'), url)
self.session.deletexml(url)
@classmethod
def delete_package(self, package):
if 'dataset' in package:
# delete dataset
url = package['dataset']
print 'Deleting dataset: %s' % (url)
try:
self.session.fetchxml('/dataset_service/delete?duri=%s' % url)
except BQCommError:
print 'Error deleting the dataset'
elif 'items' in package:
# delete all items
for url in package['items']:
print 'Deleting item: %s' % (url)
try:
self.session.deletexml(url)
except BQCommError:
print 'Error deleting the item'
# # delete dataset
# if 'dataset' in package:
# url = package['dataset']
# print 'Deleting dataset: %s'%(url)
# try:
# self.session.deletexml(url)
# except BQCommError:
# print 'Error deleting the dataset'
# # delete all items
# if 'items' in package:
# for url in package['items']:
# print 'Deleting item: %s'%(url)
# try:
# self.session.deletexml(url)
# except BQCommError:
# print 'Error deleting the item'
@classmethod
def ensure_bisque_file(self, filename, metafile=None):
path = self.fetch_file(filename)
if metafile is None:
filename = u'%s/%s' % (TEST_PATH, filename)
resource = etree.Element('resource', name=filename)
return self.upload_file(path, resource=resource)
else:
metafile = self.fetch_file(metafile)
return self.upload_file(path,
resource=etree.parse(metafile).getroot())
@classmethod
def ensure_bisque_package(self, package):
path = self.fetch_file(package['file'])
r = self.upload_file(path, resource=etree.XML(package['resource']))
package['resource'] = r
if r is None:
return None
print 'Uploaded id: %s url: %s' % (r.get('resource_uniq'),
r.get('uri'))
#print etree.tostring(r)
if r.tag != 'dataset':
package['items'] = [r.get('uri')]
else:
package['dataset'] = r.get('uri')
values = r.xpath('value')
if len(values) != package['count']:
print 'Error: uploaded %s has %s elements but needs %s' % (
package['file'], len(values), package['count'])
if r.get('name') != package['name']:
print 'Error: uploaded %s name is %s but should be %s' % (
package['file'], r.get('name'), package['name'])
package['items'] = [x.text for x in values]
package['last'] = self.session.fetchxml(package['items'][-1],
view='deep')
#print 'Last item\n'
#print etree.tostring(package['last'])
@classmethod
def cleanup_tests_dir(self):
print 'Cleaning-up %s' % local_store_tests
for root, dirs, files in os.walk(local_store_tests, topdown=False):
for name in files:
os.remove(os.path.join(root, name))
def validate_image_variant(self,
resource,
filename,
commands,
meta_required=None):
path = os.path.join(local_store_tests, filename)
try:
#image = fromXml(resource, session=self.session)
image = self.session.factory.from_etree(resource)
px = image.pixels()
for c, a in commands:
px = px.command(c, a)
px.fetch(path)
except BQCommError:
logging.exception('Comm error')
self.fail('Communication error while fetching image')
if meta_required is not None:
meta_test = metadata_read(path)
self.assertTrue(meta_test is not None,
msg='Retrieved image can not be read')
self.assertTrue(
compare_info(meta_required, meta_test),
msg='Retrieved metadata differs from test template')
def validate_xml(self, resource, filename, commands, xml_parts_required):
path = os.path.join(local_store_tests, filename)
try:
#image = fromXml(resource, session=self.session)
image = self.session.factory.from_etree(resource)
px = image.pixels()
for c, a in commands:
px = px.command(c, a)
px.fetch(path)
except BQCommError:
self.fail()
xml_test = etree.parse(path).getroot()
#print etree.tostring(xml_test)
self.assertTrue(compare_xml(xml_parts_required, xml_test),
msg='Retrieved XML differs from test template')
class SeedSize(object):
def setup(self):
if not os.path.exists(self.images):
os.makedirs(self.images)
self.bq.update_mex('initializing')
if self.is_dataset:
results = fetch_dataset(self.bq, self.resource_url, self.images)
else:
results = fetch_image_pixels(self.bq, self.resource_url, self.images)
with open(self.image_map_name, 'wb') as f:
pickle.dump(results, f)
return 0
def start(self):
self.bq.update_mex('executing')
# Matlab requires trailing slash
r = subprocess.call([EXEC, 'images/'])
return r
def teardown(self):
with open(self.image_map_name, 'rb') as f:
self.url2file = pickle.load(f) #
self.file2url = dict((v,k) for k,v in self.url2file.iteritems())
summary = os.path.join(self.images, 'summary.csv')
if not os.path.exists (summary):
self.bq.fail_mex (msg = "did not find any seeds: missing %s" % summary)
return 0
summary_tags = self._read_summary(summary)
# Post all submex for files and return xml list of results
tags = []
gobjects = []
submexes = []
if not self.is_dataset:
localfiles = glob.glob(os.path.join(self.images, '*C.csv'))
gobs = self._read_results(localfiles[0])
#tags = [{ 'name':'image_url', 'value' : self.resource_url}]
tags = [{ 'name': 'outputs',
'tag' : [{'name': 'Summary', 'tag' : summary_tags} ,
{'name': 'seed-resource', 'type':'image', 'value':self.resource_url,
'gobject' : [{ 'name': 'seeds', 'type' : 'seedsize', 'gobject':gobs }], }],
}]
else:
submexes = self._get_submexes()
tags = [
{ 'name': 'execute_options',
'tag' : [ {'name': 'iterable', 'value' : 'image_url' } ]
},
{ 'name': 'outputs',
'tag' : [{'name': 'Summary', 'tag' : summary_tags },
{'name': 'mex_url', 'value': self.mex_url, 'type': 'mex'},
{'name': 'image_url', 'type':'dataset', 'value':self.resource_url,}]
},
]
# for i, submex in enumerate(mexlist):
# tag, image_url = gettag(submex, 'image_url')
# gob, gob_url = gettag(submex, 'SeedSize')
# mexlink = { 'name' : 'submex',
# 'tag' : [{ 'name':'mex_url', 'value':submex.get('uri')},
# { 'name':'image_url', 'value' : image_url},
# { 'name':'gobject_url', 'value' : gob.get('uri') } ]
# }
# tags.append(mexlink)
self.bq.finish_mex(tags = tags, gobjects = gobjects, children= [('mex', submexes)])
return 0
def _get_submexes(self):
submex = []
localfiles = glob.glob(os.path.join(self.images, '*C.csv'))
result2url = dict( (os.path.splitext(f)[0] + 'C.csv', u) for f, u in self.file2url.items())
for result in localfiles:
gobs = self._read_results(result)
if result not in result2url:
logging.error ("Can't find url for %s given files %s and map %s" %
result, localfiles, result2url)
mex = { 'type' : self.bq.mex.type,
'name' : self.bq.mex.name,
'value': 'FINISHED',
'tag': [ { 'name': 'inputs',
'tag' : [ {'name': 'image_url', 'value' : result2url [result] } ]
},
{ 'name': 'outputs',
'tag' : [{'name': 'seed-resource', 'type':'image', 'value': result2url [result],
'gobject':{ 'name': 'seeds', 'type': 'seedsize', 'gobject': gobs}, }] }]
}
submex.append (mex)
return submex
#url = self.bq.service_url('data_service', 'mex', query={ 'view' : 'deep' })
#response = self.bq.postxml(url, d2xml({'request' : {'mex': submex}} ))
#return response
def _read_summary(self, csvfile):
#%mean(area), mean(minoraxislen), mean(majoraxislen), standarddev(area),
#standarddev(minoraxislen), standarddev(majoraxislen), total seedcount,
#mean thresholdused, weighted mean of percentclusters1, weighted mean of percentclusters2
f= open(csvfile,'rb')
rows = csv.reader (f)
tag_names = [ 'mean_area', 'mean_minoraxis', 'mean_majoraxis',
'std_area', 'std_minoraxis', 'std_majoraxis',
'seedcount',
'mean_threshhold',
'weighted_mean_cluster_1','weighted_mean_cluster_2',
]
# Read one row(rows.next()) and zip ( name, col) unpacking in d2xml format
summary_tags = [ { 'name': n[0], 'value' : n[1] }
for n in itertools.izip(tag_names, rows.next()) ]
f.close()
return summary_tags
def _read_results(self, csvfile):
results = []
f= open(csvfile,'rb')
rows = csv.reader (f)
for col in rows:
results.append( {
'type' : 'seed',
'tag' : [ { 'name': 'area', 'value': col[0]},
{ 'name': 'major', 'value': col[2]},
{ 'name': 'minor', 'value': col[1]} ],
'ellipse' : {
'vertex' : [ { 'x': col[3], 'y':col[4], 'index':0 },
{ 'x': float(col[3]) - abs(float(col[8]) - float(col[3])), 'y':col[9], 'index':1 },
{ 'x': col[6], 'y':col[7], 'index':2 }]
}
})
f.close()
return results
def run(self):
logging.basicConfig(level=logging.DEBUG)
parser = optparse.OptionParser()
parser.add_option('-d','--debug', action="store_true")
parser.add_option('-n','--dryrun', action="store_true")
#parser.add_option('--resource_url')
#parser.add_option('--mex_url')
#parser.add_option('--staging_path')
#parser.add_option('--bisque_token')
#parser.add_option('--credentials')
# Parse named arguments from list
(options, args) = parser.parse_args()
named_args =dict( [ y for y in [ x.split ('=') for x in args ] if len (y) == 2] )
args = [ x for x in args if '=' not in x ]
staging_path = '.'
self.auth_token = named_args.get ('bisque_token')
self.image_map_name = os.path.join(staging_path, IMAGE_MAP)
self.resource_url = named_args.get ('image_url')
self.mex_url = named_args.get ('mex_url')
self.images = os.path.join(staging_path, 'images') + os.sep
if self.auth_token:
self.bq = BQSession().init_mex(self.mex_url, self.auth_token)
else:
user,pwd = options.credentials.split(':')
self.bq = BQSession().init_local(user,pwd)
resource_xml = self.bq.fetchxml (self.resource_url, view='short')
self.is_dataset = resource_xml.tag == 'dataset'
if len(args) == 1:
commands = [ args.pop(0)]
else:
commands =['setup','start', 'teardown']
#if command not in ('setup','teardown', 'start'):
# parser.error('Command must be start, setup or teardown')
# maltab code requires trailing slash..
try:
for command in commands:
command = getattr(self, command)
r = command()
except Exception, e:
logging.exception ("problem during %s" % command)
self.bq.fail_mex(msg = "Exception during %s: %s" % (command, str(e)))
sys.exit(1)
sys.exit(r)
image_uri = sys.argv[1]
config = ConfigParser.ConfigParser()
config.read('config.cfg')
root = config.get('Host', 'root') or 'localhost:8080'
user = config.get('Host', 'user') or 'test'
pswd = config.get('Host', 'password') or 'test'
session = BQSession().init_local(user,
pswd,
bisque_root=root,
create_mex=False)
# fetch image sizes
meta = session.fetchxml('%s?meta' %
image_uri.replace('data_service', 'image_service'))
w = int(meta.xpath('//tag[@name="image_num_x"]')[0].get('value'))
h = int(meta.xpath('//tag[@name="image_num_y"]')[0].get('value'))
z = int(meta.xpath('//tag[@name="image_num_z"]')[0].get('value'))
t = int(meta.xpath('//tag[@name="image_num_t"]')[0].get('value'))
# create mex
mex = etree.Element('mex',
name='ExternalAnnotator %s' % num_gobs,
value='FINISHED')
inputs = etree.SubElement(mex, 'tag', name='inputs')
etree.SubElement(inputs,
'tag',
name='resource_url',
type='image',
class BaseRunner(object):
"""Base runner for ways of running a module in some runtime evironment.
Runtime environments include the command line, condor and hadoop
Each runner basically prepares the module environment, runs or stops
modules, and allows the status of a run to be queried
Runners interact with module environments (see base_env)
For each module it is expected that custom launcher will be written.
A simple example might be:
from bisque.util.launcher import Launcher
class MyModuleLauncher (Launcher):
execute='myrealmodule'
...
if __name__ == "__main__":
MyModuleLauncher().main()
The engine will launch a subprocess with the following template:
launcher arg1 arg2 arg3 mex=http://somehost/ms/mex/1212 start
The launcher code will strip off the last 2 arguments and
pass the other arguments to the real module.
The last argument is command it must be one of the following
start, stop, status
"""
name = "Base"
env = None # A mapping of environment variables (or Inherit)
executable = [] # A command line list of the arguments
environments = []
launcher = None # Use Default launcher
mexhandled = "true"
def __init__(self, **kw):
self.parser = optparse.OptionParser()
self.parser.add_option("-n",
"--dryrun",
action="store_true",
default=False)
self.parser.add_option('-v',
'--verbose',
action="store_true",
default=False)
self.parser.add_option('-d',
'--debug',
action="store_true",
default=False)
self.session = None
self.process_environment = dict(os.environ)
self.log = logging.getLogger("bq.engine_service.BaseRunner")
self.mexid = None
self.mexes = []
self.prerun = None
self.postrun = None
self.entrypoint_executable = None
self.iterables = None
def debug(self, msg, *args):
#if self.options.verbose:
self.log.debug(msg, *args)
def info(self, msg, *args):
self.log.info(msg, *args)
def error(self, msg, *args):
self.log.error(msg, *args)
###########################################
# Config
def read_config(self, **kw):
"""Initial state of a runner. The module-runtime.cfg is read
and the relevent runner section is applied. The environment
list is created and setup in order to construct the
environments next
"""
self.config = AttrDict(executable="", environments="")
self.sections = {}
self.debug("BaseRunner: read_config")
# Load any bisque related variable into this runner
runtime_bisque_cfg = find_config_path('runtime-bisque.cfg')
if os.path.exists(runtime_bisque_cfg):
self.bisque_cfg = ConfigFile(runtime_bisque_cfg)
self.load_section(None, self.bisque_cfg)
self.config['files'] = self.config.setdefault(
'files', '') + ',' + runtime_bisque_cfg
else:
self.info("BaseRunner: missing runtime-bisque.cfg")
module_dir = kw.get('module_dir', os.getcwd())
runtime_module_cfg = os.path.join(module_dir, 'runtime-module.cfg')
if not os.path.exists(runtime_module_cfg):
self.info("BaseRunner: missing %s", runtime_module_cfg)
return
self.module_cfg = ConfigFile(runtime_module_cfg)
# Process Command section
self.load_section(None, self.module_cfg) # Globals
def load_section(self, name, cfg):
name = name or "__GLOBAL__"
section = self.sections.setdefault(name, {})
section.update(cfg.get(name, asdict=True))
self.config.update(section)
#for k,v in section.items():
# setattr(self,k,v)
#########################################################
# Helpers
def create_environments(self, **kw):
"""Build the set of environments listed by the module config"""
self.environments = strtolist(self.config.environments)
envs = []
for name in self.environments:
env = ENV_MAP.get(name, None)
if env is not None:
envs.append(env(runner=self))
continue
self.log.warn('Unknown environment: %s ignoring', name)
self.environments = envs
self.info('created environments %s', envs)
def process_config(self, **kw):
"""Configuration occurs in several passes.
1. read config file and associated runtime section
create environments
2. Process the config values converting types
"""
self.mexhandled = strtobool(self.mexhandled)
# Find any configuration parameters in the envs
# and add them to the class
for env in self.environments:
env.process_config(self)
def setup_environments(self, **kw):
"""Call setup_environment during "start" processing
Prepares environment and returns defined environment variable to be passed to jobs
"""
self.debug("setup_environments: %s", kw)
process_env = self.process_environment.copy()
for env in self.environments:
env.setup_environment(self, **kw)
self.process_environment = process_env
# Run during finish
def teardown_environments(self, **kw):
'Call teardown_environment during "finish" processing'
for env in reversed(self.environments):
env.teardown_environment(self, **kw)
##########################################
# Initialize run state
def init_runstate(self, arguments, **kw):
"""Initialize mex and module and deal with other arguments.
"""
mex_tree = kw.pop('mex_tree', None)
module_tree = kw.pop('module_tree', None)
bisque_token = kw.pop('bisque_token', None)
module_dir = kw.pop('module_dir', os.getcwd())
# ---- the next items are preserved across execution phases ----
# list of dict representing each mex : variables and arguments
self.mexes = []
self.rundir = module_dir
self.outputs = []
self.entrypoint_executable = []
self.prerun = None
self.postrun = None
#self.process_environment (pre-set)
#self.options (pre-set)
# ---- the previous items are preserved across execution phases ----
# Add remaining arguments to the executable line
# Ensure the loaded executable is a list
if isinstance(self.config.executable, str):
executable = shlex.split(self.config.executable)
#self.executable.extend (arguments)
topmex = AttrDict(self.config)
topmex.update(
dict(
named_args={},
executable=list(executable),
# arguments = [],
mex_url=mex_tree is not None and mex_tree.get('uri') or None,
bisque_token=bisque_token,
staging_path=None,
rundir=self.rundir))
topmex.mex_id = topmex.mex_url.rsplit('/',
1)[1] if topmex.mex_url else ''
self.mexes.append(topmex)
# Scan argument looking for named arguments (these are used by condor_dag to re-initialize mex vars)
for arg in arguments:
tag, sep, val = arg.partition('=')
if sep == '=':
topmex.named_args[tag] = val
if tag in topmex:
topmex[tag] = val
# Pull out arguments from mex
if mex_tree is not None and module_tree is not None:
mexparser = MexParser()
mex_inputs = mexparser.prepare_inputs(module_tree, mex_tree,
bisque_token)
module_options = mexparser.prepare_options(module_tree, mex_tree)
self.outputs = mexparser.prepare_outputs(module_tree, mex_tree)
argument_style = module_options.get('argument_style', 'positional')
# see if we have pre/postrun option
self.prerun = module_options.get('prerun_entrypoint', None)
self.postrun = module_options.get('postrun_entrypoint', None)
topmex.named_args.update(mexparser.prepare_mex_params(mex_inputs))
topmex.executable.extend(
mexparser.prepare_mex_params(mex_inputs, argument_style))
topmex.rundir = self.rundir
#topmex.options = module_options
# remember topmex executable for pre/post runs
# Create a nested list of arguments (in case of submex)
submexes = mex_tree.xpath('/mex/mex')
for mex in submexes:
sub_inputs = mexparser.prepare_inputs(module_tree, mex)
submex = AttrDict(self.config)
submex.update(
dict(
named_args=dict(topmex.named_args),
# arguments =list(topmex.arguments),
executable=list(executable), #+ topmex.arguments,
mex_url=mex.get('uri'),
mex_id=mex.get('uri').rsplit('/', 1)[1],
bisque_token=bisque_token,
staging_path=None,
rundir=self.rundir))
#if argument_style == 'named':
# submex.named_args.update ( [x.split('=') for x in sub_inputs] )
submex.named_args.update(
mexparser.prepare_mex_params(sub_inputs))
submex.executable.extend(
mexparser.prepare_mex_params(sub_inputs, argument_style))
self.mexes.append(submex)
# Submex's imply that we are iterated.
# We can set up some options here and remove any execution
# for the top mex.
topmex.iterables = len(
self.mexes) > 1 and mexparser.process_iterables(
module_tree, mex_tree)
self.info("processing %d mexes -> %s" % (len(self.mexes), self.mexes))
def store_runstate(self):
staging_path = self.mexes[0].get('staging_path') or '.'
state_file = "state%s.bq" % self.mexes[0].get('mex_id', '')
# pickle the object variables
with open(os.path.join(staging_path, state_file), 'wb') as f:
pickle.dump(self.mexes, f)
pickle.dump(self.rundir, f)
pickle.dump([
et.tostring(tree)
for tree in self.outputs if tree and tree.tag == 'tag'
], f)
pickle.dump(self.entrypoint_executable, f)
pickle.dump(self.prerun, f)
pickle.dump(self.postrun, f)
pickle.dump(self.process_environment, f)
pickle.dump(self.options, f)
def load_runstate(self):
staging_path = self.mexes[0].get('staging_path') or '.'
state_file = "state%s.bq" % self.mexes[0].get('mex_id', '')
# entered in a later processing phase (e.g., Condor finish) => unpickle
with open(os.path.join(staging_path, state_file), 'rb') as f:
#self.pool = None # not preserved for now
self.mexes = pickle.load(f)
self.rundir = pickle.load(f)
self.outputs = [et.fromstring(tree) for tree in pickle.load(f)]
self.entrypoint_executable = pickle.load(f)
self.prerun = pickle.load(f)
self.postrun = pickle.load(f)
self.process_environment = pickle.load(f)
self.options = pickle.load(f)
##################################################
# Command sections
# A command is launched with last argument on the command line of the launcher
# Each command must return the either the next fommand to run or None to stop
# Derived classes should overload these functions
def command_start(self, **kw):
self.info("starting %d mexes -> %s", len(self.mexes), self.mexes)
if self.session is None:
self.session = BQSession().init_mex(self.mexes[0].mex_url,
self.mexes[0].bisque_token)
status = "starting"
self.entrypoint_executable = self.mexes[0].executable
if self.mexes[0].iterables:
self.mexes[0].executable = None
status = 'running parallel'
# add empty "outputs" section in topmex
#self.session.update_mex(status=status, tags=[{'name':'outputs'}])
self.session.update_mex(
status=status
) # dima: modules add outputs section and the empty one complicates module UI
# if there is a prerun, run it now
if self.prerun:
self.info("prerun starting")
self.command_single_entrypoint(self.prerun, self.command_execute,
**kw)
return None
return self.command_execute
def command_execute(self, **kw):
"""Execute the internal executable"""
return self.command_finish
def command_finish(self, **kw):
"""Cleanup the environment and perform any needed actions
after the module completion
"""
self.info("finishing %d mexes -> %s", len(self.mexes), self.mexes)
# if there is a postrun (aka "reduce phase"), run it now
if self.postrun:
self.info("postrun starting")
self.command_single_entrypoint(self.postrun, self.command_finish2,
**kw)
return None
self.command_finish2(**kw)
def command_finish2(self, **kw):
self.teardown_environments()
if self.mexes[0].iterables:
if self.session is None:
self.session = BQSession().init_mex(self.mexes[0].mex_url,
self.mexes[0].bisque_token)
# outputs
# mex_url
# dataset_url (if present and no list/range iterable)
# multiparam (if at least one list/range iterable)
need_multiparam = False
for iter_name, iter_val, iter_type in self.mexes[0].iterables:
if iter_type in ['list', 'range']:
need_multiparam = True
break
outtag = et.Element('tag', name='outputs')
if need_multiparam:
# some list/range params => add single multiparam element if allowed by module def
multiparam_name = None
for output in self.outputs:
if output.get('type', '') == 'multiparam':
multiparam_name = output.get('name')
break
if multiparam_name:
# get inputs section for some submex to read actual types later
# TODO: this can be simplified with xpath_query in 0.6
inputs_subtree = None
if len(self.mexes) > 1:
submextree = self.session.fetchxml(
self.mexes[1].mex_url,
view='full') # get latest MEX doc
inputs_subtree = submextree.xpath(
'./tag[@name="inputs"]')
inputs_subtree = inputs_subtree and self.session.fetchxml(
inputs_subtree[0].get('uri'), view='deep')
multitag = et.SubElement(outtag,
'tag',
name=multiparam_name,
type='multiparam')
colnames = et.SubElement(multitag, 'tag', name='title')
coltypes = et.SubElement(multitag, 'tag', name='xmap')
colxpaths = et.SubElement(multitag, 'tag', name='xpath')
et.SubElement(multitag,
'tag',
name='xreduce',
value='vector')
for iter_name, iter_val, iter_type in self.mexes[
0].iterables:
actual_type = inputs_subtree and inputs_subtree.xpath(
'.//tag[@name="%s" and @type]/@type'
) # read actual types from any submex
actual_type = actual_type[
0] if actual_type else 'string'
et.SubElement(colnames, 'value', value=iter_name)
et.SubElement(coltypes,
'value',
value="tag-value-%s" % actual_type)
et.SubElement(colxpaths,
'value',
value='./mex//tag[@name="%s"]' %
iter_name)
# last column is the submex URI
et.SubElement(colnames, 'value', value="submex_uri")
et.SubElement(coltypes, 'value', value="resource-uri")
et.SubElement(colxpaths, 'value', value='./mex')
else:
self.log.warn(
"List or range parameters in Mex but no multiparam output tag in Module"
)
else:
# no list/range params => add iterables as always
for iter_name, iter_val, iter_type in self.mexes[0].iterables:
et.SubElement(outtag,
'tag',
name=iter_name,
value=iter_val,
type=iter_type)
et.SubElement(outtag,
'tag',
name='mex_url',
value=self.mexes[0].mex_url,
type='mex')
self.session.finish_mex(tags=[outtag])
return None
def command_single_entrypoint(self, entrypoint, callback, **kw):
return None
def command_kill(self, **kw):
"""Kill the running module if possible
"""
return False
def command_status(self, **kw):
return None
def check(self, module_tree=None, **kw):
"check whether the module seems to be runnable"
self.read_config(**kw)
# check for a disabled module
enabled = self.config.get('module_enabled', 'true').lower() == "true"
if not enabled:
self.info('Module is disabled')
return False
# Add remaining arguments to the executable line
# Ensure the loaded executable is a list
if isinstance(self.config.executable, str):
executable = shlex.split(self.config.executable)
if os.name == 'nt':
return True
canrun = executable and which(executable[0]) is not None
if not canrun:
self.error("Executable cannot be run %s" % executable)
return canrun
def main(self, **kw):
# Find and read a config file for the module
created_pool = False
try:
self.pool = kw.pop('pool', None)
if self.pool is None:
self.pool = ProcessManager(POOL_SIZE)
created_pool = True
self.read_config(**kw)
args = kw.pop('arguments', None)
# Pull out command line arguments
self.options, arguments = self.parser.parse_args(args)
command_str = arguments.pop()
# the following always has to run first so that e.g., staging dirs are set up properly
self.init_runstate(arguments, **kw)
self.create_environments(**kw)
self.process_config(**kw)
if command_str == 'start':
# new run => setup environments from scratch
self.setup_environments()
else:
# continued run => load saved run state
try:
self.load_runstate()
except OSError:
# could not load state => OK if it was command that can run without previous 'start' (e.g., 'kill')
if command_str not in ['kill']:
raise
self.mexes[0].arguments = arguments
command = getattr(self, 'command_%s' % command_str, None)
while command:
self.info("COMMAND_RUNNER %s" % command)
command = command(**kw)
if command_str not in ['kill']:
# store run state since we may come back (e.g., condor finish)
self.store_runstate()
if created_pool:
self.pool.stop()
return 0
except ModuleEnvironmentError, e:
self.log.exception("Problem occured in module")
raise RunnerException(str(e), self.mexes)
except RunnerException, e:
raise
