def main(self, image_url, mex_url=None, bisque_token=None, bq=None):
# Allow for testing by passing an alreay initialized session
if bq is None:
bq = BQSession().init_mex(mex_url, bisque_token)
# Fetch the image metadata
image = bq.load(image_url)
# Fetch embedded tags from image service
meta = image.pixels().meta().fetch()
meta = ET.XML(meta)
tags = []
# Create a new tag 'MetaData' to be placed on the image
md = BQTag(name='MetaData')
# Filter the embedded metadata and place subtags in MetaData
for t in meta.getiterator('tag'):
if t.get('name') in wanted_tags:
md.addTag(name=t.get('name'), value=t.get('value'))
# Add the new tag to the image
image.addTag(tag=md)
metadata_tag = bq.save(md, image.uri + "/tag")
bq.finish_mex(tags=[{
'name':
'outputs',
'tag': [{
'name': 'metadata',
'value': metadata_tag.uri,
'type': 'tag'
}]
}])
sys.exit(0)
def main(self, mex_url=None, bisque_token=None, bq=None):
# Allow for testing by passing an alreay initialized session
if bq is None:
bq = BQSession().init_mex(mex_url, bisque_token)
pars = bq.parameters()
image_url = pars.get('dataset_url', None)
annotation_type = pars.get('annotation_type', None)
annotation_attribute = pars.get('annotation_attribute', None)
value_old = pars.get('value_old', None)
bq.update_mex('Starting')
mex_id = mex_url.split('/')[-1]
if image_url is None or len(image_url) < 2:
datasets = bq.fetchxml('/data_service/dataset')
datasets = [d.get('uri') for d in datasets.xpath('dataset')]
else:
datasets = [image_url]
total = 0
changes = []
# delete annotations for each element in the dataset
for ds_url in datasets:
dataset = bq.fetchxml(ds_url, view='deep')
dataset_name = dataset.get('name')
bq.update_mex('processing "%s"' % dataset_name)
refs = dataset.xpath('value[@type="object"]')
for r in refs:
url = r.text
image = bq.fetchxml(url, view='deep')
uuid = image.get('resource_uniq')
modified = delete(image,
value_old,
ann_type=annotation_type,
ann_attr=annotation_attribute)
if len(modified) > 0:
bq.postxml(url, image, method='PUT')
total = total + len(modified)
changes.append({
'name':
'%s (%s)' %
(image.get('name'), image.get('resource_uniq')),
'value':
'%s' % len(modified)
})
changes.insert(0, {'name': 'Total', 'value': '%s' % total})
bq.finish_mex(tags=[{
'name': 'outputs',
'tag': [{
'name': 'deleted',
'tag': changes,
}]
}])
def main(self, mex_url=None, bisque_token=None, image_url=None, bq=None):
# Allow for testing by passing an alreay initialized session
if bq is None:
bq = BQSession().init_mex(mex_url, bisque_token)
bq.update_mex('Classifying...')
pars = bq.parameters()
image_url = pars.get('data_url', None)
model_url = pars.get('model_url', None)
store_on_image = pars.get('store_on_image', False)
method = pars.get('method', 'points')
points = int(pars.get('number_of_points', 10))
confidence = int(pars.get('confidence', 95))
border = int(pars.get('border', 0))
_, image_uniq = image_url.rsplit('/', 1)
_, model_uniq = model_url.rsplit('/', 1)
url = '%s/connoisseur/%s/classify:%s/method:%s/points:%s/confidence:%s/border:%s' % (
bq.bisque_root, model_uniq, image_uniq, method, points, confidence,
border)
# dima: the moving of the image is not of the best taste, this should be changed later
if method != 'segmentation':
url += '/format:xml'
txt = bq.c.fetch(url,
headers={
'Content-Type': 'text/xml',
'Accept': 'text/xml'
})
gobs = etree.fromstring(txt)
else:
color_mode = 'colors'
url += '/colors:colors'
filename = '%s_%s_conf%.2f_n%s_b%s.png' % (
image_uniq, color_mode, confidence, points, border)
filename = bq.c.fetch(url, path=filename)
image_url = upload_file(bq, filename) or ''
gobs = None
bq.update_mex('Storing results...')
outputs = etree.Element('tag', name="outputs")
img = etree.SubElement(outputs,
'tag',
name="MyImage",
type="image",
value=image_url)
if gobs is not None:
img.append(gobs)
if store_on_image is True:
r = bq.postxml(image_url, xml=txt)
bq.finish_mex(tags=[outputs])
def main(self, image_url, mex_url = None, bisque_token=None, bq = None, args = None):
# Allow for testing by passing an alreay initialized session
if bq is None:
bq = BQSession().init_mex(mex_url, bisque_token)
# Fetch the blob links
if not os.path.exists ('videos'):
os.makedirs ('videos')
video = fetch_blob(bq, image_url, 'videos')
print "VIDEO file ", video
#pass arguments to MotionMeerkat scripts, located in MotionMeerkat/main.py
#Structure arguments
#Format call string
callargs = ["python MotionMeerkat/main.py",
"--i", video.values()[0],
"--threshT", args[0],
"--sub", args[1],
"--mogh", args[2],
"--mogv", args[3],
"--accA", args[4],
"--burn", args[5],
"--frameSET",
"--frame_rate", "1",
"--makeV", "none",
"--fileD", "Output"]
print "Calling ", " ".join(callargs)
#run MotionMeerkat
r = call(" ".join(callargs), shell=True)
if r != 0:
bq.fail_mex ("Meerkat returned non-zero")
#Post Results
#get filename to get the destination of the csv to be posted
fn=video.values()[0]
fbs= os.path.basename(fn)
head, tail = os.path.splitext(fbs)
#post file
frames_blob = bq.postblob(str("Output/" + head + "/Frames.csv"))
#get file location from regex
uri=re.search("uri=\"(.*?)\"", frames_blob).group(1)
tags = [{ 'name': 'outputs','tag' : [{'name': 'frames_csv', 'type':'file', 'value':uri}]}]
bq.finish_mex(tags = tags)
sys.exit(0)
class BaseRunner(object):
"""Base runner for ways of running a module in some runtime evironment.
Runtime environments include the command line, condor and hadoop
Each runner basically prepares the module environment, runs or stops
modules, and allows the status of a run to be queried
Runners interact with module environments (see base_env)
For each module it is expected that custom launcher will be written.
A simple example might be:
from bisque.util.launcher import Launcher
class MyModuleLauncher (Launcher):
execute='myrealmodule'
...
if __name__ == "__main__":
MyModuleLauncher().main()
The engine will launch a subprocess with the following template:
launcher arg1 arg2 arg3 mex=http://somehost/ms/mex/1212 start
The launcher code will strip off the last 2 arguments and
pass the other arguments to the real module.
The last argument is command it must be one of the following
start, stop, status
"""
name = "Base"
env = None # A mapping of environment variables (or Inherit)
executable = [] # A command line list of the arguments
environments = []
launcher = None # Use Default launcher
mexhandled = "true"
def __init__(self, **kw):
self.parser = optparse.OptionParser()
self.parser.add_option("-n",
"--dryrun",
action="store_true",
default=False)
self.parser.add_option('-v',
'--verbose',
action="store_true",
default=False)
self.parser.add_option('-d',
'--debug',
action="store_true",
default=False)
self.session = None
self.process_environment = dict(os.environ)
self.log = logging.getLogger("bq.engine_service.BaseRunner")
self.mexid = None
self.mexes = []
self.prerun = None
self.postrun = None
self.entrypoint_executable = None
self.iterables = None
def debug(self, msg, *args):
#if self.options.verbose:
self.log.debug(msg, *args)
def info(self, msg, *args):
self.log.info(msg, *args)
def error(self, msg, *args):
self.log.error(msg, *args)
###########################################
# Config
def read_config(self, **kw):
"""Initial state of a runner. The module-runtime.cfg is read
and the relevent runner section is applied. The environment
list is created and setup in order to construct the
environments next
"""
self.config = AttrDict(executable="", environments="")
self.sections = {}
self.debug("BaseRunner: read_config")
# Load any bisque related variable into this runner
runtime_bisque_cfg = find_config_path('runtime-bisque.cfg')
if os.path.exists(runtime_bisque_cfg):
self.bisque_cfg = ConfigFile(runtime_bisque_cfg)
self.load_section(None, self.bisque_cfg)
self.config['files'] = self.config.setdefault(
'files', '') + ',' + runtime_bisque_cfg
else:
self.info("BaseRunner: missing runtime-bisque.cfg")
module_dir = kw.get('module_dir', os.getcwd())
runtime_module_cfg = os.path.join(module_dir, 'runtime-module.cfg')
if not os.path.exists(runtime_module_cfg):
self.info("BaseRunner: missing %s", runtime_module_cfg)
return
self.module_cfg = ConfigFile(runtime_module_cfg)
# Process Command section
self.load_section(None, self.module_cfg) # Globals
def load_section(self, name, cfg):
name = name or "__GLOBAL__"
section = self.sections.setdefault(name, {})
section.update(cfg.get(name, asdict=True))
self.config.update(section)
#for k,v in section.items():
# setattr(self,k,v)
#########################################################
# Helpers
def create_environments(self, **kw):
"""Build the set of environments listed by the module config"""
self.environments = strtolist(self.config.environments)
envs = []
for name in self.environments:
env = ENV_MAP.get(name, None)
if env is not None:
envs.append(env(runner=self))
continue
self.log.warn('Unknown environment: %s ignoring', name)
self.environments = envs
self.info('created environments %s', envs)
def process_config(self, **kw):
"""Configuration occurs in several passes.
1. read config file and associated runtime section
create environments
2. Process the config values converting types
"""
self.mexhandled = strtobool(self.mexhandled)
# Find any configuration parameters in the envs
# and add them to the class
for env in self.environments:
env.process_config(self)
def setup_environments(self, **kw):
"""Call setup_environment during "start" processing
Prepares environment and returns defined environment variable to be passed to jobs
"""
self.debug("setup_environments: %s", kw)
process_env = self.process_environment.copy()
for env in self.environments:
env.setup_environment(self, **kw)
self.process_environment = process_env
# Run during finish
def teardown_environments(self, **kw):
'Call teardown_environment during "finish" processing'
for env in reversed(self.environments):
env.teardown_environment(self, **kw)
##########################################
# Initialize run state
def init_runstate(self, arguments, **kw):
"""Initialize mex and module and deal with other arguments.
"""
mex_tree = kw.pop('mex_tree', None)
module_tree = kw.pop('module_tree', None)
bisque_token = kw.pop('bisque_token', None)
module_dir = kw.pop('module_dir', os.getcwd())
# ---- the next items are preserved across execution phases ----
# list of dict representing each mex : variables and arguments
self.mexes = []
self.rundir = module_dir
self.outputs = []
self.entrypoint_executable = []
self.prerun = None
self.postrun = None
#self.process_environment (pre-set)
#self.options (pre-set)
# ---- the previous items are preserved across execution phases ----
# Add remaining arguments to the executable line
# Ensure the loaded executable is a list
if isinstance(self.config.executable, str):
executable = shlex.split(self.config.executable)
#self.executable.extend (arguments)
topmex = AttrDict(self.config)
topmex.update(
dict(
named_args={},
executable=list(executable),
# arguments = [],
mex_url=mex_tree is not None and mex_tree.get('uri') or None,
bisque_token=bisque_token,
staging_path=None,
rundir=self.rundir))
topmex.mex_id = topmex.mex_url.rsplit('/',
1)[1] if topmex.mex_url else ''
self.mexes.append(topmex)
# Scan argument looking for named arguments (these are used by condor_dag to re-initialize mex vars)
for arg in arguments:
tag, sep, val = arg.partition('=')
if sep == '=':
topmex.named_args[tag] = val
if tag in topmex:
topmex[tag] = val
# Pull out arguments from mex
if mex_tree is not None and module_tree is not None:
mexparser = MexParser()
mex_inputs = mexparser.prepare_inputs(module_tree, mex_tree,
bisque_token)
module_options = mexparser.prepare_options(module_tree, mex_tree)
self.outputs = mexparser.prepare_outputs(module_tree, mex_tree)
argument_style = module_options.get('argument_style', 'positional')
# see if we have pre/postrun option
self.prerun = module_options.get('prerun_entrypoint', None)
self.postrun = module_options.get('postrun_entrypoint', None)
topmex.named_args.update(mexparser.prepare_mex_params(mex_inputs))
topmex.executable.extend(
mexparser.prepare_mex_params(mex_inputs, argument_style))
topmex.rundir = self.rundir
#topmex.options = module_options
# remember topmex executable for pre/post runs
# Create a nested list of arguments (in case of submex)
submexes = mex_tree.xpath('/mex/mex')
for mex in submexes:
sub_inputs = mexparser.prepare_inputs(module_tree, mex)
submex = AttrDict(self.config)
submex.update(
dict(
named_args=dict(topmex.named_args),
# arguments =list(topmex.arguments),
executable=list(executable), #+ topmex.arguments,
mex_url=mex.get('uri'),
mex_id=mex.get('uri').rsplit('/', 1)[1],
bisque_token=bisque_token,
staging_path=None,
rundir=self.rundir))
#if argument_style == 'named':
# submex.named_args.update ( [x.split('=') for x in sub_inputs] )
submex.named_args.update(
mexparser.prepare_mex_params(sub_inputs))
submex.executable.extend(
mexparser.prepare_mex_params(sub_inputs, argument_style))
self.mexes.append(submex)
# Submex's imply that we are iterated.
# We can set up some options here and remove any execution
# for the top mex.
topmex.iterables = len(
self.mexes) > 1 and mexparser.process_iterables(
module_tree, mex_tree)
self.info("processing %d mexes -> %s" % (len(self.mexes), self.mexes))
def store_runstate(self):
staging_path = self.mexes[0].get('staging_path') or '.'
state_file = "state%s.bq" % self.mexes[0].get('mex_id', '')
# pickle the object variables
with open(os.path.join(staging_path, state_file), 'wb') as f:
pickle.dump(self.mexes, f)
pickle.dump(self.rundir, f)
pickle.dump([
et.tostring(tree)
for tree in self.outputs if tree and tree.tag == 'tag'
], f)
pickle.dump(self.entrypoint_executable, f)
pickle.dump(self.prerun, f)
pickle.dump(self.postrun, f)
pickle.dump(self.process_environment, f)
pickle.dump(self.options, f)
def load_runstate(self):
staging_path = self.mexes[0].get('staging_path') or '.'
state_file = "state%s.bq" % self.mexes[0].get('mex_id', '')
# entered in a later processing phase (e.g., Condor finish) => unpickle
with open(os.path.join(staging_path, state_file), 'rb') as f:
#self.pool = None # not preserved for now
self.mexes = pickle.load(f)
self.rundir = pickle.load(f)
self.outputs = [et.fromstring(tree) for tree in pickle.load(f)]
self.entrypoint_executable = pickle.load(f)
self.prerun = pickle.load(f)
self.postrun = pickle.load(f)
self.process_environment = pickle.load(f)
self.options = pickle.load(f)
##################################################
# Command sections
# A command is launched with last argument on the command line of the launcher
# Each command must return the either the next fommand to run or None to stop
# Derived classes should overload these functions
def command_start(self, **kw):
self.info("starting %d mexes -> %s", len(self.mexes), self.mexes)
if self.session is None:
self.session = BQSession().init_mex(self.mexes[0].mex_url,
self.mexes[0].bisque_token)
status = "starting"
self.entrypoint_executable = self.mexes[0].executable
if self.mexes[0].iterables:
self.mexes[0].executable = None
status = 'running parallel'
# add empty "outputs" section in topmex
#self.session.update_mex(status=status, tags=[{'name':'outputs'}])
self.session.update_mex(
status=status
) # dima: modules add outputs section and the empty one complicates module UI
# if there is a prerun, run it now
if self.prerun:
self.info("prerun starting")
self.command_single_entrypoint(self.prerun, self.command_execute,
**kw)
return None
return self.command_execute
def command_execute(self, **kw):
"""Execute the internal executable"""
return self.command_finish
def command_finish(self, **kw):
"""Cleanup the environment and perform any needed actions
after the module completion
"""
self.info("finishing %d mexes -> %s", len(self.mexes), self.mexes)
# if there is a postrun (aka "reduce phase"), run it now
if self.postrun:
self.info("postrun starting")
self.command_single_entrypoint(self.postrun, self.command_finish2,
**kw)
return None
self.command_finish2(**kw)
def command_finish2(self, **kw):
self.teardown_environments()
if self.mexes[0].iterables:
if self.session is None:
self.session = BQSession().init_mex(self.mexes[0].mex_url,
self.mexes[0].bisque_token)
# outputs
# mex_url
# dataset_url (if present and no list/range iterable)
# multiparam (if at least one list/range iterable)
need_multiparam = False
for iter_name, iter_val, iter_type in self.mexes[0].iterables:
if iter_type in ['list', 'range']:
need_multiparam = True
break
outtag = et.Element('tag', name='outputs')
if need_multiparam:
# some list/range params => add single multiparam element if allowed by module def
multiparam_name = None
for output in self.outputs:
if output.get('type', '') == 'multiparam':
multiparam_name = output.get('name')
break
if multiparam_name:
# get inputs section for some submex to read actual types later
# TODO: this can be simplified with xpath_query in 0.6
inputs_subtree = None
if len(self.mexes) > 1:
submextree = self.session.fetchxml(
self.mexes[1].mex_url,
view='full') # get latest MEX doc
inputs_subtree = submextree.xpath(
'./tag[@name="inputs"]')
inputs_subtree = inputs_subtree and self.session.fetchxml(
inputs_subtree[0].get('uri'), view='deep')
multitag = et.SubElement(outtag,
'tag',
name=multiparam_name,
type='multiparam')
colnames = et.SubElement(multitag, 'tag', name='title')
coltypes = et.SubElement(multitag, 'tag', name='xmap')
colxpaths = et.SubElement(multitag, 'tag', name='xpath')
et.SubElement(multitag,
'tag',
name='xreduce',
value='vector')
for iter_name, iter_val, iter_type in self.mexes[
0].iterables:
actual_type = inputs_subtree and inputs_subtree.xpath(
'.//tag[@name="%s" and @type]/@type'
) # read actual types from any submex
actual_type = actual_type[
0] if actual_type else 'string'
et.SubElement(colnames, 'value', value=iter_name)
et.SubElement(coltypes,
'value',
value="tag-value-%s" % actual_type)
et.SubElement(colxpaths,
'value',
value='./mex//tag[@name="%s"]' %
iter_name)
# last column is the submex URI
et.SubElement(colnames, 'value', value="submex_uri")
et.SubElement(coltypes, 'value', value="resource-uri")
et.SubElement(colxpaths, 'value', value='./mex')
else:
self.log.warn(
"List or range parameters in Mex but no multiparam output tag in Module"
)
else:
# no list/range params => add iterables as always
for iter_name, iter_val, iter_type in self.mexes[0].iterables:
et.SubElement(outtag,
'tag',
name=iter_name,
value=iter_val,
type=iter_type)
et.SubElement(outtag,
'tag',
name='mex_url',
value=self.mexes[0].mex_url,
type='mex')
self.session.finish_mex(tags=[outtag])
return None
def command_single_entrypoint(self, entrypoint, callback, **kw):
return None
def command_kill(self, **kw):
"""Kill the running module if possible
"""
return False
def command_status(self, **kw):
return None
def check(self, module_tree=None, **kw):
"check whether the module seems to be runnable"
self.read_config(**kw)
# check for a disabled module
enabled = self.config.get('module_enabled', 'true').lower() == "true"
if not enabled:
self.info('Module is disabled')
return False
# Add remaining arguments to the executable line
# Ensure the loaded executable is a list
if isinstance(self.config.executable, str):
executable = shlex.split(self.config.executable)
if os.name == 'nt':
return True
canrun = executable and which(executable[0]) is not None
if not canrun:
self.error("Executable cannot be run %s" % executable)
return canrun
def main(self, **kw):
# Find and read a config file for the module
created_pool = False
try:
self.pool = kw.pop('pool', None)
if self.pool is None:
self.pool = ProcessManager(POOL_SIZE)
created_pool = True
self.read_config(**kw)
args = kw.pop('arguments', None)
# Pull out command line arguments
self.options, arguments = self.parser.parse_args(args)
command_str = arguments.pop()
# the following always has to run first so that e.g., staging dirs are set up properly
self.init_runstate(arguments, **kw)
self.create_environments(**kw)
self.process_config(**kw)
if command_str == 'start':
# new run => setup environments from scratch
self.setup_environments()
else:
# continued run => load saved run state
try:
self.load_runstate()
except OSError:
# could not load state => OK if it was command that can run without previous 'start' (e.g., 'kill')
if command_str not in ['kill']:
raise
self.mexes[0].arguments = arguments
command = getattr(self, 'command_%s' % command_str, None)
while command:
self.info("COMMAND_RUNNER %s" % command)
command = command(**kw)
if command_str not in ['kill']:
# store run state since we may come back (e.g., condor finish)
self.store_runstate()
if created_pool:
self.pool.stop()
return 0
except ModuleEnvironmentError, e:
self.log.exception("Problem occured in module")
raise RunnerException(str(e), self.mexes)
except RunnerException, e:
raise
