def check_pool_status(self, p, status):
# Check that all have finished or failed
with self.pool.pool_lock:
p['status'] = status
all_status = [p['status'] for p in self.processes]
for status in all_status:
if status not in ('finished', 'failed'):
return
self.info("All processes have returned %s", all_status)
# all are done.. so check if we finished correctly
if 'failed' not in all_status:
self.mexes[0].status = 'finished'
self.command_finish(**self.execute_kw)
return
# there was a failue:
msg = '\n'.join(
p.get('fail_message') for p in self.processes
if p['status'] == 'fail')
self.mexes[0].status = 'failed'
self.error(msg)
if self.session is None:
self.session = BQSession().init_mex(self.mexes[0].mex_url,
self.mexes[0].bisque_token)
if self.session.mex.value not in ('FAILED', 'FINISHED'):
self.session.fail_mex(msg)
def setbasicauth(bisquehost, username, password):
s = BQSession()
s.init_cas(username, password, bisque_root=bisquehost, create_mex=False)
r = s.c.post(bisquehost + "/auth_service/setbasicauth",
data={
'username': username,
'passwd': password
})
print r.text
def command_kill(self, **kw):
"""Kill the running module if possible
"""
self.info("Kill On %s", platform.node())
mex = kw.get('mex_tree')
topmex = self.mexes[0]
if mex is not None:
mex_id = mex.get('resource_uniq')
# get all condor schedds
schedd_names = []
cmd = ['condor_status', '-long', '-schedd']
pk = subprocess.Popen(cmd, stdout=subprocess.PIPE)
for line in pk.stdout:
toks = line.split('=', 1)
if len(toks) == 2:
if toks[0].strip().lower() == 'name':
schedd_names.append(
toks[1].strip().strip('"').strip("'"))
self.debug("schedds found: %s" % schedd_names)
pk.communicate()
if pk.returncode != 0:
self.debug("condor_status failed")
process = dict(command_line=cmd, mex=topmex)
self.command_failed(process, pk.returncode)
return None
# for each one: condor_rm with condition "mexid == <mexid>"
for schedd_name in schedd_names:
cmd = [
'condor_rm', '-name', schedd_name, '-constraint',
'MexID =?= "%s"' % mex_id
]
self.debug("running %s", cmd)
pk = subprocess.Popen(cmd,
cwd=topmex.get('staging_path'),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
message, err = pk.communicate()
self.info("condor_rm %s status = %s message = %s err = %s" %
(schedd_name, pk.returncode, message, err))
if pk.returncode != 0:
self.debug("condor_rm failed")
process = dict(command_line=cmd, mex=topmex)
self.command_failed(process, pk.returncode)
return None
if self.session is None:
mex_url = topmex.named_args['mex_url']
token = topmex.named_args['bisque_token']
self.session = BQSession().init_mex(mex_url, token)
self.session.fail_mex(msg='job stopped by user')
else:
self.debug("No mex provided")
return None
def run(self):
logging.basicConfig(level=logging.DEBUG)
parser = optparse.OptionParser()
parser.add_option('-d','--debug', action="store_true")
parser.add_option('-n','--dryrun', action="store_true")
#parser.add_option('--resource_url')
#parser.add_option('--mex_url')
#parser.add_option('--staging_path')
#parser.add_option('--bisque_token')
#parser.add_option('--credentials')
# Parse named arguments from list
(options, args) = parser.parse_args()
named_args =dict( [ y for y in [ x.split ('=') for x in args ] if len (y) == 2] )
args = [ x for x in args if '=' not in x ]
staging_path = '.'
self.auth_token = named_args.get ('bisque_token')
self.image_map_name = os.path.join(staging_path, IMAGE_MAP)
self.resource_url = named_args.get ('image_url')
self.mex_url = named_args.get ('mex_url')
self.images = os.path.join(staging_path, 'images') + os.sep
if self.auth_token:
self.bq = BQSession().init_mex(self.mex_url, self.auth_token)
else:
user,pwd = options.credentials.split(':')
self.bq = BQSession().init_local(user,pwd)
resource_xml = self.bq.fetchxml (self.resource_url, view='short')
self.is_dataset = resource_xml.tag == 'dataset'
if len(args) == 1:
commands = [ args.pop(0)]
else:
commands =['setup','start', 'teardown']
#if command not in ('setup','teardown', 'start'):
# parser.error('Command must be start, setup or teardown')
# maltab code requires trailing slash..
try:
for command in commands:
command = getattr(self, command)
r = command()
except Exception, e:
logging.exception ("problem during %s" % command)
self.bq.fail_mex(msg = "Exception during %s: %s" % (command, str(e)))
sys.exit(1)
def main(self, mex_url=None, bisque_token=None, image_url=None, bq=None):
# Allow for testing by passing an alreay initialized session
if bq is None:
bq = BQSession().init_mex(mex_url, bisque_token)
bq.update_mex('Classifying...')
pars = bq.parameters()
image_url = pars.get('data_url', None)
model_url = pars.get('model_url', None)
store_on_image = pars.get('store_on_image', False)
method = pars.get('method', 'points')
points = int(pars.get('number_of_points', 10))
confidence = int(pars.get('confidence', 95))
border = int(pars.get('border', 0))
_, image_uniq = image_url.rsplit('/', 1)
_, model_uniq = model_url.rsplit('/', 1)
url = '%s/connoisseur/%s/classify:%s/method:%s/points:%s/confidence:%s/border:%s' % (
bq.bisque_root, model_uniq, image_uniq, method, points, confidence,
border)
# dima: the moving of the image is not of the best taste, this should be changed later
if method != 'segmentation':
url += '/format:xml'
txt = bq.c.fetch(url,
headers={
'Content-Type': 'text/xml',
'Accept': 'text/xml'
})
gobs = etree.fromstring(txt)
else:
color_mode = 'colors'
url += '/colors:colors'
filename = '%s_%s_conf%.2f_n%s_b%s.png' % (
image_uniq, color_mode, confidence, points, border)
filename = bq.c.fetch(url, path=filename)
image_url = upload_file(bq, filename) or ''
gobs = None
bq.update_mex('Storing results...')
outputs = etree.Element('tag', name="outputs")
img = etree.SubElement(outputs,
'tag',
name="MyImage",
type="image",
value=image_url)
if gobs is not None:
img.append(gobs)
if store_on_image is True:
r = bq.postxml(image_url, xml=txt)
bq.finish_mex(tags=[outputs])
def setUpClass(self):
config = ConfigParser.ConfigParser()
config.read('config.cfg')
self.root = config.get('Host', 'root') or 'localhost:8080'
self.user = config.get('Host', 'user') or 'test'
self.pswd = config.get('Host', 'password') or 'test'
self.session = BQSession().init_local(self.user,
self.pswd,
bisque_root=self.root,
create_mex=False)
def test_open_session(config):
"""
Test Initalizing a BQSession locally
"""
host = config.get('host.root')
user = config.get('host.user')
pwd = config.get('host.password')
bqsession = BQSession().init_local(user,
pwd,
bisque_root=host,
create_mex=False)
bqsession.close()
def test_initalize_mex_locally(config):
"""
Test initalizing a mex locally
"""
host = config.get('host.root')
user = config.get('host.user')
pwd = config.get('host.password')
bqsession = BQSession().init_local(user,
pwd,
bisque_root=host,
create_mex=True)
assert bqsession.mex.uri
bqsession.close()
def command_failed(self, process, retcode):
"""Update the bisque server with a failed command for a mex"""
mex = process['mex']
mex.status = "failed"
command = " ".join(process['command_line'])
msg = "%s: returned (non-zero) %s" % (command, retcode)
self.error("condor_failed: " + msg)
# update process mex
if self.session is None:
self.session = BQSession().init_mex(self.mexes[0].mex_url,
self.mexes[0].bisque_token)
if self.session.mex.value not in ('FAILED', 'FINISHED'):
self.session.fail_mex(msg)
def test_module():
from bqapi import BQSession
session = BQSession().init_local('admin', 'admin', 'http://localhost:8080')
admin = session.service('admin')
data = session.service('data_service')
#admin.user(uniq).login().fetch ()
xml = data.get("user",
params={
'wpublic': '1',
'resource_name': 'admin'
},
render='xml')
user_uniq = xml.find("user").get('resource_uniq')
admin.fetch('/user/{}/login'.format(user_uniq))
def setUpClass(self):
config = ConfigParser.ConfigParser()
config.read('config.cfg')
self.root = config.get('Host', 'root') or 'localhost:8080'
self.user = config.get('Host', 'user') or 'test'
self.pswd = config.get('Host', 'password') or 'test'
self.session = BQSession().init_local(self.user,
self.pswd,
bisque_root=self.root,
create_mex=False)
# download and upload test images ang get their IDs
self.resource_unicode_jpeg = self.ensure_bisque_file(
image_unicode_jpeg.decode('utf-8'))
self.resource_unicode_mov = self.ensure_bisque_file(
image_unicode_mov.decode('utf-8'))
self.resource_unicode_oib = self.ensure_bisque_file(
image_unicode_oib.decode('utf-8'))
self.resource_unicode_tiff = self.ensure_bisque_file(
image_unicode_tiff.decode('utf-8'))
self.resource_unicode_ims = self.ensure_bisque_file(
image_unicode_ims.decode('utf-8'))
self.resource_unicode_dicom = self.ensure_bisque_file(
image_unicode_dicom.decode('utf-8'))
self.resource_unicode_svs = self.ensure_bisque_file(
image_unicode_svs.decode('utf-8'))
self.resource_latin1_tiff = self.ensure_bisque_file(
image_latin1_tiff.decode('utf-8'))
def setUpClass(self):
config = ConfigParser.ConfigParser()
config.read('config.cfg')
self.root = config.get('Host', 'root') or 'localhost:8080'
self.user = config.get('Host', 'user') or 'test'
self.pswd = config.get('Host', 'password') or 'test'
self.session = BQSession().init_local(self.user,
self.pswd,
bisque_root=self.root,
create_mex=False)
print '\nUploading images\n'
self.resources_plane = []
self.resources_plane.append(self.ensure_bisque_file(image_planar))
self.resources_plane.append(self.ensure_bisque_file(image_planar))
self.resources_plane.append(self.ensure_bisque_file(image_planar))
self.resources_plane.append(self.ensure_bisque_file(image_planar))
self.resources_plane.append(self.ensure_bisque_file(image_planar))
self.resources_pyr = []
self.resources_pyr.append(self.ensure_bisque_file(image_pyramid))
self.resources_pyr.append(self.ensure_bisque_file(image_pyramid))
self.resources_pyr.append(self.ensure_bisque_file(image_pyramid))
self.resources_pyr.append(self.ensure_bisque_file(image_pyramid))
self.resources_pyr.append(self.ensure_bisque_file(image_pyramid))
self.resources_valid_plane = [self.ensure_bisque_file(image_planar)]
self.resources_valid_pyr = [self.ensure_bisque_file(image_pyramid)]
self.coordinates = [(0, 0), (1, 3), (2, 3), (7, 1), (2, 2), (5, 2),
(7, 2), (1, 1), (6, 3), (7, 3)]
def setUp():
global results_location
global store_local_location
global file1_location
global filename1
global bqsession
global FeatureResource
config = ConfigParser.ConfigParser()
config.read('setup.cfg')
root = config.get('Host', 'root') or 'localhost:8080'
user = config.get('Host', 'user') or 'test'
pwd = config.get('Host', 'password') or 'test'
results_location = config.get('Store', 'results_location') or 'Results'
_mkdir(results_location)
store_location = config.get('Store', 'location') or None
if store_location is None: raise NameError('Requre a store location to run test properly')
store_local_location = config.get('Store', 'local_location') or 'SampleData'
filename1 = config.get('Store','filename1') or None
if filename1 is None: raise NameError('Requre an image to run test properly')
file1_location = fetch_file(filename1, store_location, store_local_location)
FeatureResource = namedtuple('FeatureResource',['image','mask','gobject'])
FeatureResource.__new__.__defaults__ = (None, None, None)
#start session
bqsession = BQSession().init_local(user, pwd, bisque_root=root, create_mex=False)
def run(self):
parser = optparse.OptionParser()
parser.add_option('-d', '--debug', action="store_true")
parser.add_option('-n', '--dryrun', action="store_true")
parser.add_option('--credentials')
parser.add_option('--image_url')
(options, args) = parser.parse_args()
named = AttrDict(bisque_token=None,
mex_url=None,
staging_path=None,
image_url=None)
for arg in list(args):
tag, sep, val = arg.partition('=')
if sep == '=':
named[tag] = val
args.remove(arg)
self.named_args = named
self.staging_path = named.get('staging_path')
if named.bisque_token:
self.bq = BQSession().init_mex(named.mex_url, named.bisque_token)
self.resource_url = self.bq.parameter_value('image_url')
elif options.credentials:
user, pwd = options.credentials.split(':')
self.bq = BQSession().init_local(user, pwd)
self.resource_url = named.image_url
else:
parser.error('need bisque_token or user credential')
if self.resource_url is None:
parser.error('Need a resource_url')
if not args:
commands = ['setup', 'start', 'teardown']
else:
commands = [args]
try:
for command in commands:
command = getattr(self, command)
r = command()
except Exception, e:
logging.exception("problem during %s" % command)
self.bq.fail_mex(msg="Exception during %s: %s" % (command, e))
sys.exit(1)
def command_kill(self, **kw):
"""Kill the running module if possible
"""
mex = kw.get('mex_tree')
topmex = self.mexes[0]
if mex is not None:
mex_id = mex.get('resource_uniq')
self.pool.kill(selector_fct=lambda task: '00-' + task['mex'].get(
'mex_url').split('/00-', 1)[1].split('/', 1)[0] == mex_id)
if self.session is None:
mex_url = topmex.named_args['mex_url']
token = topmex.named_args['bisque_token']
self.session = BQSession().init_mex(mex_url, token)
self.session.fail_mex(msg='job stopped by user')
else:
self.debug("No mex provided")
return None
def _authenticate(self, attempts=25):
'''
Attempts to authenticate N times
@attempts(optional): number of attempts to authenticate
'''
for i in range(attempts):
s = BQSession().init_cas(self.username, self.password, bisque_root=self.bisque_root)
try:
if s._check_session():
self._debug_print('Authenticated successfully!')
return s
except:
self._debug_print('Unable to authenticate... trying again')
pass
time.sleep(30)
self._debug_print('Unable to authenticate.', 'error')
return None
def _authenticate(self, attempts=25):
"""
Attempts to authenticate N times
@attempts(optional): number of attempts to authenticate
"""
for i in range(attempts):
s = BQSession().init_cas(self.username, self.password, bisque_root=self.bisque_root)
try:
if s._check_session():
self._debug_print("Authenticated successfully!")
return s
except:
self._debug_print("Unable to authenticate... trying again")
pass
time.sleep(30)
self._debug_print("Unable to authenticate.", "error")
return None
def command_finish(self, **kw):
# Cleanup condor stuff and look for error files.
topmex = self.mexes[0]
job_return = int(topmex.named_args.get('condor_job_return', 0))
#job_return = int(topmex.arguments.pop())
self.info("condor_finish %s: return=%s", topmex.executable, job_return)
if job_return != 0:
if self.session is None:
mex_url = topmex.named_args['mex_url']
token = topmex.named_args['bisque_token']
self.session = BQSession().init_mex(mex_url, token)
# Possible look for log files and append to message here
#if os.path.exists(''):
# pass
self.session.fail_mex(msg='job failed with return code %s' %
job_return)
return None
topmex.status = "finished"
return super(CondorRunner, self).command_finish(**kw)
def test_initalize_session_From_mex(config):
"""
Test initalizing a session from a mex
"""
host = config.get('host.root')
user = config.get('host.user')
pwd = config.get('host.password')
bqsession = BQSession().init_local(user, pwd, bisque_root=host)
mex_url = bqsession.mex.uri
token = bqsession.mex.resource_uniq
bqmex = BQSession().init_mex(mex_url, token, user, bisque_root=host)
bqmex.close()
bqsession.close()
def main(self, image_url, mex_url=None, bisque_token=None, bq=None):
# Allow for testing by passing an alreay initialized session
if bq is None:
bq = BQSession().init_mex(mex_url, bisque_token)
# Fetch the image metadata
image = bq.load(image_url)
# Fetch embedded tags from image service
meta = image.pixels().meta().fetch()
meta = ET.XML(meta)
tags = []
# Create a new tag 'MetaData' to be placed on the image
md = BQTag(name='MetaData')
# Filter the embedded metadata and place subtags in MetaData
for t in meta.getiterator('tag'):
if t.get('name') in wanted_tags:
md.addTag(name=t.get('name'), value=t.get('value'))
# Add the new tag to the image
image.addTag(tag=md)
metadata_tag = bq.save(md, image.uri + "/tag")
bq.finish_mex(tags=[{
'name':
'outputs',
'tag': [{
'name': 'metadata',
'value': metadata_tag.uri,
'type': 'tag'
}]
}])
sys.exit(0)
def mexsession(config):
"Create a BQApi BQSession object based on config"
host = config.get ( 'host.root')
user = config.get ( 'host.user')
passwd = config.get ( 'host.password')
bq = BQSession()
bq.config = config
bq.init_local (user, passwd, bisque_root = host, create_mex = True)
yield bq
bq.close()
def command_start(self, **kw):
self.info("starting %d mexes -> %s", len(self.mexes), self.mexes)
if self.session is None:
self.session = BQSession().init_mex(self.mexes[0].mex_url,
self.mexes[0].bisque_token)
status = "starting"
self.entrypoint_executable = self.mexes[0].executable
if self.mexes[0].iterables:
self.mexes[0].executable = None
status = 'running parallel'
# add empty "outputs" section in topmex
#self.session.update_mex(status=status, tags=[{'name':'outputs'}])
self.session.update_mex(
status=status
) # dima: modules add outputs section and the empty one complicates module UI
# if there is a prerun, run it now
if self.prerun:
self.info("prerun starting")
self.command_single_entrypoint(self.prerun, self.command_execute,
**kw)
return None
return self.command_execute
def setUpClass(self):
config = ConfigParser.ConfigParser()
config.read('config.cfg')
self.root = config.get('Host', 'root') or 'localhost:8080'
self.user = config.get('Host', 'user') or 'test'
self.pswd = config.get('Host', 'password') or 'test'
self.session = BQSession().init_local(self.user, self.pswd, bisque_root=self.root, create_mex=False)
# download and upload test images ang get their IDs
self.resource_2d_uint8 = self.ensure_bisque_file(image_rgb_uint8)
self.resource_3d_uint16 = self.ensure_bisque_file(image_zstack_uint16)
self.resource_2d_float = self.ensure_bisque_file(image_float)
def setup_simple_feature_test(ns):
"""
Setup feature requests test
"""
config = ConfigParser.ConfigParser()
config.read(CONFIG_FILE)
root = config.get('Host', 'root') or DEFAULT_ROOT
user = config.get('Host', 'user') or DEFAULT_USER
pwd = config.get('Host', 'password') or DEFAULT_PASSWORD
results_location = config.get('Store', 'results_dir') or DEFAULT_RESULTS_DIR
store_location = config.get('Store', 'location') or None
store_local_location = config.get('Store', 'local_dir') or DEFAULT_LOCAL_DIR
temp_store = config.get('Store', 'temp_dir') or DEFAULT_TEMPORARY_DIR
test_image = config.get('SimpleTest', 'test_image') or None
feature_response_results = config.get('SimpleTest', 'feature_response') or DEFAULT_FEATURE_RESPONSE_HDF5
feature_past_response_results = config.get('SimpleTest','feature_sample') or DEFAULT_FEATURE_SAMPLE_HDF5
if store_location is None: raise NameError('Requre a store location to run test properly')
if test_image is None: raise NameError('Requre an image to run test properly')
_mkdir(store_local_location)
_mkdir(results_location)
_mkdir(temp_store)
results_table_path = os.path.join(results_location, feature_response_results)
if os.path.exists(results_table_path):
os.remove(results_table_path)
test_image_location = fetch_file(test_image, store_location, store_local_location)
#initalize session
session = BQSession().init_local(user, pwd, bisque_root=root)
#set to namespace
ns.root = root
ns.store_location = store_location
ns.session = session
ns.results_location = results_location
ns.store_local_location = store_local_location
ns.test_image_location = test_image_location
ns.feature_response_results = feature_response_results
ns.feature_past_response_results = feature_past_response_results
ns.test_image = test_image
ns.temp_store = temp_store
def setUpClass(self):
config = ConfigParser.ConfigParser()
config.read('config.cfg')
self.root = config.get('Host', 'root') or 'localhost:8080'
self.user = config.get('Host', 'user') or 'test'
self.pswd = config.get('Host', 'password') or 'test'
self.session = BQSession().init_local(self.user,
self.pswd,
bisque_root=self.root,
create_mex=False)
# download and upload test images ang get their IDs
self.resource_nikon_nd2 = self.ensure_bisque_file(image_nikon_nd2)
self.resource_nikon_nd2_deconv = self.ensure_bisque_file(
image_nikon_nd2_deconv)
self.resource_svs = self.ensure_bisque_file(image_svs)
def setUpClass(self):
config = ConfigParser.ConfigParser()
config.read('config.cfg')
self.root = config.get('Host', 'root') or 'localhost:8080'
self.user = config.get('Host', 'user') or 'test'
self.pswd = config.get('Host', 'password') or 'test'
self.session = BQSession().init_local(self.user,
self.pswd,
bisque_root=self.root,
create_mex=False)
# download and upload test images ang get their IDs
self.resource_imaris_hela = self.ensure_bisque_file(image_imaris_hela)
self.resource_imaris_r18 = self.ensure_bisque_file(image_imaris_r18)
self.resource_zeiss_czi_rat = self.ensure_bisque_file(
image_zeiss_czi_rat)
self.resource_dicom_3d = self.ensure_bisque_file(image_dicom_3d)
self.resource_dicom_2d = self.ensure_bisque_file(image_dicom_2d)
def main(self, image_url, mex_url = None, bisque_token=None, bq = None, args = None):
# Allow for testing by passing an alreay initialized session
if bq is None:
bq = BQSession().init_mex(mex_url, bisque_token)
# Fetch the blob links
if not os.path.exists ('videos'):
os.makedirs ('videos')
video = fetch_blob(bq, image_url, 'videos')
print "VIDEO file ", video
#pass arguments to MotionMeerkat scripts, located in MotionMeerkat/main.py
#Structure arguments
#Format call string
callargs = ["python MotionMeerkat/main.py",
"--i", video.values()[0],
"--threshT", args[0],
"--sub", args[1],
"--mogh", args[2],
"--mogv", args[3],
"--accA", args[4],
"--burn", args[5],
"--frameSET",
"--frame_rate", "1",
"--makeV", "none",
"--fileD", "Output"]
print "Calling ", " ".join(callargs)
#run MotionMeerkat
r = call(" ".join(callargs), shell=True)
if r != 0:
bq.fail_mex ("Meerkat returned non-zero")
#Post Results
#get filename to get the destination of the csv to be posted
fn=video.values()[0]
fbs= os.path.basename(fn)
head, tail = os.path.splitext(fbs)
#post file
frames_blob = bq.postblob(str("Output/" + head + "/Frames.csv"))
#get file location from regex
uri=re.search("uri=\"(.*?)\"", frames_blob).group(1)
tags = [{ 'name': 'outputs','tag' : [{'name': 'frames_csv', 'type':'file', 'value':uri}]}]
bq.finish_mex(tags = tags)
sys.exit(0)
def main(self, mex_url=None, auth_token=None, bq=None, **kw):
# Allow for testing by passing an alreay initialized session
if bq is None:
bq = BQSession().init_mex(mex_url, auth_token)
# check for list parameters
params = bq.get_mex_inputs()
if isinstance(params,
dict) or not isinstance(params, collections.Iterable):
params = [params]
# pass values directly as args
for single_params in params:
for param_name in single_params:
if 'value' in single_params[param_name]:
single_params[param_name] = single_params[param_name].get(
'value')
# TODO: measure block startup time
self.start_block(bq, params)
for kw in params:
# TODO: measure single item time
# TODO: run in parallel
if 'mex_url' in kw:
# set (innermost) single item mex
sub_bq = BQSession().init_mex(kw['mex_url'], auth_token)
else:
sub_bq = bq
self.process_single(sub_bq, **kw)
if 'mex_url' in kw:
sub_bq.close()
# TODO: measure block teardown time
self.end_block(bq)
sys.exit(0)
H = 2048
##################################################################
# create session
##################################################################
#config = ConfigParser.ConfigParser()
#config.read(config_file)
#root = config.get('Host', 'root') or 'http://bisque.ece.ucsb.edu'
#user = config.get('Host', 'user') or 'Mex'
#pswd = config.get('Host', 'password') or 'Mary:00-rvFkDRQyoheksMZnhBhi2n'
#session = BQSession().init_local(user, pswd, bisque_root=root, create_mex=False)
mex_url = 'http://bisque.ece.ucsb.edu/module_service/mex/00-rvFkDRQyoheksMZnhBhi2n'
bisque_token = 'Mary:00-rvFkDRQyoheksMZnhBhi2n'
session = BQSession().init_mex(mex_url, bisque_token)
##################################################################
# index excel file
##################################################################
# filename-row hash
meta = {}
data = pd.read_excel(fn)
Ks = data.keys()
N = len(data['ImageName'])
for i in range(N):
fn = data['ImageName'][i].strip(' ')
m = {}
outputs = etree.Element('tag', name='outputs')
summary = etree.SubElement(outputs, 'tag', name='summary')
for r,v in resources.iteritems():
etree.SubElement(summary, 'tag', name=r, value=v)
bq.finish_mex(tags=[outputs])
if __name__ == "__main__":
import optparse
parser = optparse.OptionParser()
parser.add_option("-c", "--credentials", dest="credentials",
help="credentials are in the form user:password")
(options, args) = parser.parse_args()
M = AnnotationHistograms()
if options.credentials is None:
mex_url, auth_token = args[:2]
bq = BQSession().init_mex(mex_url, auth_token)
else:
mex_url = ''
if not options.credentials:
parser.error('need credentials')
user,pwd = options.credentials.split(':')
bq = BQSession().init_local(user, pwd)
try:
M.main(mex_url=mex_url, bq=bq )
except Exception, e:
bq.fail_mex(traceback.format_exc())
sys.exit(0)
from bqapi import BQSession, BQCommError
from bqapi.util import save_blob, localpath2url
##################################################################
# Upload
##################################################################
config = ConfigParser.ConfigParser()
config.read('config.cfg')
root = config.get('Host', 'root') or 'localhost:8080'
user = config.get('Host', 'user') or 'test'
pswd = config.get('Host', 'password') or 'test'
session = BQSession().init_local(user, pswd, bisque_root=root, create_mex=False)
request = '<image name="%s">'%'series_t_jpeg'
request = '%s<value>%s</value>'%(request, localpath2url('f:/dima/develop/python/bq5irods/data/imagedir/admin/tests/multi_file/series_t_jpeg/t_000241.jpg'))
request = '%s<value>%s</value>'%(request, localpath2url('f:/dima/develop/python/bq5irods/data/imagedir/admin/tests/multi_file/series_t_jpeg'))
request = '%s<tag name="%s" type="%s">'%(request, 'image_meta', 'image_meta')
request = '%s<tag name="%s" value="%s" type="number" />'%(request, 'image_num_z', '1')
request = '%s<tag name="%s" value="%s" type="number" />'%(request, 'image_num_t', '80')
request = '%s<tag name="%s" value="%s" />'%(request, 'dimensions', 'XYCZT')
request = '%s<tag name="%s" value="%s" type="number" />'%(request, 'pixel_resolution_x', '0.4')
request = '%s<tag name="%s" value="%s" type="number" />'%(request, 'pixel_resolution_y', '0.4')
request = '%s<tag name="%s" value="%s" type="number" />'%(request, 'pixel_resolution_t', '2.0')
request = '%s<tag name="%s" value="%s" />'%(request, 'pixel_resolution_unit_x', 'micron')
request = '%s<tag name="%s" value="%s" />'%(request, 'pixel_resolution_unit_y', 'micron')
request = '%s<tag name="%s" value="%s" />'%(request, 'pixel_resolution_unit_t', 'seconds')
request = '%s</tag>'%(request)
class ImageServiceTestBase(unittest.TestCase):
"""
Test image service operations
"""
@classmethod
def setUpClass(self):
config = ConfigParser.ConfigParser()
config.read('config.cfg')
self.root = config.get('Host', 'root') or 'localhost:8080'
self.user = config.get('Host', 'user') or 'test'
self.pswd = config.get('Host', 'password') or 'test'
self.session = BQSession().init_local(self.user,
self.pswd,
bisque_root=self.root,
create_mex=False)
# download and upload test images ang get their IDs
#self.uniq_2d_uint8 = self.ensure_bisque_file(image_rgb_uint8)
#self.uniq_3d_uint16 = self.ensure_bisque_file(image_zstack_uint16)
@classmethod
def tearDownClass(self):
#self.delete_resource(self.uniq_2d_uint8)
#self.delete_resource(self.uniq_3d_uint16)
self.cleanup_tests_dir()
pass
@classmethod
def fetch_file(self, filename):
_mkdir(local_store_images)
_mkdir(local_store_tests)
url = posixpath.join(url_image_store, filename).encode('utf-8')
path = os.path.join(local_store_images, filename)
if not os.path.exists(path):
urllib.urlretrieve(url, path)
return path
@classmethod
def upload_file(self, path, resource=None):
#if resource is not None:
# print etree.tostring(resource)
r = save_blob(self.session, path, resource=resource)
if r is None or r.get('uri') is None:
print 'Error uploading: %s' % path.encode('ascii', 'replace')
return None
print 'Uploaded id: %s url: %s' % (r.get('resource_uniq'),
r.get('uri'))
return r
@classmethod
def delete_resource(self, r):
if r is None:
return
url = r.get('uri')
print 'Deleting id: %s url: %s' % (r.get('resource_uniq'), url)
self.session.deletexml(url)
@classmethod
def delete_package(self, package):
if 'dataset' in package:
# delete dataset
url = package['dataset']
print 'Deleting dataset: %s' % (url)
try:
self.session.fetchxml('/dataset_service/delete?duri=%s' % url)
except BQCommError:
print 'Error deleting the dataset'
elif 'items' in package:
# delete all items
for url in package['items']:
print 'Deleting item: %s' % (url)
try:
self.session.deletexml(url)
except BQCommError:
print 'Error deleting the item'
# # delete dataset
# if 'dataset' in package:
# url = package['dataset']
# print 'Deleting dataset: %s'%(url)
# try:
# self.session.deletexml(url)
# except BQCommError:
# print 'Error deleting the dataset'
# # delete all items
# if 'items' in package:
# for url in package['items']:
# print 'Deleting item: %s'%(url)
# try:
# self.session.deletexml(url)
# except BQCommError:
# print 'Error deleting the item'
@classmethod
def ensure_bisque_file(self, filename, metafile=None):
path = self.fetch_file(filename)
if metafile is None:
filename = u'%s/%s' % (TEST_PATH, filename)
resource = etree.Element('resource', name=filename)
return self.upload_file(path, resource=resource)
else:
metafile = self.fetch_file(metafile)
return self.upload_file(path,
resource=etree.parse(metafile).getroot())
@classmethod
def ensure_bisque_package(self, package):
path = self.fetch_file(package['file'])
r = self.upload_file(path, resource=etree.XML(package['resource']))
package['resource'] = r
if r is None:
return None
print 'Uploaded id: %s url: %s' % (r.get('resource_uniq'),
r.get('uri'))
#print etree.tostring(r)
if r.tag != 'dataset':
package['items'] = [r.get('uri')]
else:
package['dataset'] = r.get('uri')
values = r.xpath('value')
if len(values) != package['count']:
print 'Error: uploaded %s has %s elements but needs %s' % (
package['file'], len(values), package['count'])
if r.get('name') != package['name']:
print 'Error: uploaded %s name is %s but should be %s' % (
package['file'], r.get('name'), package['name'])
package['items'] = [x.text for x in values]
package['last'] = self.session.fetchxml(package['items'][-1],
view='deep')
#print 'Last item\n'
#print etree.tostring(package['last'])
@classmethod
def cleanup_tests_dir(self):
print 'Cleaning-up %s' % local_store_tests
for root, dirs, files in os.walk(local_store_tests, topdown=False):
for name in files:
os.remove(os.path.join(root, name))
def validate_image_variant(self,
resource,
filename,
commands,
meta_required=None):
path = os.path.join(local_store_tests, filename)
try:
#image = fromXml(resource, session=self.session)
image = self.session.factory.from_etree(resource)
px = image.pixels()
for c, a in commands:
px = px.command(c, a)
px.fetch(path)
except BQCommError:
logging.exception('Comm error')
self.fail('Communication error while fetching image')
if meta_required is not None:
meta_test = metadata_read(path)
self.assertTrue(meta_test is not None,
msg='Retrieved image can not be read')
self.assertTrue(
compare_info(meta_required, meta_test),
msg='Retrieved metadata differs from test template')
def validate_xml(self, resource, filename, commands, xml_parts_required):
path = os.path.join(local_store_tests, filename)
try:
#image = fromXml(resource, session=self.session)
image = self.session.factory.from_etree(resource)
px = image.pixels()
for c, a in commands:
px = px.command(c, a)
px.fetch(path)
except BQCommError:
self.fail()
xml_test = etree.parse(path).getroot()
#print etree.tostring(xml_test)
self.assertTrue(compare_xml(xml_parts_required, xml_test),
msg='Retrieved XML differs from test template')
