Example #1
0
 def check_pool_status(self, p, status):
     # Check that all have finished or failed
     with self.pool.pool_lock:
         p['status'] = status
         all_status = [p['status'] for p in self.processes]
         for status in all_status:
             if status not in ('finished', 'failed'):
                 return
     self.info("All processes have returned %s", all_status)
     # all are done.. so check if we finished correctly
     if 'failed' not in all_status:
         self.mexes[0].status = 'finished'
         self.command_finish(**self.execute_kw)
         return
     # there was a failue:
     msg = '\n'.join(
         p.get('fail_message') for p in self.processes
         if p['status'] == 'fail')
     self.mexes[0].status = 'failed'
     self.error(msg)
     if self.session is None:
         self.session = BQSession().init_mex(self.mexes[0].mex_url,
                                             self.mexes[0].bisque_token)
     if self.session.mex.value not in ('FAILED', 'FINISHED'):
         self.session.fail_mex(msg)
Example #2
0
def setbasicauth(bisquehost, username, password):
    s = BQSession()
    s.init_cas(username, password, bisque_root=bisquehost, create_mex=False)
    r = s.c.post(bisquehost + "/auth_service/setbasicauth",
                 data={
                     'username': username,
                     'passwd': password
                 })
    print r.text
Example #3
0
    def command_kill(self, **kw):
        """Kill the running module if possible
        """
        self.info("Kill On %s", platform.node())
        mex = kw.get('mex_tree')
        topmex = self.mexes[0]
        if mex is not None:
            mex_id = mex.get('resource_uniq')

            # get all condor schedds
            schedd_names = []
            cmd = ['condor_status', '-long', '-schedd']
            pk = subprocess.Popen(cmd, stdout=subprocess.PIPE)
            for line in pk.stdout:
                toks = line.split('=', 1)
                if len(toks) == 2:
                    if toks[0].strip().lower() == 'name':
                        schedd_names.append(
                            toks[1].strip().strip('"').strip("'"))
            self.debug("schedds found: %s" % schedd_names)
            pk.communicate()
            if pk.returncode != 0:
                self.debug("condor_status failed")
                process = dict(command_line=cmd, mex=topmex)
                self.command_failed(process, pk.returncode)
                return None

            # for each one: condor_rm with condition "mexid == <mexid>"
            for schedd_name in schedd_names:
                cmd = [
                    'condor_rm', '-name', schedd_name, '-constraint',
                    'MexID =?= "%s"' % mex_id
                ]
                self.debug("running %s", cmd)
                pk = subprocess.Popen(cmd,
                                      cwd=topmex.get('staging_path'),
                                      stdout=subprocess.PIPE,
                                      stderr=subprocess.PIPE)
                message, err = pk.communicate()
                self.info("condor_rm %s status = %s message = %s err = %s" %
                          (schedd_name, pk.returncode, message, err))
                if pk.returncode != 0:
                    self.debug("condor_rm failed")
                    process = dict(command_line=cmd, mex=topmex)
                    self.command_failed(process, pk.returncode)
                    return None

            if self.session is None:
                mex_url = topmex.named_args['mex_url']
                token = topmex.named_args['bisque_token']
                self.session = BQSession().init_mex(mex_url, token)
            self.session.fail_mex(msg='job stopped by user')
        else:
            self.debug("No mex provided")

        return None
Example #4
0
    def run(self):
        logging.basicConfig(level=logging.DEBUG)

        parser  = optparse.OptionParser()
        parser.add_option('-d','--debug', action="store_true")
        parser.add_option('-n','--dryrun', action="store_true")
        #parser.add_option('--resource_url')
        #parser.add_option('--mex_url')
        #parser.add_option('--staging_path')
        #parser.add_option('--bisque_token')
        #parser.add_option('--credentials')

        # Parse named arguments from list


        (options, args) = parser.parse_args()
        named_args =dict( [ y for y in [ x.split ('=') for x in args ] if len (y) == 2] )
        args  =  [  x for x in args if '=' not in x  ]

        staging_path = '.'
        self.auth_token = named_args.get ('bisque_token')
        self.image_map_name = os.path.join(staging_path, IMAGE_MAP)
        self.resource_url = named_args.get ('image_url')
        self.mex_url = named_args.get ('mex_url')
        self.images = os.path.join(staging_path, 'images') + os.sep


        if self.auth_token:
            self.bq = BQSession().init_mex(self.mex_url, self.auth_token)
        else:
            user,pwd = options.credentials.split(':')
            self.bq = BQSession().init_local(user,pwd)

        resource_xml = self.bq.fetchxml (self.resource_url, view='short')
        self.is_dataset = resource_xml.tag == 'dataset'

        if len(args) == 1:
            commands = [ args.pop(0)]
        else:
            commands =['setup','start', 'teardown']

        #if command not in ('setup','teardown', 'start'):
        #    parser.error('Command must be start, setup or teardown')


        # maltab code requires trailing slash..

        try:
            for command in commands:
                command = getattr(self, command)
                r = command()
        except Exception, e:
            logging.exception ("problem during %s" % command)
            self.bq.fail_mex(msg = "Exception during %s: %s" % (command,  str(e)))
            sys.exit(1)
Example #5
0
    def main(self, mex_url=None, bisque_token=None, image_url=None, bq=None):
        #  Allow for testing by passing an alreay initialized session
        if bq is None:
            bq = BQSession().init_mex(mex_url, bisque_token)

        bq.update_mex('Classifying...')
        pars = bq.parameters()
        image_url = pars.get('data_url', None)
        model_url = pars.get('model_url', None)
        store_on_image = pars.get('store_on_image', False)
        method = pars.get('method', 'points')

        points = int(pars.get('number_of_points', 10))
        confidence = int(pars.get('confidence', 95))
        border = int(pars.get('border', 0))

        _, image_uniq = image_url.rsplit('/', 1)
        _, model_uniq = model_url.rsplit('/', 1)
        url = '%s/connoisseur/%s/classify:%s/method:%s/points:%s/confidence:%s/border:%s' % (
            bq.bisque_root, model_uniq, image_uniq, method, points, confidence,
            border)

        # dima: the moving of the image is not of the best taste, this should be changed later
        if method != 'segmentation':
            url += '/format:xml'
            txt = bq.c.fetch(url,
                             headers={
                                 'Content-Type': 'text/xml',
                                 'Accept': 'text/xml'
                             })
            gobs = etree.fromstring(txt)
        else:
            color_mode = 'colors'
            url += '/colors:colors'
            filename = '%s_%s_conf%.2f_n%s_b%s.png' % (
                image_uniq, color_mode, confidence, points, border)
            filename = bq.c.fetch(url, path=filename)
            image_url = upload_file(bq, filename) or ''
            gobs = None

        bq.update_mex('Storing results...')
        outputs = etree.Element('tag', name="outputs")
        img = etree.SubElement(outputs,
                               'tag',
                               name="MyImage",
                               type="image",
                               value=image_url)
        if gobs is not None:
            img.append(gobs)
        if store_on_image is True:
            r = bq.postxml(image_url, xml=txt)

        bq.finish_mex(tags=[outputs])
Example #6
0
    def setUpClass(self):
        config = ConfigParser.ConfigParser()
        config.read('config.cfg')

        self.root = config.get('Host', 'root') or 'localhost:8080'
        self.user = config.get('Host', 'user') or 'test'
        self.pswd = config.get('Host', 'password') or 'test'

        self.session = BQSession().init_local(self.user,
                                              self.pswd,
                                              bisque_root=self.root,
                                              create_mex=False)
Example #7
0
def test_open_session(config):
    """
        Test Initalizing a BQSession locally
    """
    host = config.get('host.root')
    user = config.get('host.user')
    pwd = config.get('host.password')

    bqsession = BQSession().init_local(user,
                                       pwd,
                                       bisque_root=host,
                                       create_mex=False)
    bqsession.close()
Example #8
0
def test_initalize_mex_locally(config):
    """
        Test initalizing a mex locally
    """
    host = config.get('host.root')
    user = config.get('host.user')
    pwd = config.get('host.password')
    bqsession = BQSession().init_local(user,
                                       pwd,
                                       bisque_root=host,
                                       create_mex=True)
    assert bqsession.mex.uri
    bqsession.close()
Example #9
0
 def command_failed(self, process, retcode):
     """Update the bisque server  with a failed command for a mex"""
     mex = process['mex']
     mex.status = "failed"
     command = " ".join(process['command_line'])
     msg = "%s: returned (non-zero) %s" % (command, retcode)
     self.error("condor_failed: " + msg)
     # update process mex
     if self.session is None:
         self.session = BQSession().init_mex(self.mexes[0].mex_url,
                                             self.mexes[0].bisque_token)
     if self.session.mex.value not in ('FAILED', 'FINISHED'):
         self.session.fail_mex(msg)
Example #10
0
def test_module():
    from bqapi import BQSession
    session = BQSession().init_local('admin', 'admin', 'http://localhost:8080')
    admin = session.service('admin')
    data = session.service('data_service')
    #admin.user(uniq).login().fetch ()
    xml = data.get("user",
                   params={
                       'wpublic': '1',
                       'resource_name': 'admin'
                   },
                   render='xml')
    user_uniq = xml.find("user").get('resource_uniq')
    admin.fetch('/user/{}/login'.format(user_uniq))
    def setUpClass(self):
        config = ConfigParser.ConfigParser()
        config.read('config.cfg')

        self.root = config.get('Host', 'root') or 'localhost:8080'
        self.user = config.get('Host', 'user') or 'test'
        self.pswd = config.get('Host', 'password') or 'test'

        self.session = BQSession().init_local(self.user,
                                              self.pswd,
                                              bisque_root=self.root,
                                              create_mex=False)

        # download and upload test images ang get their IDs
        self.resource_unicode_jpeg = self.ensure_bisque_file(
            image_unicode_jpeg.decode('utf-8'))
        self.resource_unicode_mov = self.ensure_bisque_file(
            image_unicode_mov.decode('utf-8'))
        self.resource_unicode_oib = self.ensure_bisque_file(
            image_unicode_oib.decode('utf-8'))
        self.resource_unicode_tiff = self.ensure_bisque_file(
            image_unicode_tiff.decode('utf-8'))
        self.resource_unicode_ims = self.ensure_bisque_file(
            image_unicode_ims.decode('utf-8'))
        self.resource_unicode_dicom = self.ensure_bisque_file(
            image_unicode_dicom.decode('utf-8'))
        self.resource_unicode_svs = self.ensure_bisque_file(
            image_unicode_svs.decode('utf-8'))
        self.resource_latin1_tiff = self.ensure_bisque_file(
            image_latin1_tiff.decode('utf-8'))
Example #12
0
    def setUpClass(self):
        config = ConfigParser.ConfigParser()
        config.read('config.cfg')
        self.root = config.get('Host', 'root') or 'localhost:8080'
        self.user = config.get('Host', 'user') or 'test'
        self.pswd = config.get('Host', 'password') or 'test'
        self.session = BQSession().init_local(self.user,
                                              self.pswd,
                                              bisque_root=self.root,
                                              create_mex=False)

        print '\nUploading images\n'

        self.resources_plane = []
        self.resources_plane.append(self.ensure_bisque_file(image_planar))
        self.resources_plane.append(self.ensure_bisque_file(image_planar))
        self.resources_plane.append(self.ensure_bisque_file(image_planar))
        self.resources_plane.append(self.ensure_bisque_file(image_planar))
        self.resources_plane.append(self.ensure_bisque_file(image_planar))

        self.resources_pyr = []
        self.resources_pyr.append(self.ensure_bisque_file(image_pyramid))
        self.resources_pyr.append(self.ensure_bisque_file(image_pyramid))
        self.resources_pyr.append(self.ensure_bisque_file(image_pyramid))
        self.resources_pyr.append(self.ensure_bisque_file(image_pyramid))
        self.resources_pyr.append(self.ensure_bisque_file(image_pyramid))

        self.resources_valid_plane = [self.ensure_bisque_file(image_planar)]
        self.resources_valid_pyr = [self.ensure_bisque_file(image_pyramid)]

        self.coordinates = [(0, 0), (1, 3), (2, 3), (7, 1), (2, 2), (5, 2),
                            (7, 2), (1, 1), (6, 3), (7, 3)]
Example #13
0
def setUp():
    global results_location
    global store_local_location
    global file1_location
    global filename1
    global bqsession
    global FeatureResource

    config = ConfigParser.ConfigParser()
    config.read('setup.cfg')
    root = config.get('Host', 'root') or 'localhost:8080'
    user = config.get('Host', 'user') or 'test'
    pwd = config.get('Host', 'password') or 'test'
    results_location = config.get('Store', 'results_location') or 'Results'
    _mkdir(results_location)

    store_location = config.get('Store', 'location') or None
    if store_location is None: raise NameError('Requre a store location to run test properly')

    store_local_location = config.get('Store', 'local_location') or 'SampleData'
    filename1 = config.get('Store','filename1') or None
    if filename1 is None: raise NameError('Requre an image to run test properly')
    file1_location = fetch_file(filename1, store_location, store_local_location)

    FeatureResource = namedtuple('FeatureResource',['image','mask','gobject'])
    FeatureResource.__new__.__defaults__ = (None, None, None)
    #start session
    bqsession = BQSession().init_local(user, pwd, bisque_root=root, create_mex=False)
Example #14
0
    def run(self):
        parser = optparse.OptionParser()
        parser.add_option('-d', '--debug', action="store_true")
        parser.add_option('-n', '--dryrun', action="store_true")
        parser.add_option('--credentials')
        parser.add_option('--image_url')

        (options, args) = parser.parse_args()
        named = AttrDict(bisque_token=None,
                         mex_url=None,
                         staging_path=None,
                         image_url=None)
        for arg in list(args):
            tag, sep, val = arg.partition('=')
            if sep == '=':
                named[tag] = val
                args.remove(arg)
        self.named_args = named
        self.staging_path = named.get('staging_path')

        if named.bisque_token:
            self.bq = BQSession().init_mex(named.mex_url, named.bisque_token)
            self.resource_url = self.bq.parameter_value('image_url')
        elif options.credentials:
            user, pwd = options.credentials.split(':')
            self.bq = BQSession().init_local(user, pwd)
            self.resource_url = named.image_url
        else:
            parser.error('need bisque_token or user credential')

        if self.resource_url is None:
            parser.error('Need a resource_url')

        if not args:
            commands = ['setup', 'start', 'teardown']
        else:
            commands = [args]

        try:
            for command in commands:
                command = getattr(self, command)
                r = command()
        except Exception, e:
            logging.exception("problem during %s" % command)
            self.bq.fail_mex(msg="Exception during %s: %s" % (command, e))
            sys.exit(1)
Example #15
0
    def command_kill(self, **kw):
        """Kill the running module if possible
        """
        mex = kw.get('mex_tree')
        topmex = self.mexes[0]
        if mex is not None:
            mex_id = mex.get('resource_uniq')
            self.pool.kill(selector_fct=lambda task: '00-' + task['mex'].get(
                'mex_url').split('/00-', 1)[1].split('/', 1)[0] == mex_id)
            if self.session is None:
                mex_url = topmex.named_args['mex_url']
                token = topmex.named_args['bisque_token']
                self.session = BQSession().init_mex(mex_url, token)
            self.session.fail_mex(msg='job stopped by user')
        else:
            self.debug("No mex provided")

        return None
Example #16
0
	def _authenticate(self, attempts=25):
		'''
		Attempts to authenticate N times
		
		@attempts(optional): number of attempts to authenticate
		'''
		for i in range(attempts):
			s = BQSession().init_cas(self.username, self.password, bisque_root=self.bisque_root)
			try:
				if s._check_session():
					self._debug_print('Authenticated successfully!')
					return s
			except:
				self._debug_print('Unable to authenticate... trying again')
				pass
			time.sleep(30)
			
		self._debug_print('Unable to authenticate.', 'error')
		return None
Example #17
0
    def _authenticate(self, attempts=25):
        """
		Attempts to authenticate N times
		
		@attempts(optional): number of attempts to authenticate
		"""
        for i in range(attempts):
            s = BQSession().init_cas(self.username, self.password, bisque_root=self.bisque_root)
            try:
                if s._check_session():
                    self._debug_print("Authenticated successfully!")
                    return s
            except:
                self._debug_print("Unable to authenticate... trying again")
                pass
            time.sleep(30)

        self._debug_print("Unable to authenticate.", "error")
        return None
Example #18
0
    def command_finish(self, **kw):
        # Cleanup condor stuff and look for error files.

        topmex = self.mexes[0]
        job_return = int(topmex.named_args.get('condor_job_return', 0))
        #job_return = int(topmex.arguments.pop())
        self.info("condor_finish %s: return=%s", topmex.executable, job_return)
        if job_return != 0:
            if self.session is None:
                mex_url = topmex.named_args['mex_url']
                token = topmex.named_args['bisque_token']
                self.session = BQSession().init_mex(mex_url, token)
            # Possible look for log files and append to message here
            #if os.path.exists(''):
            #    pass
            self.session.fail_mex(msg='job failed with return code %s' %
                                  job_return)
            return None
        topmex.status = "finished"
        return super(CondorRunner, self).command_finish(**kw)
Example #19
0
def test_initalize_session_From_mex(config):
    """
        Test initalizing a session from a mex
    """
    host = config.get('host.root')
    user = config.get('host.user')
    pwd = config.get('host.password')
    bqsession = BQSession().init_local(user, pwd, bisque_root=host)
    mex_url = bqsession.mex.uri
    token = bqsession.mex.resource_uniq
    bqmex = BQSession().init_mex(mex_url, token, user, bisque_root=host)
    bqmex.close()
    bqsession.close()
Example #20
0
    def main(self, image_url, mex_url=None, bisque_token=None, bq=None):
        #  Allow for testing by passing an alreay initialized session
        if bq is None:
            bq = BQSession().init_mex(mex_url, bisque_token)
        # Fetch the image metadata
        image = bq.load(image_url)

        # Fetch embedded tags from image service
        meta = image.pixels().meta().fetch()
        meta = ET.XML(meta)
        tags = []
        # Create a new tag 'MetaData' to be placed on the image
        md = BQTag(name='MetaData')
        # Filter the embedded metadata and place subtags in MetaData
        for t in meta.getiterator('tag'):
            if t.get('name') in wanted_tags:
                md.addTag(name=t.get('name'), value=t.get('value'))
        # Add the new tag to the image
        image.addTag(tag=md)
        metadata_tag = bq.save(md, image.uri + "/tag")
        bq.finish_mex(tags=[{
            'name':
            'outputs',
            'tag': [{
                'name': 'metadata',
                'value': metadata_tag.uri,
                'type': 'tag'
            }]
        }])
        sys.exit(0)
Example #21
0
def mexsession(config):
    "Create a BQApi BQSession object based on config"
    host = config.get ( 'host.root')
    user = config.get ( 'host.user')
    passwd = config.get ( 'host.password')

    bq = BQSession()
    bq.config = config
    bq.init_local (user, passwd, bisque_root = host, create_mex = True)
    yield  bq
    bq.close()
Example #22
0
    def command_start(self, **kw):
        self.info("starting %d mexes -> %s", len(self.mexes), self.mexes)
        if self.session is None:
            self.session = BQSession().init_mex(self.mexes[0].mex_url,
                                                self.mexes[0].bisque_token)
        status = "starting"
        self.entrypoint_executable = self.mexes[0].executable
        if self.mexes[0].iterables:
            self.mexes[0].executable = None
            status = 'running parallel'
        # add empty "outputs" section in topmex
        #self.session.update_mex(status=status, tags=[{'name':'outputs'}])
        self.session.update_mex(
            status=status
        )  # dima: modules add outputs section and the empty one complicates module UI
        # if there is a prerun, run it now
        if self.prerun:
            self.info("prerun starting")
            self.command_single_entrypoint(self.prerun, self.command_execute,
                                           **kw)
            return None

        return self.command_execute
Example #23
0
    def setUpClass(self):
        config = ConfigParser.ConfigParser()
        config.read('config.cfg')

        self.root = config.get('Host', 'root') or 'localhost:8080'
        self.user = config.get('Host', 'user') or 'test'
        self.pswd = config.get('Host', 'password') or 'test'

        self.session = BQSession().init_local(self.user, self.pswd,  bisque_root=self.root, create_mex=False)

        # download and upload test images ang get their IDs
        self.resource_2d_uint8  = self.ensure_bisque_file(image_rgb_uint8)
        self.resource_3d_uint16 = self.ensure_bisque_file(image_zstack_uint16)
        self.resource_2d_float  = self.ensure_bisque_file(image_float)
Example #24
0
def setup_simple_feature_test(ns):
    """
        Setup feature requests test
    """
    config = ConfigParser.ConfigParser()
    config.read(CONFIG_FILE)
    root = config.get('Host', 'root') or DEFAULT_ROOT
    user = config.get('Host', 'user') or DEFAULT_USER
    pwd = config.get('Host', 'password') or DEFAULT_PASSWORD
    results_location = config.get('Store', 'results_dir') or DEFAULT_RESULTS_DIR
    store_location = config.get('Store', 'location') or None
    store_local_location = config.get('Store', 'local_dir') or DEFAULT_LOCAL_DIR
    temp_store = config.get('Store', 'temp_dir') or DEFAULT_TEMPORARY_DIR
    test_image = config.get('SimpleTest', 'test_image') or None
    feature_response_results = config.get('SimpleTest', 'feature_response') or DEFAULT_FEATURE_RESPONSE_HDF5
    feature_past_response_results = config.get('SimpleTest','feature_sample') or DEFAULT_FEATURE_SAMPLE_HDF5

    if store_location is None: raise NameError('Requre a store location to run test properly')
    if test_image is None: raise NameError('Requre an image to run test properly')

    _mkdir(store_local_location)
    _mkdir(results_location)
    _mkdir(temp_store)

    results_table_path = os.path.join(results_location, feature_response_results)
    if os.path.exists(results_table_path):
        os.remove(results_table_path)


    test_image_location = fetch_file(test_image, store_location, store_local_location)

    #initalize session
    session = BQSession().init_local(user, pwd, bisque_root=root)

    #set to namespace
    ns.root = root
    ns.store_location = store_location
    ns.session = session
    ns.results_location = results_location
    ns.store_local_location = store_local_location
    ns.test_image_location = test_image_location
    ns.feature_response_results = feature_response_results
    ns.feature_past_response_results = feature_past_response_results
    ns.test_image = test_image
    ns.temp_store = temp_store
    def setUpClass(self):
        config = ConfigParser.ConfigParser()
        config.read('config.cfg')

        self.root = config.get('Host', 'root') or 'localhost:8080'
        self.user = config.get('Host', 'user') or 'test'
        self.pswd = config.get('Host', 'password') or 'test'

        self.session = BQSession().init_local(self.user,
                                              self.pswd,
                                              bisque_root=self.root,
                                              create_mex=False)

        # download and upload test images ang get their IDs
        self.resource_nikon_nd2 = self.ensure_bisque_file(image_nikon_nd2)
        self.resource_nikon_nd2_deconv = self.ensure_bisque_file(
            image_nikon_nd2_deconv)
        self.resource_svs = self.ensure_bisque_file(image_svs)
Example #26
0
    def setUpClass(self):
        config = ConfigParser.ConfigParser()
        config.read('config.cfg')

        self.root = config.get('Host', 'root') or 'localhost:8080'
        self.user = config.get('Host', 'user') or 'test'
        self.pswd = config.get('Host', 'password') or 'test'

        self.session = BQSession().init_local(self.user,
                                              self.pswd,
                                              bisque_root=self.root,
                                              create_mex=False)

        # download and upload test images ang get their IDs
        self.resource_imaris_hela = self.ensure_bisque_file(image_imaris_hela)
        self.resource_imaris_r18 = self.ensure_bisque_file(image_imaris_r18)
        self.resource_zeiss_czi_rat = self.ensure_bisque_file(
            image_zeiss_czi_rat)
        self.resource_dicom_3d = self.ensure_bisque_file(image_dicom_3d)
        self.resource_dicom_2d = self.ensure_bisque_file(image_dicom_2d)
    def main(self, image_url,  mex_url = None, bisque_token=None, bq = None, args = None):
        #  Allow for testing by passing an alreay initialized session
        if bq is None:
            bq = BQSession().init_mex(mex_url, bisque_token)
        # Fetch the blob  links
        if not os.path.exists ('videos'):
            os.makedirs ('videos')
        video = fetch_blob(bq, image_url, 'videos')
        print "VIDEO file ", video
	 #pass arguments to MotionMeerkat scripts, located in MotionMeerkat/main.py	
	#Structure arguments
	#Format call string

        callargs = ["python MotionMeerkat/main.py", 
                    "--i", video.values()[0], 
                    "--threshT", args[0],
                    "--sub", args[1],
                    "--mogh", args[2],
                    "--mogv", args[3],
                    "--accA", args[4],
                    "--burn", args[5],
                    "--frameSET", 
                    "--frame_rate", "1",
                    "--makeV", "none",
                    "--fileD", "Output"]
	print "Calling ", " ".join(callargs)

	#run MotionMeerkat
	r =  call(" ".join(callargs), shell=True)
        if r != 0:
		bq.fail_mex ("Meerkat returned non-zero")
		#Post Results
	
	#get filename to get the destination of the csv to be posted
	fn=video.values()[0]
	fbs= os.path.basename(fn)
	head, tail = os.path.splitext(fbs)
	
	#post file
	frames_blob = bq.postblob(str("Output/" + head + "/Frames.csv"))
	
	#get file location from regex
	uri=re.search("uri=\"(.*?)\"", frames_blob).group(1)
	tags = [{ 'name': 'outputs','tag' : [{'name': 'frames_csv', 'type':'file', 'value':uri}]}]
	bq.finish_mex(tags = tags)
	sys.exit(0)
Example #28
0
    def main(self, mex_url=None, auth_token=None, bq=None, **kw):
        #  Allow for testing by passing an alreay initialized session
        if bq is None:
            bq = BQSession().init_mex(mex_url, auth_token)

        # check for list parameters
        params = bq.get_mex_inputs()
        if isinstance(params,
                      dict) or not isinstance(params, collections.Iterable):
            params = [params]
        # pass values directly as args
        for single_params in params:
            for param_name in single_params:
                if 'value' in single_params[param_name]:
                    single_params[param_name] = single_params[param_name].get(
                        'value')

        # TODO: measure block startup time
        self.start_block(bq, params)

        for kw in params:
            # TODO: measure single item time
            # TODO: run in parallel
            if 'mex_url' in kw:
                # set (innermost) single item mex
                sub_bq = BQSession().init_mex(kw['mex_url'], auth_token)
            else:
                sub_bq = bq
            self.process_single(sub_bq, **kw)
            if 'mex_url' in kw:
                sub_bq.close()

        # TODO: measure block teardown time
        self.end_block(bq)

        sys.exit(0)
Example #29
0
H = 2048

##################################################################
# create session
##################################################################

#config = ConfigParser.ConfigParser()
#config.read(config_file)
#root = config.get('Host', 'root') or 'http://bisque.ece.ucsb.edu'
#user = config.get('Host', 'user') or 'Mex'
#pswd = config.get('Host', 'password') or 'Mary:00-rvFkDRQyoheksMZnhBhi2n'
#session = BQSession().init_local(user, pswd,  bisque_root=root, create_mex=False)

mex_url = 'http://bisque.ece.ucsb.edu/module_service/mex/00-rvFkDRQyoheksMZnhBhi2n'
bisque_token = 'Mary:00-rvFkDRQyoheksMZnhBhi2n'
session = BQSession().init_mex(mex_url, bisque_token)

##################################################################
# index excel file
##################################################################


# filename-row hash
meta = {}
data = pd.read_excel(fn)
Ks = data.keys()
N = len(data['ImageName'])

for i in range(N):
    fn = data['ImageName'][i].strip(' ')
    m = {}
Example #30
0
        outputs = etree.Element('tag', name='outputs')
        summary = etree.SubElement(outputs, 'tag', name='summary')
        for r,v in resources.iteritems():
            etree.SubElement(summary, 'tag', name=r, value=v)

        bq.finish_mex(tags=[outputs])

if __name__ == "__main__":
    import optparse
    parser = optparse.OptionParser()
    parser.add_option("-c", "--credentials", dest="credentials",
                      help="credentials are in the form user:password")
    (options, args) = parser.parse_args()

    M = AnnotationHistograms()
    if options.credentials is None:
        mex_url, auth_token = args[:2]
        bq = BQSession().init_mex(mex_url, auth_token)
    else:
        mex_url = ''
        if not options.credentials:
            parser.error('need credentials')
        user,pwd = options.credentials.split(':')
        bq = BQSession().init_local(user, pwd)

    try:
        M.main(mex_url=mex_url, bq=bq )
    except Exception, e:
        bq.fail_mex(traceback.format_exc())
    sys.exit(0)
from bqapi import BQSession, BQCommError
from bqapi.util import save_blob, localpath2url

##################################################################
# Upload
##################################################################


config = ConfigParser.ConfigParser()
config.read('config.cfg')

root = config.get('Host', 'root') or 'localhost:8080'
user = config.get('Host', 'user') or 'test'
pswd = config.get('Host', 'password') or 'test'

session = BQSession().init_local(user, pswd,  bisque_root=root, create_mex=False)

request = '<image name="%s">'%'series_t_jpeg'
request = '%s<value>%s</value>'%(request, localpath2url('f:/dima/develop/python/bq5irods/data/imagedir/admin/tests/multi_file/series_t_jpeg/t_000241.jpg'))
request = '%s<value>%s</value>'%(request, localpath2url('f:/dima/develop/python/bq5irods/data/imagedir/admin/tests/multi_file/series_t_jpeg'))
request = '%s<tag name="%s" type="%s">'%(request, 'image_meta', 'image_meta')
request = '%s<tag name="%s" value="%s" type="number" />'%(request, 'image_num_z', '1')
request = '%s<tag name="%s" value="%s" type="number" />'%(request, 'image_num_t', '80')
request = '%s<tag name="%s" value="%s" />'%(request, 'dimensions', 'XYCZT')
request = '%s<tag name="%s" value="%s" type="number" />'%(request, 'pixel_resolution_x', '0.4')
request = '%s<tag name="%s" value="%s" type="number" />'%(request, 'pixel_resolution_y', '0.4')
request = '%s<tag name="%s" value="%s" type="number" />'%(request, 'pixel_resolution_t', '2.0')
request = '%s<tag name="%s" value="%s" />'%(request, 'pixel_resolution_unit_x', 'micron')
request = '%s<tag name="%s" value="%s" />'%(request, 'pixel_resolution_unit_y', 'micron')
request = '%s<tag name="%s" value="%s" />'%(request, 'pixel_resolution_unit_t', 'seconds')
request = '%s</tag>'%(request)
Example #32
0
class ImageServiceTestBase(unittest.TestCase):
    """
        Test image service operations
    """
    @classmethod
    def setUpClass(self):
        config = ConfigParser.ConfigParser()
        config.read('config.cfg')

        self.root = config.get('Host', 'root') or 'localhost:8080'
        self.user = config.get('Host', 'user') or 'test'
        self.pswd = config.get('Host', 'password') or 'test'

        self.session = BQSession().init_local(self.user,
                                              self.pswd,
                                              bisque_root=self.root,
                                              create_mex=False)

        # download and upload test images ang get their IDs
        #self.uniq_2d_uint8  = self.ensure_bisque_file(image_rgb_uint8)
        #self.uniq_3d_uint16 = self.ensure_bisque_file(image_zstack_uint16)

    @classmethod
    def tearDownClass(self):
        #self.delete_resource(self.uniq_2d_uint8)
        #self.delete_resource(self.uniq_3d_uint16)
        self.cleanup_tests_dir()
        pass

    @classmethod
    def fetch_file(self, filename):
        _mkdir(local_store_images)
        _mkdir(local_store_tests)
        url = posixpath.join(url_image_store, filename).encode('utf-8')
        path = os.path.join(local_store_images, filename)
        if not os.path.exists(path):
            urllib.urlretrieve(url, path)
        return path

    @classmethod
    def upload_file(self, path, resource=None):
        #if resource is not None:
        #    print etree.tostring(resource)
        r = save_blob(self.session, path, resource=resource)
        if r is None or r.get('uri') is None:
            print 'Error uploading: %s' % path.encode('ascii', 'replace')
            return None
        print 'Uploaded id: %s url: %s' % (r.get('resource_uniq'),
                                           r.get('uri'))
        return r

    @classmethod
    def delete_resource(self, r):
        if r is None:
            return
        url = r.get('uri')
        print 'Deleting id: %s url: %s' % (r.get('resource_uniq'), url)
        self.session.deletexml(url)

    @classmethod
    def delete_package(self, package):
        if 'dataset' in package:
            # delete dataset
            url = package['dataset']
            print 'Deleting dataset: %s' % (url)
            try:
                self.session.fetchxml('/dataset_service/delete?duri=%s' % url)
            except BQCommError:
                print 'Error deleting the dataset'
        elif 'items' in package:
            # delete all items
            for url in package['items']:
                print 'Deleting item: %s' % (url)
                try:
                    self.session.deletexml(url)
                except BQCommError:
                    print 'Error deleting the item'

        # # delete dataset
        # if 'dataset' in package:
        #     url = package['dataset']
        #     print 'Deleting dataset: %s'%(url)
        #     try:
        #         self.session.deletexml(url)
        #     except BQCommError:
        #         print 'Error deleting the dataset'

        # # delete all items
        # if 'items' in package:
        #     for url in package['items']:
        #         print 'Deleting item: %s'%(url)
        #         try:
        #             self.session.deletexml(url)
        #         except BQCommError:
        #             print 'Error deleting the item'

    @classmethod
    def ensure_bisque_file(self, filename, metafile=None):
        path = self.fetch_file(filename)
        if metafile is None:
            filename = u'%s/%s' % (TEST_PATH, filename)
            resource = etree.Element('resource', name=filename)
            return self.upload_file(path, resource=resource)
        else:
            metafile = self.fetch_file(metafile)
            return self.upload_file(path,
                                    resource=etree.parse(metafile).getroot())

    @classmethod
    def ensure_bisque_package(self, package):
        path = self.fetch_file(package['file'])
        r = self.upload_file(path, resource=etree.XML(package['resource']))
        package['resource'] = r
        if r is None:
            return None
        print 'Uploaded id: %s url: %s' % (r.get('resource_uniq'),
                                           r.get('uri'))
        #print etree.tostring(r)
        if r.tag != 'dataset':
            package['items'] = [r.get('uri')]
        else:
            package['dataset'] = r.get('uri')
            values = r.xpath('value')
            if len(values) != package['count']:
                print 'Error: uploaded %s has %s elements but needs %s' % (
                    package['file'], len(values), package['count'])
            if r.get('name') != package['name']:
                print 'Error: uploaded %s name is %s but should be %s' % (
                    package['file'], r.get('name'), package['name'])
            package['items'] = [x.text for x in values]

        package['last'] = self.session.fetchxml(package['items'][-1],
                                                view='deep')
        #print 'Last item\n'
        #print etree.tostring(package['last'])

    @classmethod
    def cleanup_tests_dir(self):
        print 'Cleaning-up %s' % local_store_tests
        for root, dirs, files in os.walk(local_store_tests, topdown=False):
            for name in files:
                os.remove(os.path.join(root, name))

    def validate_image_variant(self,
                               resource,
                               filename,
                               commands,
                               meta_required=None):
        path = os.path.join(local_store_tests, filename)
        try:
            #image = fromXml(resource, session=self.session)
            image = self.session.factory.from_etree(resource)
            px = image.pixels()
            for c, a in commands:
                px = px.command(c, a)
            px.fetch(path)
        except BQCommError:
            logging.exception('Comm error')
            self.fail('Communication error while fetching image')

        if meta_required is not None:
            meta_test = metadata_read(path)
            self.assertTrue(meta_test is not None,
                            msg='Retrieved image can not be read')
            self.assertTrue(
                compare_info(meta_required, meta_test),
                msg='Retrieved metadata differs from test template')

    def validate_xml(self, resource, filename, commands, xml_parts_required):
        path = os.path.join(local_store_tests, filename)
        try:
            #image = fromXml(resource, session=self.session)
            image = self.session.factory.from_etree(resource)
            px = image.pixels()
            for c, a in commands:
                px = px.command(c, a)
            px.fetch(path)
        except BQCommError:
            self.fail()

        xml_test = etree.parse(path).getroot()
        #print etree.tostring(xml_test)
        self.assertTrue(compare_xml(xml_parts_required, xml_test),
                        msg='Retrieved XML differs from test template')