def _instantiate_pipeline(self, pipeline_url, params): """ instantiate dream.3d pipeline file with provided parameters """ pipeline_path = urlparse.urlsplit(pipeline_url).path.split('/') pipeline_uid = pipeline_path[1] if is_uniq_code( pipeline_path[1]) else pipeline_path[2] url = self.bqSession.service_url( 'pipeline', path='/'.join( [pipeline_uid] + ["setvar:%s|%s" % (tag, params[tag]) for tag in params] + ['exbsteps:dream3d']), query={'format': 'dream3d'}) pipeline = self.bqSession.c.fetch(url) if not pipeline: # bad pipeline return None, None out_pipeline_file = os.path.join(self.options.stagingPath, 'pipeline.json') out_error_file = os.path.join(self.options.stagingPath, 'dream3d_error.txt') with open(out_pipeline_file, 'w') as fo: fo.write(pipeline) return out_pipeline_file, out_error_file
def mex_acl_handler(resource_uniq, user_uniq, newauth, action): """Special handling for mexes Share A mexes input and outputs """ mex = data_service.resource_load(uniq=resource_uniq, view='deep') # extract urls from mex inputs and outputs points_from_list = [ x.rsplit('/', 1)[1] for x in mex.xpath('./tag[@name="inputs"]/tag/@value') if x.startswith("http") ] points_to_list = [ x.rsplit('/', 1)[1] for x in mex.xpath('./tag[@name="outputs"]/tag/@value') if x.startswith("http") ] mex_resources = [ x for x in points_from_list + points_to_list if is_uniq_code(x) ] log.debug("Discovered incoming %s outgoing %s = %s", points_from_list, points_to_list, mex_resources) resource_acls(mex_resources, newauth, action=action)
def load_uri(uri, query=False): '''Load the object specified by the root tree and return a rsource @type root: Element @param root: The db object root with a uri attribute @rtype: tag_model.Taggable @return: The resource loaded from the database ''' # Check that we are looking at the right resource. try: resource = None service, clname, ida, rest = parse_bisque_uri(uri) if service not in ('data_service', 'module_service'): return None if ida and is_uniq_code(ida): log.debug("loading resource_uniq %s", ida) resource = DBSession.query(Taggable).filter_by(resource_uniq=ida) elif clname: name, dbcls = dbtype_from_tag(clname) #log.debug("loading %s -> name/type (%s/%s)(%s) " , uri, name, str(dbcls), ida) #resource = DBSession.query(dbcls).get (int (ida)) resource = DBSession.query(dbcls).filter(dbcls.id == int(ida)) if not query: resource = resource.first() #log.debug ("loaded %s" % resource) return resource except Exception: log.exception("Failed to load uri %s", uri) return None
def load(self, token, **kw): log.debug("Load %s" % token) try: if is_uniq_code(token): return resource_load(self.resource_type, uniq=token) return resource_load(self.resource_type, ident=int(token)) except ValueError, e: abort(404)
def load_token(token): if is_uniq_code(token): return resource_load(uniq=token).first() try: token = int(token) return resource_load(ident=token).first() except ValueError: pass return None
def _cache_ppops(self, pipeline_url): if not self.ppops or self.ppops_url != pipeline_url: pipeline_path = urlparse.urlsplit(pipeline_url).path.split('/') pipeline_uid = pipeline_path[1] if is_uniq_code( pipeline_path[1]) else pipeline_path[2] url = self.bqSession.service_url('pipeline', path='/'.join([pipeline_uid] + ['ppops:dream3d'])) self.ppops = json.loads(self.bqSession.c.fetch(url)) self.ppops_url = pipeline_url
def parse_bisque_uri(uri): ''' Parse a bisquie uri into service, and optional dbclass , and ID @type uri: string @param uri: a bisquik uri representation of a resourc @rtype: A quad-tuple (service, dbclass, id, rest ) @return: The parse resource ''' # (scheme, host, path, ...) url = urlparse.urlsplit(uri) if url.scheme not in ('http', 'https', ''): return None, None, None, None # /service_name/ [ id or or class ]* parts = posixpath.normpath(url[2]).strip('/').split('/') # paths are /class/id or /resource_uniq if not parts: return url[1], 'data_service', None, None # should have a service name or a uniq code if is_uniq_code(parts[0]): service = 'data_service' # class follows or nothing else: service = parts.pop(0) # first element may be a docid ida = None if parts and is_uniq_code(parts[0]): ida = parts.pop(0) clname = None rest = [] while parts: sym = parts.pop() if not sym[0].isdigit(): rest.append(sym) continue ida = sym if parts: clname = parts.pop() break return service, clname, ida, rest
def _lookup(self, service_type, *rest): """Default handler for bisque root. Search for registered sub-servers that might handle this request. return an error if none found URL are formed with <service_type>/<path>?arguments """ if request.path in NOLOGGING: log.debug('[%s] %s %s', request.identity.get('repoze.who.userid'), request.method, request.url) else: log.info('[%s] %s %s', request.identity.get('repoze.who.userid'), request.method, request.url) if log.isEnabledFor(logging.DEBUG): log.debug('HEADERS: %s %s', request.url, dict(request.headers)) log.debug('PATH %s in DIR %s', request.path, os.getcwd()) if service_type in oldnames: #service_type = oldnames[service_type] newreq = request.path_qs[1:].replace(service_type, oldnames[service_type]) log.warn('found oldname( %s ) -> newreq %s', service_type, newreq) redirect(urlparse.urljoin(request.application_url, newreq)) #log.debug ("find controller for %s " % (str(service_type) )) #log.debug ("lookup for %s/%s" , service_type, rest ) #import pdb #pdb.set_trace() service = service_registry.find_service(service_type) if service is not None: #log.debug ('found %s ' , str(service)) if service.service_type != 'auth_service' and 'expires' in session: session_update() return service, rest if is_uniq_code(service_type): log.debug("uniq code %s", request.path_qs) # Skip 1st / redirect( urlparse.urljoin(request.application_url, '/data_service/%s' % request.path_qs[1:])) log.warn('no service found %s with %s', service_type, rest) abort(404)
def _default(self, *path, **kw): #log.debug ('headers:'+ str(cherrypy.request.headers)) path = list(path) #log.debug ("path = %s %s " , path, kw) token = path.pop(0) if is_uniq_code(token): log.debug('using uniq token') resource_controller = self.get_child_resource('resource') path.insert(0, token) else: handler = RESOURCE_HANDLERS.get(token, None) # Allow override of path parsing by specific resource controller i.e. dataset etc. if handler: resource_controller = handler(self.url) else: resource_controller = self.get_child_resource(token) return resource_controller._default(*path, **kw)
def upgrade(): return op.add_column('taggable', Column('resource_number', Float)) op.add_column('taggable', Column('resource_obj', Integer, ForeignKey ('taggable.id'))) cntxt = context.get_context() SessionMaker = sessionmaker(bind=cntxt.bind) Base = declarative_base() metadata = MetaData(bind=cntxt.bind) class Taggable(Base): __table__ = Table('taggable', metadata, autoload=True) DBSession = SessionMaker() converted_numbers = 0 converted_objs = 0 rows = 0 for tv in DBSession.query(Taggable).filter(Taggable.resource_value != None).yield_per(10000): try: tv.resource_number = float(tv.resource_value) converted_numbers += 1 #print >>sys.stderr, ",n " except ValueError: try: service, clname, ida, rest = parse_bisque_uri(tv.resource_value) if ida and is_uniq_code (ida): #log.debug("loading resource_uniq %s" % ida) resource_link = DBSession.query(Taggable).filter_by(resource_uniq = ida).first() if resource_link: tv.resource_obj = resource_link.id #print >>sys.stderr, ",l:", resource_link.id converted_objs += 1 except IndexError: pass rows += 1 if rows % 1000 == 0: DBSession.commit() print >>sys.stderr, ".", print "converted %s numbers and %s objects" % (converted_numbers, converted_objs) DBSession.commit() DBSession.close()
def get_one(self, ident, **kw): "Fetch a blob based on uniq ID" log.info("get_one(%s) %s", ident, kw) try: if not is_uniq_code(ident): abort(404, "Must be resource unique code") resource = data_service.resource_load(uniq=ident) if resource is None: abort(403, "resource does not exist or permission denied") filename, _ = split_subpath(resource.get('name', str(ident))) blb = self.localpath(ident) if blb.files and len(blb.files) > 1: return export_service.export(files=[resource.get('uri')], filename=filename) localpath = os.path.normpath(blb.path) if 'localpath' in kw: tg.response.headers['Content-Type'] = 'text/xml' resource = etree.Element('resource', name=filename, value=localpath) return etree.tostring(resource) disposition = '' if 'noattach' in kw else 'attachment; ' try: disposition = '%sfilename="%s"' % (disposition, filename.encode('ascii')) except UnicodeEncodeError: disposition = '%sfilename="%s"; filename*="%s"' % ( disposition, filename.encode('utf8'), filename.encode('utf8')) content_type = self.guess_mime(filename) return forward( BQFileApp( localpath, content_type=content_type, content_disposition=disposition, ).cache_control(max_age=60 * 60 * 24 * 7 * 6)) # 6 weeks except IllegalOperation: abort(404, "Error occurent fetching blob")
def _read_pipeline(self, pipeline_url): """ read workflow json doc """ pipeline_path = urlparse.urlsplit(pipeline_url).path.split('/') pipeline_uid = pipeline_path[1] if is_uniq_code( pipeline_path[1]) else pipeline_path[2] url = self.bqSession.service_url('pipeline', path=pipeline_uid, query={'format': 'json'}) pipeline = self.bqSession.c.fetch(url) if not pipeline: # bad pipeline return None, None try: res_pipeline = json.loads(pipeline) except ValueError: # not json return None, None return res_pipeline
def _add_resource_inputs_outputs(xnode, edges, checked, unchecked): """ For the given xnode, find all other nodes that are connected to it by direct edges. For MEX type, input is all links in "inputs" section, output is all links in "outputs" section. For other types, input is all MEXs with it in "outputs" section, output is all MEXs with it in "inputs" section. Inputs: any ref in top "inputs" section without self-references Outputs: any ref in top "outputs" section without self-references or input references """ node = xnode.get('resource_uniq') if xnode.tag == 'mex': points_from_list = [ x.rsplit('/', 1)[1] for x in xnode.xpath('./tag[@name="inputs"]/tag/@value') if x.startswith("http") ] points_to_list = [ x.rsplit('/', 1)[1] for x in xnode.xpath('./tag[@name="outputs"]/tag/@value') if x.startswith("http") ] else: points_from_list = [] points_to_list = [] # TODO: the following will be very slow on large DBs... change to new query in 0.6! mexes_ref_node = data_service.query('mex', tag_query='"http*/%s"' % node, cache=False) for mex_ref_node in mexes_ref_node: mex_deep = data_service.resource_load( uniq=mex_ref_node.get('resource_uniq'), view='full') if mex_deep: found_in_inputs = False inputs_tag = mex_deep.xpath('./tag[@name="inputs"]') if inputs_tag: input_id = inputs_tag[0].get('uri') input_deep = data_service.get_resource(resource=input_id, view='full,clean') if input_deep and len( input_deep.xpath( './tag[@value="%s"]' % xnode.get("uri"))) > 0: # found node in MEX's inputs points_to_list.append( mex_ref_node.get('resource_uniq')) found_in_inputs = True if not found_in_inputs: outputs_tag = mex_deep.xpath('./tag[@name="outputs"]') if outputs_tag: output_id = outputs_tag[0].get('uri') output_deep = data_service.get_resource( resource=output_id, view='full,clean') if output_deep and len( output_deep.xpath('./tag[@value="%s"]' % xnode.get("uri"))) > 0: # found node in MEX's outputs points_from_list.append( mex_ref_node.get('resource_uniq')) # add edge unless it points to mex recursively points_from_list = [ x for x in points_from_list if is_uniq_code(x) and x != node ] # add edge unless it points to mex recursively or back to an input points_to_list = [ x for x in points_to_list if is_uniq_code(x) and x != node and x not in points_from_list ] log.debug("points_to_list %s", points_to_list) log.debug("points_from_list %s", points_from_list) for xlink in points_from_list: if (xlink, node) not in edges: log.debug("ADDING IN EDGE : %s" % str((xlink, node))) edges.add((xlink, node)) if xlink not in checked: unchecked.add(xlink) for xlink in points_to_list: if (node, xlink) not in edges: log.debug("ADDING OUT EDGE : %s" % str((node, xlink))) edges.add((node, xlink)) if xlink not in checked: unchecked.add(xlink)
def _default(self, *path, **kw): if is_uniq_code(path[0]): redirect("/client_service/view?resource=/data_service/%s" % path[0])