def _instantiate_pipeline(self, pipeline_url, params):
"""
instantiate dream.3d pipeline file with provided parameters
"""
pipeline_path = urlparse.urlsplit(pipeline_url).path.split('/')
pipeline_uid = pipeline_path[1] if is_uniq_code(
pipeline_path[1]) else pipeline_path[2]
url = self.bqSession.service_url(
'pipeline',
path='/'.join(
[pipeline_uid] +
["setvar:%s|%s" % (tag, params[tag])
for tag in params] + ['exbsteps:dream3d']),
query={'format': 'dream3d'})
pipeline = self.bqSession.c.fetch(url)
if not pipeline:
# bad pipeline
return None, None
out_pipeline_file = os.path.join(self.options.stagingPath,
'pipeline.json')
out_error_file = os.path.join(self.options.stagingPath,
'dream3d_error.txt')
with open(out_pipeline_file, 'w') as fo:
fo.write(pipeline)
return out_pipeline_file, out_error_file
def mex_acl_handler(resource_uniq, user_uniq, newauth, action):
"""Special handling for mexes
Share A mexes input and outputs
"""
mex = data_service.resource_load(uniq=resource_uniq, view='deep')
# extract urls from mex inputs and outputs
points_from_list = [
x.rsplit('/', 1)[1]
for x in mex.xpath('./tag[@name="inputs"]/tag/@value')
if x.startswith("http")
]
points_to_list = [
x.rsplit('/', 1)[1]
for x in mex.xpath('./tag[@name="outputs"]/tag/@value')
if x.startswith("http")
]
mex_resources = [
x for x in points_from_list + points_to_list if is_uniq_code(x)
]
log.debug("Discovered incoming %s outgoing %s = %s", points_from_list,
points_to_list, mex_resources)
resource_acls(mex_resources, newauth, action=action)
def load_uri(uri, query=False):
'''Load the object specified by the root tree and return a rsource
@type root: Element
@param root: The db object root with a uri attribute
@rtype: tag_model.Taggable
@return: The resource loaded from the database
'''
# Check that we are looking at the right resource.
try:
resource = None
service, clname, ida, rest = parse_bisque_uri(uri)
if service not in ('data_service', 'module_service'):
return None
if ida and is_uniq_code(ida):
log.debug("loading resource_uniq %s", ida)
resource = DBSession.query(Taggable).filter_by(resource_uniq=ida)
elif clname:
name, dbcls = dbtype_from_tag(clname)
#log.debug("loading %s -> name/type (%s/%s)(%s) " , uri, name, str(dbcls), ida)
#resource = DBSession.query(dbcls).get (int (ida))
resource = DBSession.query(dbcls).filter(dbcls.id == int(ida))
if not query:
resource = resource.first()
#log.debug ("loaded %s" % resource)
return resource
except Exception:
log.exception("Failed to load uri %s", uri)
return None
def load(self, token, **kw):
log.debug("Load %s" % token)
try:
if is_uniq_code(token):
return resource_load(self.resource_type, uniq=token)
return resource_load(self.resource_type, ident=int(token))
except ValueError, e:
abort(404)
def load_token(token):
if is_uniq_code(token):
return resource_load(uniq=token).first()
try:
token = int(token)
return resource_load(ident=token).first()
except ValueError:
pass
return None
def _cache_ppops(self, pipeline_url):
if not self.ppops or self.ppops_url != pipeline_url:
pipeline_path = urlparse.urlsplit(pipeline_url).path.split('/')
pipeline_uid = pipeline_path[1] if is_uniq_code(
pipeline_path[1]) else pipeline_path[2]
url = self.bqSession.service_url('pipeline',
path='/'.join([pipeline_uid] +
['ppops:dream3d']))
self.ppops = json.loads(self.bqSession.c.fetch(url))
self.ppops_url = pipeline_url
def parse_bisque_uri(uri):
''' Parse a bisquie uri into service, and optional dbclass , and ID
@type uri: string
@param uri: a bisquik uri representation of a resourc
@rtype: A quad-tuple (service, dbclass, id, rest )
@return: The parse resource
'''
# (scheme, host, path, ...)
url = urlparse.urlsplit(uri)
if url.scheme not in ('http', 'https', ''):
return None, None, None, None
# /service_name/ [ id or or class ]*
parts = posixpath.normpath(url[2]).strip('/').split('/')
# paths are /class/id or /resource_uniq
if not parts:
return url[1], 'data_service', None, None
# should have a service name or a uniq code
if is_uniq_code(parts[0]):
service = 'data_service'
# class follows or nothing
else:
service = parts.pop(0)
# first element may be a docid
ida = None
if parts and is_uniq_code(parts[0]):
ida = parts.pop(0)
clname = None
rest = []
while parts:
sym = parts.pop()
if not sym[0].isdigit():
rest.append(sym)
continue
ida = sym
if parts:
clname = parts.pop()
break
return service, clname, ida, rest
def _lookup(self, service_type, *rest):
"""Default handler for bisque root.
Search for registered sub-servers that might handle this request.
return an error if none found
URL are formed with
<service_type>/<path>?arguments
"""
if request.path in NOLOGGING:
log.debug('[%s] %s %s', request.identity.get('repoze.who.userid'),
request.method, request.url)
else:
log.info('[%s] %s %s', request.identity.get('repoze.who.userid'),
request.method, request.url)
if log.isEnabledFor(logging.DEBUG):
log.debug('HEADERS: %s %s', request.url, dict(request.headers))
log.debug('PATH %s in DIR %s', request.path, os.getcwd())
if service_type in oldnames:
#service_type = oldnames[service_type]
newreq = request.path_qs[1:].replace(service_type,
oldnames[service_type])
log.warn('found oldname( %s ) -> newreq %s', service_type, newreq)
redirect(urlparse.urljoin(request.application_url, newreq))
#log.debug ("find controller for %s " % (str(service_type) ))
#log.debug ("lookup for %s/%s" , service_type, rest )
#import pdb
#pdb.set_trace()
service = service_registry.find_service(service_type)
if service is not None:
#log.debug ('found %s ' , str(service))
if service.service_type != 'auth_service' and 'expires' in session:
session_update()
return service, rest
if is_uniq_code(service_type):
log.debug("uniq code %s", request.path_qs)
# Skip 1st /
redirect(
urlparse.urljoin(request.application_url,
'/data_service/%s' % request.path_qs[1:]))
log.warn('no service found %s with %s', service_type, rest)
abort(404)
def _default(self, *path, **kw):
#log.debug ('headers:'+ str(cherrypy.request.headers))
path = list(path)
#log.debug ("path = %s %s " , path, kw)
token = path.pop(0)
if is_uniq_code(token):
log.debug('using uniq token')
resource_controller = self.get_child_resource('resource')
path.insert(0, token)
else:
handler = RESOURCE_HANDLERS.get(token, None)
# Allow override of path parsing by specific resource controller i.e. dataset etc.
if handler:
resource_controller = handler(self.url)
else:
resource_controller = self.get_child_resource(token)
return resource_controller._default(*path, **kw)
def upgrade():
return
op.add_column('taggable', Column('resource_number', Float))
op.add_column('taggable', Column('resource_obj', Integer, ForeignKey ('taggable.id')))
cntxt = context.get_context()
SessionMaker = sessionmaker(bind=cntxt.bind)
Base = declarative_base()
metadata = MetaData(bind=cntxt.bind)
class Taggable(Base):
__table__ = Table('taggable', metadata, autoload=True)
DBSession = SessionMaker()
converted_numbers = 0
converted_objs = 0
rows = 0
for tv in DBSession.query(Taggable).filter(Taggable.resource_value != None).yield_per(10000):
try:
tv.resource_number = float(tv.resource_value)
converted_numbers += 1
#print >>sys.stderr, ",n "
except ValueError:
try:
service, clname, ida, rest = parse_bisque_uri(tv.resource_value)
if ida and is_uniq_code (ida):
#log.debug("loading resource_uniq %s" % ida)
resource_link = DBSession.query(Taggable).filter_by(resource_uniq = ida).first()
if resource_link:
tv.resource_obj = resource_link.id
#print >>sys.stderr, ",l:", resource_link.id
converted_objs += 1
except IndexError:
pass
rows += 1
if rows % 1000 == 0:
DBSession.commit()
print >>sys.stderr, ".",
print "converted %s numbers and %s objects" % (converted_numbers, converted_objs)
DBSession.commit()
DBSession.close()
def get_one(self, ident, **kw):
"Fetch a blob based on uniq ID"
log.info("get_one(%s) %s", ident, kw)
try:
if not is_uniq_code(ident):
abort(404, "Must be resource unique code")
resource = data_service.resource_load(uniq=ident)
if resource is None:
abort(403, "resource does not exist or permission denied")
filename, _ = split_subpath(resource.get('name', str(ident)))
blb = self.localpath(ident)
if blb.files and len(blb.files) > 1:
return export_service.export(files=[resource.get('uri')],
filename=filename)
localpath = os.path.normpath(blb.path)
if 'localpath' in kw:
tg.response.headers['Content-Type'] = 'text/xml'
resource = etree.Element('resource',
name=filename,
value=localpath)
return etree.tostring(resource)
disposition = '' if 'noattach' in kw else 'attachment; '
try:
disposition = '%sfilename="%s"' % (disposition,
filename.encode('ascii'))
except UnicodeEncodeError:
disposition = '%sfilename="%s"; filename*="%s"' % (
disposition, filename.encode('utf8'),
filename.encode('utf8'))
content_type = self.guess_mime(filename)
return forward(
BQFileApp(
localpath,
content_type=content_type,
content_disposition=disposition,
).cache_control(max_age=60 * 60 * 24 * 7 * 6)) # 6 weeks
except IllegalOperation:
abort(404, "Error occurent fetching blob")
def _read_pipeline(self, pipeline_url):
"""
read workflow json doc
"""
pipeline_path = urlparse.urlsplit(pipeline_url).path.split('/')
pipeline_uid = pipeline_path[1] if is_uniq_code(
pipeline_path[1]) else pipeline_path[2]
url = self.bqSession.service_url('pipeline',
path=pipeline_uid,
query={'format': 'json'})
pipeline = self.bqSession.c.fetch(url)
if not pipeline:
# bad pipeline
return None, None
try:
res_pipeline = json.loads(pipeline)
except ValueError:
# not json
return None, None
return res_pipeline
def _add_resource_inputs_outputs(xnode, edges, checked, unchecked):
"""
For the given xnode, find all other nodes that are connected to it by direct edges.
For MEX type, input is all links in "inputs" section, output is all links in "outputs" section.
For other types, input is all MEXs with it in "outputs" section, output is all MEXs with it in "inputs" section.
Inputs: any ref in top "inputs" section without self-references
Outputs: any ref in top "outputs" section without self-references or input references
"""
node = xnode.get('resource_uniq')
if xnode.tag == 'mex':
points_from_list = [
x.rsplit('/', 1)[1]
for x in xnode.xpath('./tag[@name="inputs"]/tag/@value')
if x.startswith("http")
]
points_to_list = [
x.rsplit('/', 1)[1]
for x in xnode.xpath('./tag[@name="outputs"]/tag/@value')
if x.startswith("http")
]
else:
points_from_list = []
points_to_list = []
# TODO: the following will be very slow on large DBs... change to new query in 0.6!
mexes_ref_node = data_service.query('mex',
tag_query='"http*/%s"' % node,
cache=False)
for mex_ref_node in mexes_ref_node:
mex_deep = data_service.resource_load(
uniq=mex_ref_node.get('resource_uniq'), view='full')
if mex_deep:
found_in_inputs = False
inputs_tag = mex_deep.xpath('./tag[@name="inputs"]')
if inputs_tag:
input_id = inputs_tag[0].get('uri')
input_deep = data_service.get_resource(resource=input_id,
view='full,clean')
if input_deep and len(
input_deep.xpath(
'./tag[@value="%s"]' % xnode.get("uri"))) > 0:
# found node in MEX's inputs
points_to_list.append(
mex_ref_node.get('resource_uniq'))
found_in_inputs = True
if not found_in_inputs:
outputs_tag = mex_deep.xpath('./tag[@name="outputs"]')
if outputs_tag:
output_id = outputs_tag[0].get('uri')
output_deep = data_service.get_resource(
resource=output_id, view='full,clean')
if output_deep and len(
output_deep.xpath('./tag[@value="%s"]' %
xnode.get("uri"))) > 0:
# found node in MEX's outputs
points_from_list.append(
mex_ref_node.get('resource_uniq'))
# add edge unless it points to mex recursively
points_from_list = [
x for x in points_from_list if is_uniq_code(x) and x != node
]
# add edge unless it points to mex recursively or back to an input
points_to_list = [
x for x in points_to_list
if is_uniq_code(x) and x != node and x not in points_from_list
]
log.debug("points_to_list %s", points_to_list)
log.debug("points_from_list %s", points_from_list)
for xlink in points_from_list:
if (xlink, node) not in edges:
log.debug("ADDING IN EDGE : %s" % str((xlink, node)))
edges.add((xlink, node))
if xlink not in checked:
unchecked.add(xlink)
for xlink in points_to_list:
if (node, xlink) not in edges:
log.debug("ADDING OUT EDGE : %s" % str((node, xlink)))
edges.add((node, xlink))
if xlink not in checked:
unchecked.add(xlink)
def _default(self, *path, **kw):
if is_uniq_code(path[0]):
redirect("/client_service/view?resource=/data_service/%s" %
path[0])