def buildGraph(request, webargs): """Build a graph based on different arguments""" try: # TODO UA use regex here verus replace. Check ndwsrest for more details. args = (webargs.replace(',', '/').split('/'))[0:-1] w_token = args[0] # TODO UA this will be replaced by NDToken and authentication if not request.user.is_superuser: m_tokens = Token.objects.filter( user=request.user.id) | Token.objects.filter(public=1) tokens = [] for v in m_tokens.values(): tokens.append(v['token_name']) if w_token not in tokens: logger.error( "Token {} does not exist or you do not have sufficient permissions to access it." .format(w_token)) raise NDWSError( "Token {} does not exist or you do not have sufficient permissions to access it." .format(w_token)) # TODO UA where are you closing this file. we need to ensure that this file is closed before it is returned. (file, filename) = ndgraph.genGraphRAMON(*args) response = HttpResponse(content_type='text/plain') response[ 'Content-Disposition'] = "attachment; filename=\"output.{}\"".format( filename) response.write(file.read()) return response except Exception as e: logger.error(e) raise NDWSError(e)
def validateConfig(self, config_data): try: ndcg = Configuration(config_data) validator = ndcg.get_validator() validator.schema = ndcg.schema validator.validate_schema() ingest_job_json = json.dumps({ 'dataset': ndcg.config_data["database"]["dataset"], 'project': ndcg.config_data["database"]["project"], 'channel': ndcg.config_data["database"]["channel"], 'resolution': ndcg.config_data["ingest_job"]["resolution"], 'x_start': ndcg.config_data["ingest_job"]["extent"]["x"][0], 'x_stop': ndcg.config_data["ingest_job"]["extent"]["x"][1], 'y_start': ndcg.config_data["ingest_job"]["extent"]["y"][0], 'y_stop': ndcg.config_data["ingest_job"]["extent"]["y"][1], 'z_start': ndcg.config_data["ingest_job"]["extent"]["z"][0], 'z_stop': ndcg.config_data["ingest_job"]["extent"]["z"][1], 't_start': ndcg.config_data["ingest_job"]["extent"]["t"][0], 't_stop': ndcg.config_data["ingest_job"]["extent"]["t"][1], 'tile_size_x': ndcg.config_data["ingest_job"]["tile_size"]["x"], 'tile_size_y': ndcg.config_data["ingest_job"]["tile_size"]["y"], 'tile_size_z': ndcg.config_data["ingest_job"]["tile_size"]["z"], 'tile_size_t': ndcg.config_data["ingest_job"]["tile_size"]["t"], }) self.ingest_job = NDIngestJob.fromJson(ingest_job_json) self.nd_proj = NDIngestProj(self.ingest_job.project, self.ingest_job.channel, self.ingest_job.resolution) except jsonschema.ValidationError as e: raise NDWSError("Schema validation failed") except Exception as e: return NDWSError("Properties not found") return True
def setField ( self, field, value ): """Mutator by field name. Then need to store the field.""" if field == 'segmentclass': self.segmentclass = value elif field == 'parentseed': self.parentseed = value elif field == 'neuron': self.neuron = value elif field == 'synapses': raise NDWSError ("Cannot set synapses in segments. It is derived from the synapse annotations.") elif field == 'organelles': raise NDWSError ("Cannot set organelles in segments. It is derived from the organelle annotations.") else: Annotation.setField ( self, field, value )
def getAnnoIds(proj, ch, resolution, xmin, xmax, ymin, ymax, zmin, zmax): """Return a list of anno ids restricted by equality predicates. Equalities are alternating in field/value in the url.""" mins = (xmin, ymin, zmin) maxs = (xmax, ymax, zmax) offset = proj.datasetcfg.get_offset(resolution) # Add a comment corner = map(max, zip(*[mins, map(sub, mins, offset)])) dim = map(sub, maxs, mins) if not proj.datasetcfg.checkCube(resolution, corner, dim): logger.error("Illegal cutout corner={}, dim={}".format(corner, dim)) raise NDWSError("Illegal cutout corner={}, dim={}".format(corner, dim)) with closing(SpatialDB(proj)) as sdb: cutout = sdb.cutout(ch, corner, dim, resolution) if cutout.isNotZeros(): annoids = np.unique(cutout.data) else: annoids = np.asarray([0], dtype=np.uint32) if annoids[0] == 0: return annoids[1:] else: return annoids
def newNDProject(self): """Create the database for a project""" with closing( MySQLdb.connect( host=self.pr.host, user=settings.DATABASES['default']['USER'], passwd=settings.DATABASES['default']['PASSWORD'], db=settings.DATABASES['default']['NAME'], connect_timeout=1)) as conn: with closing(conn.cursor()) as cursor: try: # create the database sql = "CREATE DATABASE {}".format(self.pr.dbname) cursor.execute(sql) conn.commit() except MySQLdb.Error, e: logger.error( "Failed to create database for new project {}: {}.". format(e.args[0], e.args[1])) raise NDWSError( "Failed to create database for new project {}: {}.". format(e.args[0], e.args[1]))
def deleteNDProject(self): """Delete the database for a project""" try: with closing( MySQLdb.connect( host=self.pr.host, user=settings.DATABASES['default']['USER'], passwd=settings.DATABASES['default']['PASSWORD'], connect_timeout=1)) as conn: with closing(conn.cursor()) as cursor: # delete the database sql = "DROP DATABASE {}".format(self.pr.dbname) try: cursor.execute(sql) conn.commit() except MySQLdb.Error, e: # Skipping the error if the database does not exist if e.args[0] == 1008: logger.warning("Database {} does not exist".format( self.pr.dbname)) pass else: conn.rollback() logger.error( "Failed to drop project database {}: {}. sql={}" .format(e.args[0], e.args[1], sql)) raise NDWSError( "Failed to drop project database {}: {}. sql={}" .format(e.args[0], e.args[1], sql)) except MySQLdb.OperationalError as e: logger.warning( "Cannot connect to the server at host {}. {}".format( self.pr.host, e))
def updateNDChannelNew(self, channel_name): """Create the tables for a channel""" ch = NDChannel.fromName(self.pr, channel_name) # connect to the database with closing( MySQLdb.connect( host=self.pr.host, user=settings.DATABASES['default']['USER'], passwd=settings.DATABASES['default']['PASSWORD'], db=self.pr.dbname, connect_timeout=1)) as conn: with closing(conn.cursor()) as cursor: try: # tables specific to all other non time data for res in self.pr.datasetcfg.resolutions: cursor.execute( "CREATE TABLE {} ( zindex BIGINT, timestamp INT, cube LONGBLOB, PRIMARY KEY(zindex,timestamp))" .format(ch.getNearIsoTable(res))) # Commiting at the end conn.commit() except MySQLdb.Error, e: logging.error( "Failed to create neariso tables for existing project {}: {}." .format(e.args[0], e.args[1])) raise NDWSError( "Failed to create neariso tables for existing project {}: {}." .format(e.args[0], e.args[1]))
def createIngestJob(self, user_id, config_data): """Create an ingest job based on the posted config data""" config_data = json.loads(config_data) # validate schema if self.validateConfig(config_data): try: # create the upload queue UploadQueue.createQueue( self.nd_proj, endpoint_url=ndingest_settings.SQS_ENDPOINT) self.upload_queue = UploadQueue( self.nd_proj, endpoint_url=ndingest_settings.SQS_ENDPOINT) self.ingest_job.upload_queue = self.upload_queue.url # create the ingest queue IngestQueue.createQueue( self.nd_proj, endpoint_url=ndingest_settings.SQS_ENDPOINT) self.ingest_job.ingest_queue = IngestQueue( self.nd_proj, endpoint_url=ndingest_settings.SQS_ENDPOINT).url # create the cleanup queue CleanupQueue.createQueue( self.nd_proj, endpoint_url=ndingest_settings.SQS_ENDPOINT) self.ingest_job.cleanup_queue = CleanupQueue( self.nd_proj, endpoint_url=ndingest_settings.SQS_ENDPOINT).url self.generateUploadTasks() self.ingest_job.user_id = user_id self.ingest_job.save() return NDIngestJob.serialize(self.ingest_job._job) except Exception as e: print(e) raise NDWSError(e)
def setField ( self, field, value ): """Mutator by field name. Then need to store the field.""" if field == 'segments': raise NDWSError ("Cannot set segments. It is derived from the neuron field of ANNO_SEGMENTS.") else: Annotation.setField ( self, field, value )
def putAnnotation ( self, ch, anno, options='' ): """store an HDF5 annotation to the database""" self.annodb.startTxn() try: # for updates, make sure the annotation exists and is of the right type if 'update' in options: kvdict = self.annodb.getAnnotationKV ( ch, anno.annid ) # can't update annotations that don't exist if kvdict == None: raise NDWSError ( "During update no annotation found at id {}".format(anno.annid) ) else: self.annodb.putAnnotationKV ( ch, anno.annid, anno.toDict(), update=True) # Write the user chosen annotation id else: kvdict = anno.toDict() self.annodb.putAnnotationKV ( ch, anno.annid, kvdict) self.annodb.commit() except Exception, e: self.annodb.rollback() raise
def AnnotationtoH5(anno, h5fh): """Operate polymorphically on annotations""" if anno.__class__ == annotation.AnnSynapse: return SynapsetoH5(anno, h5fh) elif anno.__class__ == annotation.AnnSeed: return SeedtoH5(anno, h5fh) if anno.__class__ == annotation.AnnSegment: return SegmenttoH5(anno, h5fh) if anno.__class__ == annotation.AnnNeuron: return NeurontoH5(anno, h5fh) if anno.__class__ == annotation.AnnOrganelle: return OrganelletoH5(anno, h5fh) if anno.__class__ == annotation.AnnNode: return NodetoH5(anno, h5fh) if anno.__class__ == annotation.AnnSkeleton: return SkeletontoH5(anno, h5fh) elif anno.__class__ == annotation.Annotation: return BasetoH5(anno, annotation.ANNO_ANNOTATION, h5fh) else: logger.warning( "(AnnotationtoH5) Does not support this annotation type yet. Type = %s" % anno.__class__) raise NDWSError( "(AnnotationtoH5) Does not support this annotation type yet. Type = %s" % anno.__class__)
def getAnnotation ( self, ch, annid ): """Return a RAMON object by identifier""" kvdict = self.annodb.getAnnotationKV ( ch, annid ) annotype = int(kvdict['ann_type']) # switch on the type of annotation if annotype is None: return None elif annotype == annotation.ANNO_SYNAPSE: anno = annotation.AnnSynapse(self.annodb,ch) elif annotype == annotation.ANNO_SEED: anno = annotation.AnnSeed(self.annodb,ch) elif annotype == annotation.ANNO_SEGMENT: anno = annotation.AnnSegment(self.annodb,ch) elif annotype == annotation.ANNO_NEURON: anno = annotation.AnnNeuron(self.annodb,ch) elif annotype == annotation.ANNO_ORGANELLE: anno = annotation.AnnOrganelle(self.annodb,ch) elif annotype == annotation.ANNO_NODE: anno = annotation.AnnNode(self.annodb,ch) elif annotype == annotation.ANNO_SKELETON: anno = annotation.AnnSkeleton(self.annodb,ch) elif annotype == annotation.ANNO_ROI: anno = annotation.AnnROI(self.annodb,ch) elif annotype == annotation.ANNO_ANNOTATION: anno = annotation.Annotation(self.annodb,ch) else: raise NDWSError ( "Unrecognized annotation type {}".format(type) ) # load the annotation anno.fromDict ( kvdict ) return anno
def queryNIFTI ( tmpfile, ch, db, proj ): """ Return a NII file that contains the entire DB""" try: # get the header in a fileobj nh = NDNiftiHeader.fromChannel(ch) cuboid = db.cutout ( ch, (0,0,0), proj.datasetcfg.dataset_dim(0), 0, timerange=ch.time_range) # transpose to nii's xyz format niidata = cuboid.data.transpose() # for 3-channel FA if niidata.dtype == np.uint32: niidata = _RGBto3dby8 ( niidata[:,:,:,0] ) # assemble the header and the data and create a nii file nii = nibabel.Nifti1Image(niidata, affine=nh.affine, header=nh.header ) # this adds a suffix and save to the tmpfile nibabel.save ( nii, tmpfile.name ) except Exception as e: logger.error("Failed to build nii file. Error {}".format(e)) raise NDWSError("Failed to build nii file. Error {}".format(e))
def setReadOnly (self, value): if value in [READONLY_TRUE, READONLY_FALSE]: self.ch.readonly = value self.ch.save() else: logger.error ( "Wrong Readonly Value {} for Channel {}".format( value, self.channel_name ) ) raise NDWSError ( "Wrong Readonly Value {} for Channel {}".format( value, self.ch.channel_name ) )
def deleteNDChannel(self, channel_name): """Delete the keys for a channel""" # KL TODO Maybe do this as a transaction or lock the cache when we drop the channel? try: # removing keys related to kvio data # channel pattern to fetch all the keys with project_name&channel_name& channel_pattern = "{}&{}&*".format(self.pr.project_name, channel_name) channel_keys = self.client.keys(channel_pattern) # delete all the keys with the pattern if channel_keys: self.client.delete(*channel_keys) # removing keys related to kvindex data index_min_pattern = "[{}&{}".format(self.pr.project_name, channel_name) index_max_pattern = "+" self.client.zremrangebylex(settings.REDIS_INDEX_KEY, index_min_pattern, index_max_pattern) # deleteting from s3 and dynamo self.s3_proj.deleteNDChannel(channel_name) except Exception as e: logger.error("Error in deleting channel {}. {}".format( channel_name, e)) raise NDWSError("Error in deleting channel {}. {}".format( channel_name, e))
def deleteNDResolution(self, channel_name, resolution): """Delete the resolution for a channel""" # KL TODO Maybe do this as a transaction or lock the cache when we drop the channel? try: # removing keys for kvio # resolution pattern to fetch all keys with project_name&channel_name&resolution& resolution_pattern = "{}&{}&{}&*".format(self.pr.project_name, channel_name, resolution) resolution_keys = self.client.keys(channel_pattern) # delete all the keys with pattern if resolution_keys: self.client.delete(*resolution_keys) # removing keys for kvindex data index_min_pattern = "[{}&{}&{}".format(self.pr.project_name, channel_name, resolution) index_max_pattern = "+" self.client.zremrangebylex(settings.REDIS_INDEX_KEY, index_min_pattern, index_max_pattern) # deleteting from s3 and dynamo self.s3_proj.deleteNDResolution(channel_name, resolution) except Exception as e: logger.error( "Error in deleting resolution {} channel {}. {}".format( resolution, channel_name, e)) raise NDWSError( "Error in deleting resolution {} channel {}. {}".format( resolution, channel_name, e))
def getRamonTable(self): """Return the name of the ramon table""" if self.pr.kvengine == MYSQL: return "{}_ramon".format(self.ch.channel_name) else: logger.error("RAMON not support for KV Engine {}".format(self.pr.kvengine)) raise NDWSError("RAMON not support for KV Engine {}".format(self.pr.kvengine))
def deleteNDChannel(self, channel_name): """Delete the tables for this channel""" ch = NDChannel.fromName(self.pr, channel_name) table_list = [] if ch.channel_type in ANNOTATION_CHANNELS: # delete the ids table table_list.append(ch.getIdsTable()) # delete the ramon table table_list.append(ch.getRamonTable()) for res in self.pr.datasetcfg.resolutions: # delete the res tables table_list.append(ch.getTable(res)) if self.pr.datasetcfg.scalingoption == ZSLICES: table_list.append(ch.getNearIsoTable(res)) # delete the index tables table_list.append(ch.getS3IndexTable(res)) if ch.channel_type in ANNOTATION_CHANNELS: # delete the exceptions tables table_list = table_list + [ ch.getIdxTable(res), ch.getExceptionsTable(res) ] try: with closing( MySQLdb.connect( host=self.pr.host, user=settings.DATABASES['default']['USER'], passwd=settings.DATABASES['default']['PASSWORD'], db=self.pr.dbname, connect_timeout=1)) as conn: with closing(conn.cursor()) as cursor: # delete the tables for this channel sql = "DROP TABLES IF EXISTS {}".format( ','.join(table_list)) try: cursor.execute(sql) conn.commit() except MySQLdb.Error, e: # Skipping the error if the table does not exist if e.args[0] == 1051: pass if e.args[0] == 1049: pass else: conn.rollback() logger.error( "Failed to drop channel tables {}: {}. sql={}". format(e.args[0], e.args[1], sql)) raise NDWSError( "Failed to drop channel tables {}: {}. sql={}". format(e.args[0], e.args[1], sql)) except Exception as e: logger.warning("Database {} on host {} not found".format( self.pr.dbname, self.pr.host))
def propagate (token, channel_name): """Propagate the given project for all resolutions""" try: buildStack(token, channel_name) except Exception, e: logger.error("Error in propagate. {}".format(e)) raise NDWSError("Error in propagate. {}".format(e))
def __init__(self, pr): """Create the database connection""" self.pr = pr # Connect to the redis cluster try: self.client = redis.StrictRedis(host=self.pr.host, port=6379, db=0) self.pipe = self.client.pipeline(transaction=False) # delete from S3 when deleting from Redis self.s3_proj = S3ProjectDB(pr) except redis.ConnectionError as e: logger.error("Cannot connect to Redis server. {}".format(e)) raise NDWSError("Cannot connect to Redis server. {}".format(e)) except Exception as e: logger.error( "Unknown error while connecting to Redis. {}".format(e)) raise NDWSError( "Unknown error while connecting to Redis. {}".format(e))
def ingest (token_name, channel_name, resolution, data_url, file_format, file_type): """Call the remote ingest function here""" try: ingest_data = IngestData(token_name, channel_name, resolution, data_url, file_format, file_type) ingest_data.ingest() except Exception, e: logger.error("Error in ingest. {}".format(e)) raise NDWSError("Error in ingest. {}".format(e))
def fromTokenName(cls, token_name): try: tk = Token.objects.get(token_name=token_name) pr = Project.objects.get(project_name=tk.project_id) return cls(pr) except ObjectDoesNotExist, e: logger.error("Token {} does not exist. {}".format(token_name, e)) raise NDWSError("Token {} does not exist. {}".format( token_name, e))
def setField ( self, field, value ): """Mutator by field name. Then need to store the field.""" if field == 'parent': self.parent = value elif field == 'children': raise NDWSError ("Cannot set children. It is derived from the parent field of ANNO_NODE.") else: Annotation.setField ( self, field, value )
def deleteChannel(webargs, post_data): """Delete a list of channels using a JSON file""" # Get the token and load the project try: m = re.match("(\w+)/deleteChannel/$", webargs) token_name = m.group(1) except Exception, e: logger.error("Error in URL format") raise NDWSError("Error in URL format")
def setField ( self, field, value ): """Mutator by field name. Then need to store the field.""" if field == 'nodetype': self.nodetype = value elif field == 'children': raise NDWSError ("Cannot set children. It is derived from the parent field of ANNO_NODE.") elif field == 'location': self.location = np.array([float(x) for x in value.split(',')], dtype=np.float) if len(self.location) != 3: raise NDWSError ("Illegal arguments to set field location: %s" % value) elif field == 'parent': self.parent = value elif field == 'radius': self.radius = value elif field == 'skeleton': self.skeleton = value else: Annotation.setField ( self, field, value )
def querySWC(swcfile, ch, db, proj, skelids=None): """Query the list of skelids (skeletons) and populate an open file swcfile with lines of swc data.""" db.startTxn() try: # write out metadata about where this came from # ND version number and schema number swcfile.write('# ND (NeuroData) Version {} Schema {}\n'.format( proj.getNDVersion(), proj.getSchemaVersion())) # ND project and channel swcfile.write('# Project {} Channel {}\n'.format( proj.getProjectName(), ch.getChannelName())) # get a skeleton for metadata and populate the comments field if skelids != None: skel = db.getAnnotation(ch, skelids[0]) # write each key value line out as a comment for (k, v) in skel.toDict().iteritems(): # match a comment if re.match("^#.*\n", str(v)): swcfile.write(v) else: swcfile.write("# {} {}\n".format(k, v)) # iterate over all nodes for skel in skelids: for nodeid in db.querySkeletonNodes(ch, skel): node = db.getAnnotation(ch, nodeid) #RB nodes are floating point values. let's not scale. # # scale points to resolution # xpos = xpos/(2**res) # ypos = ypos/(2**res) # # check for isotropic # if db.datasetcfg.scalingoption == ISOTROPIC: # zpos = zpos/(2**res) # write an node in swc # n T x y z R P swcfile.write("{} {} {} {} {} {} {}\n".format( node.annid, node.nodetype, node.location[0], node.location[1], node.location[2], node.radius, node.parent)) db.commit() except Exception, e: db.rollback() logger.warning("Failed to get SWC file {}".format(e)) raise NDWSError("Failed to get SWC file {}".format(e))
def ingest(self): """Identify the data style and ingest accordingly""" if self.file_format in ['SLICE']: self.ingestImageStack() elif self.file_format in ['CATMAID']: self.ingestCatmaidStack() else: logger.error("Format {} not supported.".format(self.file_format)) raise NDWSError("Format {} not supported.".format( self.file_format))
def setField ( self, field, value ): """Mutator by field name. Then need to store the field.""" if field == 'skeletontype': self.skeletontype = value elif field == 'skeletonnodes': raise NDWSError ("Cannot set nodes. It is derived from the parent field of ANNO_NODE.") elif field == 'rootnode': self.rootnode = value else: Annotation.setField ( self, field, value )
def extractProjectDict(pr_dict): """Generate a project object from the JSON flle""" pr = {} tk = {} try: pr['project_name'] = pr_dict['project_name'] except Exception, e: logger.error("Missing required fields") raise NDWSError("Missing required fields")
def setPropagate (self, value): if value in [NOT_PROPAGATED, PROPAGATED]: self.ch.propagate = value self.setReadOnly ( READONLY_FALSE ) self.ch.save() elif value in [UNDER_PROPAGATION]: self.ch.propagate = value self.setReadOnly ( READONLY_TRUE ) self.ch.save() else: logger.error ( "Wrong Propagate Value {} for Channel {}".format( value, self.ch.channel_name ) ) raise NDWSError ( "Wrong Propagate Value {} for Channel {}".format( value, self.ch.channel_name ) )