def scale_convert(selected_files, dl_format, ds_factor, ATLASES, email=None, dwnld_loc=None, zip_fn=None): # Debug print "Entering scale function ..." try: if dl_format == "graphml" and ds_factor == 0: temp = zipfiles(selected_files, use_genus=True, zip_out_fn=zip_fn) else: files_to_zip = {} for fn in selected_files: # No matter what we need a temp file print "Creating temp file ..." tmpfile = tempfile.NamedTemporaryFile("w", delete=False, dir="/data/pytmp") print "Temp file %s created ..." % tmpfile.name tmpfile.close() # Downsample only valid for *BIG* human brains! # *NOTE: If smallgraphs are ingested this will break if ds_factor and get_genus(fn) == "human": if isinstance(ds_factor, int): print "downsampling to factor %d" % ds_factor g = downsample(igraph_io.read_arbitrary(fn, "graphml"), ds_factor) print "downsample complete" else: g = downsample(igraph_io.read_arbitrary(fn, "graphml"), atlas=nib_load(ATLASES[ds_factor])) else: g = igraph_io.read_arbitrary(fn, "graphml") # Write to `some` format if dl_format == "mm": igraph_io.write_mm(g, tmpfile.name) else: g.write(tmpfile.name, format=dl_format) files_to_zip[fn] = tmpfile.name temp = zipfiles(files_to_zip, use_genus=True, zip_out_fn=zip_fn, gformat=dl_format) # Del temp files for tmpfn in files_to_zip.values(): print "Deleting %s ..." % tmpfn os.remove(tmpfn) except Exception, msg: print "An exception was thrown and caught with message %s!" % msg if email: msg = """ Hello,\n\nYour most recent job failed to complete. \nYou may have some partially completed data at {}.\n\n """ sendJobFailureEmail(email, msg, dwnld_loc) return else: return 'An error occurred while processing your request. Please send an email to \
def _ingest_files(fns, genus, tb_name): print "Connecting to database %s ..." % db_args["default"]["NAME"] db = MySQLdb.connect(host=db_args["default"]["HOST"], user=db_args["default"]["USER"], passwd=db_args["default"]["PASSWORD"], db=db_args["default"]["NAME"]) db.autocommit(True) with closing(db.cursor()) as cursor: cursor.connection.autocommit(True) for graph_fn in fns: print "Processing %s ..." % graph_fn mtime = os.stat(graph_fn).st_mtime # get modification time g_changed = True # In DB and modified test_qry ="select g.mtime from %s.%s as g where g.filepath = \"%s\";" % (db_args["default"]["NAME"], tb_name, graph_fn) if cursor.execute(test_qry): # Means graph already in DB if cursor.fetchall()[0][0] == os.stat(graph_fn).st_mtime: # Means graphs hasn't changed since ingest g_changed = False print "Ignoring %s ..." % graph_fn else: cursor.execute("delete from %s.%s where filepath = \"%s\";" % (db_args["default"]["NAME"], tb_name, graph_fn)) print " ===> Updating %s ..." % graph_fn if g_changed: # Means graph has changed since ingest OR was never in DB to start with # Collect all the attributes etc .. g = igraph_io.read_arbitrary(graph_fn, informat="graphml", headers_only=True) vertex_attrs = g.vs.attribute_names() edge_attrs = g.es.attribute_names() graph_attrs = g.attributes() vcount = g.vcount() ecount = g.ecount() # Give some default values if none exist if "sensor" in graph_attrs: sensor = g["sensor"] else: sensor = "" if "source" in graph_attrs: source = g["source"] else: source = "" if "region" in graph_attrs: region = g["region"] else: region = "" if "project" in graph_attrs: project = g["project"] else: project = "" #url = "http://openconnecto.me/mrdata/graphs/"+("/".join(graph_fn.replace("\\", "/").split('/')[-2:])) url = get_download_path(graph_fn) # This statement puts each graph into the DB qry_stmt = "insert into %s.%s values (\"%s\",\"%s\",\"%s\",\"%s\",%d,%d,\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",%f,\"%s\");" \ % (db_args["default"]["NAME"], tb_name, os.path.abspath(graph_fn), genus, region, project, np.int64(np.float64(vcount)), np.int64(np.float64(ecount)), str(graph_attrs)[1:-1].replace("'",""), str(vertex_attrs)[1:-1].replace("'",""), str(edge_attrs)[1:-1].replace("'",""), sensor, source, mtime, url) cursor.execute(qry_stmt)
def main(): parser = argparse.ArgumentParser(description="") parser.add_argument("infn", action="store", help="Input file name") parser.add_argument("-f", "--factor", action="store", type=int, help="Downsampling factor") parser.add_argument("-a", "--ds_atlas", action="store", help="Pre-Downsampled atlas file name") parser.add_argument("outfn", action="store", help="Output file name") parser.add_argument("--informat", "-i", action="store", default="graphml", help="Input format of the graph") parser.add_argument("--outformat", "-o", action="store", default="graphml", help="Output format of the graph") result = parser.parse_args() if result.factor >= 0: g = igraph_io.read_arbitrary(result.infn, informat=result.informat) new_graph = downsample(g, factor=result.factor) elif result.ds_atlas: g = igraph_io.read_arbitrary(result.infn, informat=result.informat) new_graph = downsample(g, ds_atlas=nib.load(result.ds_atlas)) else: sys.stderr.write("[ERROR]: either -f or -a flag must be specified\n") exit(-1) new_graph.write(result.outfn, format=result.outformat)
def _ingest_files(fns, genus, tb_name): print "Connecting to database %s ..." % db_args["default"]["NAME"] db = MySQLdb.connect(host=db_args["default"]["HOST"], user=db_args["default"]["USER"], passwd=db_args["default"]["PASSWORD"], db=db_args["default"]["NAME"]) db.autocommit(True) with closing(db.cursor()) as cursor: cursor.connection.autocommit(True) for graph_fn in fns: print "Processing %s ..." % graph_fn mtime = os.stat(graph_fn).st_mtime # get modification time g_changed = True # In DB and modified test_qry = "select g.mtime from %s.%s as g where g.filepath = \"%s\";" % ( db_args["default"]["NAME"], tb_name, graph_fn) if cursor.execute(test_qry): # Means graph already in DB if cursor.fetchall()[0][0] == os.stat( graph_fn ).st_mtime: # Means graphs hasn't changed since ingest g_changed = False print "Ignoring %s ..." % graph_fn else: cursor.execute( "delete from %s.%s where filepath = \"%s\";" % (db_args["default"]["NAME"], tb_name, graph_fn)) print " ===> Updating %s ..." % graph_fn if g_changed: # Means graph has changed since ingest OR was never in DB to start with # Collect all the attributes etc .. g = igraph_io.read_arbitrary(graph_fn, informat="graphml", headers_only=True) vertex_attrs = g.vs.attribute_names() edge_attrs = g.es.attribute_names() graph_attrs = g.attributes() vcount = g.vcount() ecount = g.ecount() # Give some default values if none exist if "sensor" in graph_attrs: sensor = g["sensor"] else: sensor = "" if "source" in graph_attrs: source = g["source"] else: source = "" if "region" in graph_attrs: region = g["region"] else: region = "" if "project" in graph_attrs: project = g["project"] else: project = "" #url = "http://openconnecto.me/mrdata/graphs/"+("/".join(graph_fn.replace("\\", "/").split('/')[-2:])) url = get_download_path(graph_fn) # This statement puts each graph into the DB qry_stmt = "insert into %s.%s values (\"%s\",\"%s\",\"%s\",\"%s\",%d,%d,\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",%f,\"%s\");" \ % (db_args["default"]["NAME"], tb_name, os.path.abspath(graph_fn), genus, region, project, np.int64(np.float64(vcount)), np.int64(np.float64(ecount)), str(graph_attrs)[1:-1].replace("'",""), str(vertex_attrs)[1:-1].replace("'",""), str(edge_attrs)[1:-1].replace("'",""), sensor, source, mtime, url) cursor.execute(qry_stmt)