def scale_convert(selected_files, dl_format, ds_factor, ATLASES, email=None, dwnld_loc=None, zip_fn=None):
# Debug
print "Entering scale function ..."
try:
if dl_format == "graphml" and ds_factor == 0:
temp = zipfiles(selected_files, use_genus=True, zip_out_fn=zip_fn)
else:
files_to_zip = {}
for fn in selected_files:
# No matter what we need a temp file
print "Creating temp file ..."
tmpfile = tempfile.NamedTemporaryFile("w", delete=False, dir="/data/pytmp")
print "Temp file %s created ..." % tmpfile.name
tmpfile.close()
# Downsample only valid for *BIG* human brains!
# *NOTE: If smallgraphs are ingested this will break
if ds_factor and get_genus(fn) == "human":
if isinstance(ds_factor, int):
print "downsampling to factor %d" % ds_factor
g = downsample(igraph_io.read_arbitrary(fn, "graphml"), ds_factor)
print "downsample complete"
else:
g = downsample(igraph_io.read_arbitrary(fn, "graphml"), atlas=nib_load(ATLASES[ds_factor]))
else:
g = igraph_io.read_arbitrary(fn, "graphml")
# Write to `some` format
if dl_format == "mm":
igraph_io.write_mm(g, tmpfile.name)
else:
g.write(tmpfile.name, format=dl_format)
files_to_zip[fn] = tmpfile.name
temp = zipfiles(files_to_zip, use_genus=True, zip_out_fn=zip_fn, gformat=dl_format)
# Del temp files
for tmpfn in files_to_zip.values():
print "Deleting %s ..." % tmpfn
os.remove(tmpfn)
except Exception, msg:
print "An exception was thrown and caught with message %s!" % msg
if email:
msg = """
Hello,\n\nYour most recent job failed to complete.
\nYou may have some partially completed data at {}.\n\n
"""
sendJobFailureEmail(email, msg, dwnld_loc)
return
else:
return 'An error occurred while processing your request. Please send an email to \
def _ingest_files(fns, genus, tb_name):
print "Connecting to database %s ..." % db_args["default"]["NAME"]
db = MySQLdb.connect(host=db_args["default"]["HOST"], user=db_args["default"]["USER"],
passwd=db_args["default"]["PASSWORD"], db=db_args["default"]["NAME"])
db.autocommit(True)
with closing(db.cursor()) as cursor:
cursor.connection.autocommit(True)
for graph_fn in fns:
print "Processing %s ..." % graph_fn
mtime = os.stat(graph_fn).st_mtime # get modification time
g_changed = True
# In DB and modified
test_qry ="select g.mtime from %s.%s as g where g.filepath = \"%s\";" % (db_args["default"]["NAME"], tb_name, graph_fn)
if cursor.execute(test_qry): # Means graph already in DB
if cursor.fetchall()[0][0] == os.stat(graph_fn).st_mtime: # Means graphs hasn't changed since ingest
g_changed = False
print "Ignoring %s ..." % graph_fn
else:
cursor.execute("delete from %s.%s where filepath = \"%s\";" % (db_args["default"]["NAME"], tb_name, graph_fn))
print " ===> Updating %s ..." % graph_fn
if g_changed: # Means graph has changed since ingest OR was never in DB to start with
# Collect all the attributes etc ..
g = igraph_io.read_arbitrary(graph_fn, informat="graphml", headers_only=True)
vertex_attrs = g.vs.attribute_names()
edge_attrs = g.es.attribute_names()
graph_attrs = g.attributes()
vcount = g.vcount()
ecount = g.ecount()
# Give some default values if none exist
if "sensor" in graph_attrs: sensor = g["sensor"]
else: sensor = ""
if "source" in graph_attrs: source = g["source"]
else: source = ""
if "region" in graph_attrs: region = g["region"]
else: region = ""
if "project" in graph_attrs: project = g["project"]
else: project = ""
#url = "http://openconnecto.me/mrdata/graphs/"+("/".join(graph_fn.replace("\\", "/").split('/')[-2:]))
url = get_download_path(graph_fn)
# This statement puts each graph into the DB
qry_stmt = "insert into %s.%s values (\"%s\",\"%s\",\"%s\",\"%s\",%d,%d,\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",%f,\"%s\");" \
% (db_args["default"]["NAME"], tb_name, os.path.abspath(graph_fn), genus, region, project,
np.int64(np.float64(vcount)), np.int64(np.float64(ecount)), str(graph_attrs)[1:-1].replace("'",""),
str(vertex_attrs)[1:-1].replace("'",""),
str(edge_attrs)[1:-1].replace("'",""), sensor, source, mtime, url)
cursor.execute(qry_stmt)
def main():
parser = argparse.ArgumentParser(description="")
parser.add_argument("infn", action="store", help="Input file name")
parser.add_argument("-f", "--factor", action="store", type=int, help="Downsampling factor")
parser.add_argument("-a", "--ds_atlas", action="store", help="Pre-Downsampled atlas file name")
parser.add_argument("outfn", action="store", help="Output file name")
parser.add_argument("--informat", "-i", action="store", default="graphml", help="Input format of the graph")
parser.add_argument("--outformat", "-o", action="store", default="graphml", help="Output format of the graph")
result = parser.parse_args()
if result.factor >= 0:
g = igraph_io.read_arbitrary(result.infn, informat=result.informat)
new_graph = downsample(g, factor=result.factor)
elif result.ds_atlas:
g = igraph_io.read_arbitrary(result.infn, informat=result.informat)
new_graph = downsample(g, ds_atlas=nib.load(result.ds_atlas))
else:
sys.stderr.write("[ERROR]: either -f or -a flag must be specified\n")
exit(-1)
new_graph.write(result.outfn, format=result.outformat)
def _ingest_files(fns, genus, tb_name):
print "Connecting to database %s ..." % db_args["default"]["NAME"]
db = MySQLdb.connect(host=db_args["default"]["HOST"],
user=db_args["default"]["USER"],
passwd=db_args["default"]["PASSWORD"],
db=db_args["default"]["NAME"])
db.autocommit(True)
with closing(db.cursor()) as cursor:
cursor.connection.autocommit(True)
for graph_fn in fns:
print "Processing %s ..." % graph_fn
mtime = os.stat(graph_fn).st_mtime # get modification time
g_changed = True
# In DB and modified
test_qry = "select g.mtime from %s.%s as g where g.filepath = \"%s\";" % (
db_args["default"]["NAME"], tb_name, graph_fn)
if cursor.execute(test_qry): # Means graph already in DB
if cursor.fetchall()[0][0] == os.stat(
graph_fn
).st_mtime: # Means graphs hasn't changed since ingest
g_changed = False
print "Ignoring %s ..." % graph_fn
else:
cursor.execute(
"delete from %s.%s where filepath = \"%s\";" %
(db_args["default"]["NAME"], tb_name, graph_fn))
print " ===> Updating %s ..." % graph_fn
if g_changed: # Means graph has changed since ingest OR was never in DB to start with
# Collect all the attributes etc ..
g = igraph_io.read_arbitrary(graph_fn,
informat="graphml",
headers_only=True)
vertex_attrs = g.vs.attribute_names()
edge_attrs = g.es.attribute_names()
graph_attrs = g.attributes()
vcount = g.vcount()
ecount = g.ecount()
# Give some default values if none exist
if "sensor" in graph_attrs: sensor = g["sensor"]
else: sensor = ""
if "source" in graph_attrs: source = g["source"]
else: source = ""
if "region" in graph_attrs: region = g["region"]
else: region = ""
if "project" in graph_attrs: project = g["project"]
else: project = ""
#url = "http://openconnecto.me/mrdata/graphs/"+("/".join(graph_fn.replace("\\", "/").split('/')[-2:]))
url = get_download_path(graph_fn)
# This statement puts each graph into the DB
qry_stmt = "insert into %s.%s values (\"%s\",\"%s\",\"%s\",\"%s\",%d,%d,\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",%f,\"%s\");" \
% (db_args["default"]["NAME"], tb_name, os.path.abspath(graph_fn), genus, region, project,
np.int64(np.float64(vcount)), np.int64(np.float64(ecount)), str(graph_attrs)[1:-1].replace("'",""),
str(vertex_attrs)[1:-1].replace("'",""),
str(edge_attrs)[1:-1].replace("'",""), sensor, source, mtime, url)
cursor.execute(qry_stmt)
def scale_convert(selected_files,
dl_format,
ds_factor,
ATLASES,
email=None,
dwnld_loc=None,
zip_fn=None):
# Debug
print "Entering scale function ..."
try:
if dl_format == "graphml" and ds_factor == 0:
temp = zipfiles(selected_files, use_genus=True, zip_out_fn=zip_fn)
else:
files_to_zip = {}
for fn in selected_files:
# No matter what we need a temp file
print "Creating temp file ..."
tmpfile = tempfile.NamedTemporaryFile("w",
delete=False,
dir="/data/pytmp")
print "Temp file %s created ..." % tmpfile.name
tmpfile.close()
# Downsample only valid for *BIG* human brains!
# *NOTE: If smallgraphs are ingested this will break
if ds_factor and get_genus(fn) == "human":
if isinstance(ds_factor, int):
print "downsampling to factor %d" % ds_factor
g = downsample(igraph_io.read_arbitrary(fn, "graphml"),
ds_factor)
print "downsample complete"
else:
g = downsample(igraph_io.read_arbitrary(fn, "graphml"),
atlas=nib_load(ATLASES[ds_factor]))
else:
g = igraph_io.read_arbitrary(fn, "graphml")
# Write to `some` format
if dl_format == "mm":
igraph_io.write_mm(g, tmpfile.name)
else:
g.write(tmpfile.name, format=dl_format)
files_to_zip[fn] = tmpfile.name
temp = zipfiles(files_to_zip,
use_genus=True,
zip_out_fn=zip_fn,
gformat=dl_format)
# Del temp files
for tmpfn in files_to_zip.values():
print "Deleting %s ..." % tmpfn
os.remove(tmpfn)
except Exception, msg:
print "An exception was thrown and caught with message %s!" % msg
if email:
msg = """
Hello,\n\nYour most recent job failed to complete.
\nYou may have some partially completed data at {}.\n\n
"""
sendJobFailureEmail(email, msg, dwnld_loc)
return
else:
return 'An error occurred while processing your request. Please send an email to \