def load_graph_and_inputs(args):
from dispel4py.utils import load_graph
graph = load_graph(args.module, args.attr)
if graph is None:
return None, None
graph.flatten()
inputs = create_inputs(args, graph)
return graph, inputs
def load_graph_and_inputs(args):
from dispel4py.utils import load_graph
graph = load_graph(args.module, args.attr)
if graph is None:
return None, None
graph.flatten()
inputs = create_inputs(args, graph)
return graph, inputs
def run():
from pyspark import SparkContext, SparkConf
conf = SparkConf()
conf.setAppName('dispel4py')
conf.set("spark.storage.memoryFraction", "0.5")
sc = SparkContext(
conf=conf)
from dispel4py.new import processor
from dispel4py.utils import load_graph
args = parse_args()
graph = load_graph(args.module, args.attr)
if graph is None:
return
graph.flatten()
inputs = processor.create_inputs(args, graph)
process(sc, graph, inputs=inputs, args=args)
def load_graph_and_inputs(args):
from dispel4py.utils import load_graph
graph = load_graph(args.module, args.attr)
if graph is None:
return None, None
graph.flatten()
inputs = create_inputs(args, graph)
if args.provenance:
if not os.path.exists(args.provenance):
print("Can't load provenance configuration %s" % args.provenance)
else:
from dispel4py.provenance import init_provenance_config, configure_prov_run, ProvenanceType
prov_config, remaining = init_provenance_config(args, inputs)
## Ignore returned remaining command line arguments. Will be taken care of in main()
print(prov_config)
configure_prov_run(graph,
provImpClass=(ProvenanceType, ),
sprovConfig=prov_config)
return graph, inputs
metavar='iterations',
type=int,
help='number of iterations',
default=1)
parser.add_argument('-s',
'--simple',
help='force simple processing',
action='store_true')
parser.add_argument('-n',
'--num',
metavar='num_processes',
type=int,
help='number of processes to run')
args = parser.parse_args()
graph = load_graph(args.module, args.attr)
if graph is None:
sys.exit(1)
graph.flatten()
# run only once if no input data
inputs = {}
if args.file:
try:
with open(args.file) as inputfile:
inputs = json.loads(inputfile.read())
print "Processing input file %s" % args.file
except:
print 'Failed to read input file %s' % args.file
sys.exit(1)
elif args.data:
def main():
from importlib import import_module
import json
import os
from dispel4py.utils import load_graph
parser = create_arg_parser()
args, remaining = parser.parse_known_args()
# args = parser.parse_args()
graph = load_graph(args.module, args.attr)
if graph is None:
sys.exit(1)
graph.flatten()
# run only once if no input data
inputs = {}
if args.file:
try:
with open(args.file) as inputfile:
inputs = json.loads(inputfile.read())
print "Processing input file %s" % args.file
except:
print 'Failed to read input file %s' % args.file
sys.exit(1)
elif args.data:
inputs = json.loads(args.data)
else:
if args.iter == 1:
print 'Processing 1 iteration.'
else:
print 'Processing %s iterations.' % args.iter
roots = set()
for node in graph.graph.nodes():
if _is_root(node, graph):
inputs[node.getContainedObject()] = args.iter
# map input names to ids if necessary
for node in graph.graph.nodes():
pe = node.getContainedObject()
try:
d = inputs.pop(pe)
inputs[pe.id] = d
except:
pass
try:
d = inputs.pop(pe.name)
inputs[pe.id] = d
except:
pass
try:
# see if platform is in the mappings file as a simple name
target = config[args.target]
except KeyError:
# it is a proper module name - fingers crossed...
target = args.target
try:
parse_args = getattr(import_module(target), 'parse_args')
args = parse_args(remaining, args)
except:
# no other arguments required for target
pass
process = getattr(import_module(target), 'process')
if target == "dispel4py.new.simple_process":
try:
error_message = process(graph, inputs=inputs, args=args)
except Exception:
import ipdb, sys, traceback # NOQA
_, _, tb = sys.exc_info()
traceback.print_exc()
ipdb.post_mortem(tb)
else:
error_message = process(graph, inputs=inputs, args=args)
if error_message:
parser.print_usage()
print error_message