示例#1
0
def load_graph_and_inputs(args):
    from dispel4py.utils import load_graph

    graph = load_graph(args.module, args.attr)
    if graph is None:
        return None, None

    graph.flatten()
    inputs = create_inputs(args, graph)
    return graph, inputs
示例#2
0
def load_graph_and_inputs(args):
    from dispel4py.utils import load_graph

    graph = load_graph(args.module, args.attr)
    if graph is None:
        return None, None

    graph.flatten()
    inputs = create_inputs(args, graph)
    return graph, inputs
示例#3
0
def run():
    from pyspark import SparkContext, SparkConf

    conf = SparkConf()
    conf.setAppName('dispel4py')
    conf.set("spark.storage.memoryFraction", "0.5")
    sc = SparkContext(
        conf=conf)

    from dispel4py.new import processor
    from dispel4py.utils import load_graph

    args = parse_args()

    graph = load_graph(args.module, args.attr)
    if graph is None:
        return
    graph.flatten()

    inputs = processor.create_inputs(args, graph)

    process(sc, graph, inputs=inputs, args=args)
示例#4
0
def load_graph_and_inputs(args):
    from dispel4py.utils import load_graph
    graph = load_graph(args.module, args.attr)
    if graph is None:
        return None, None

    graph.flatten()
    inputs = create_inputs(args, graph)

    if args.provenance:
        if not os.path.exists(args.provenance):
            print("Can't load provenance configuration %s" % args.provenance)
        else:
            from dispel4py.provenance import init_provenance_config, configure_prov_run, ProvenanceType
            prov_config, remaining = init_provenance_config(args, inputs)
            ## Ignore returned remaining command line arguments. Will be taken care of in main()
            print(prov_config)
            configure_prov_run(graph,
                               provImpClass=(ProvenanceType, ),
                               sprovConfig=prov_config)

    return graph, inputs
示例#5
0
                        metavar='iterations',
                        type=int,
                        help='number of iterations',
                        default=1)
    parser.add_argument('-s',
                        '--simple',
                        help='force simple processing',
                        action='store_true')
    parser.add_argument('-n',
                        '--num',
                        metavar='num_processes',
                        type=int,
                        help='number of processes to run')
    args = parser.parse_args()

    graph = load_graph(args.module, args.attr)
    if graph is None:
        sys.exit(1)
    graph.flatten()

    # run only once if no input data
    inputs = {}
    if args.file:
        try:
            with open(args.file) as inputfile:
                inputs = json.loads(inputfile.read())
            print "Processing input file %s" % args.file
        except:
            print 'Failed to read input file %s' % args.file
            sys.exit(1)
    elif args.data:
示例#6
0
def main():
    from importlib import import_module
    import json
    import os

    from dispel4py.utils import load_graph

    parser = create_arg_parser()
    args, remaining = parser.parse_known_args()
    # args = parser.parse_args()

    graph = load_graph(args.module, args.attr)
    if graph is None:
        sys.exit(1)
    graph.flatten()

    # run only once if no input data
    inputs = {}
    if args.file:
        try:
            with open(args.file) as inputfile:
                inputs = json.loads(inputfile.read())
            print "Processing input file %s" % args.file
        except:
            print 'Failed to read input file %s' % args.file
            sys.exit(1)
    elif args.data:
        inputs = json.loads(args.data)
    else:
        if args.iter == 1:
            print 'Processing 1 iteration.'
        else:
            print 'Processing %s iterations.' % args.iter
        roots = set()
        for node in graph.graph.nodes():
            if _is_root(node, graph):
                inputs[node.getContainedObject()] = args.iter

    # map input names to ids if necessary
    for node in graph.graph.nodes():
        pe = node.getContainedObject()
        try:
            d = inputs.pop(pe)
            inputs[pe.id] = d
        except:
            pass
        try:
            d = inputs.pop(pe.name)
            inputs[pe.id] = d
        except:
            pass

    try:
        # see if platform is in the mappings file as a simple name
        target = config[args.target]
    except KeyError:
        # it is a proper module name - fingers crossed...
        target = args.target
    try:
        parse_args = getattr(import_module(target), 'parse_args')
        args = parse_args(remaining, args)
    except:
        # no other arguments required for target
        pass
    process = getattr(import_module(target), 'process')

    if target == "dispel4py.new.simple_process":
        try:
            error_message = process(graph, inputs=inputs, args=args)
        except Exception:
            import ipdb, sys, traceback  # NOQA
            _, _, tb = sys.exc_info()
            traceback.print_exc()
            ipdb.post_mortem(tb)
    else:
        error_message = process(graph, inputs=inputs, args=args)

    if error_message:
        parser.print_usage()
        print error_message