def main(): usage = """usage: %prog [options] /path/to/dump.rdb Example 1 : %prog -k "user-" -k "friends-" /var/redis/6379/dump.rdb Example 2 : %prog /var/redis/6379/dump.rdb""" parser = OptionParser(usage=usage) parser.add_option("-k", "--key", dest="keys", action="append", help="Keys that should be grouped together. Multiple prefix can be provided") (options, args) = parser.parse_args() if len(args) == 0: parser.error("Redis RDB file not specified") dump_file = args[0] if not options.keys: options.keys = [] stat2d = MyStatsAggregator(options.keys) callback = MemoryCallback(stat2d, 64) parser = RdbParser(callback) try: parser.parse(dump_file) except Exception, e: print 'error: ', e
def main(): usage = """usage: %prog [options] /path/to/dump.rdb Example 1 : %prog -k "user.*" -k "friends.*" -f memoryreport.html /var/redis/6379/dump.rdb Example 2 : %prog /var/redis/6379/dump.rdb""" parser = OptionParser(usage=usage) parser.add_option("-f", "--file", dest="output", help="Output file", metavar="FILE") parser.add_option("-k", "--key", dest="keys", action="append", help="Keys that should be grouped together. Multiple regexes can be provided") (options, args) = parser.parse_args() if len(args) == 0: parser.error("Redis RDB file not specified") dump_file = args[0] if not options.output: output = "redis_memory_report.html" else: output = options.output stats = StatsAggregator() callback = MemoryCallback(stats, 64) parser = RdbParser(callback) parser.parse(dump_file) stats_as_json = stats.get_json() t = open(os.path.join(os.path.dirname(__file__),"report.html.template")).read() report_template = Template(t) html = report_template.substitute(REPORT_JSON = stats_as_json) print(html)
def main(): usage = """usage: %prog [options] /path/to/dump.rdb Example : %prog --command json -k "user.*" /var/redis/6379/dump.rdb""" parser = OptionParser(usage=usage) parser.add_option("-c", "--command", dest="command", help="Command to execute. Valid commands are json or diff", metavar="FILE") parser.add_option("-f", "--file", dest="output", help="Output file", metavar="FILE") parser.add_option("-n", "--db", dest="dbs", action="append", help="Database Number. Multiple databases can be provided. If not specified, all databases will be included.") parser.add_option("-k", "--key", dest="keys", default=None, help="Keys to export. This can be a regular expression") parser.add_option("-t", "--type", dest="types", action="append", help="""Data types to include. Possible values are string, hash, set, sortedset, list. Multiple typees can be provided. If not specified, all data types will be returned""") (options, args) = parser.parse_args() if len(args) == 0: parser.error("Redis RDB file not specified") dump_file = args[0] filters = {} if options.dbs: filters['dbs'] = [] for x in options.dbs: try: filters['dbs'].append(int(x)) except ValueError: raise Exception('Invalid database number %s' %x) if options.keys: filters['keys'] = options.keys if options.types: filters['types'] = [] for x in options.types: if not x in VALID_TYPES: raise Exception('Invalid type provided - %s. Expected one of %s' % (x, (", ".join(VALID_TYPES)))) else: filters['types'].append(x) destination = sys.stdout if options.output: desitnaion = open(options.output, 'wb') cmds = { 'diff': DiffCallback, 'json': JSONCallback, 'memory': lambda r : MemoryCallback(r, 64) } for key,cb in cmds.items(): if key != options.command: continue with destination as f: parser = RdbParser(cb(f), filters=filters) parser.parse(dump_file) return True raise Exception('Invalid Caommand %s' % options.output)
def main(): parser = OptionParser(usage="Usage: %prog [options] [[--] <application> [<args>]]") collection_group = OptionGroup(parser, "Collection options") collection_group.add_option("--file", metavar="FILE", dest="file", help="") collection_group.add_option("--func", metavar="OUT", dest="function", help="") collection_group.add_option("--out", metavar="OUT", dest="out", help="") collection_group.add_option("--cache", metavar="CACHE", dest="cache", help="") collection_group.add_option("--format", metavar="FORMAT", dest="format", help="") parser.add_option_group(collection_group) (options, arguments) = parser.parse_args() if options.file: if not os.path.exists(options.file): parser.error("file not found") else: parser.error("file not specified") info = HpccInfo('xstftool', options) parser = Parser(options.format, info, options.out, options.cache) print datetime.datetime.now().ctime() + ' Start' if options.cache is None: parser.parse() else: if not parser.read_cache(): parser.parse() parser.to_print() print "\n" + datetime.datetime.now().ctime() + ' End' return 0
def main(): from optparse import OptionParser parser = OptionParser() parser.add_option("-p", "--port", dest="port", type="int", default=8822, help="AMGA instance port") parser.add_option("-i", "--interval", dest="interval", type="int", default=1, help="Poll interval") (opts, args) = parser.parse_args() if len(args) == 0: host = "localhost" else: host = args[0] print "Querying: " + host + ":" + str(opts.port) parser = Parser() log = None # count = 0 while True: (serverType, xmlData) = pollAMGA(host, opts.port) #print xmlData oldLog = log log = parser.parse(serverType, xmlData) print log # count += 1 # print "Queries:",count time.sleep(opts.interval)
def main(): from optparse import OptionParser parser = OptionParser() parser.add_option("-p", "--port", dest="port", type="int", default=8822, help="AMGA instance port") parser.add_option("-i", "--interval", dest="interval", type="int", default=1, help="Poll interval") (opts, args) = parser.parse_args() if len(args) == 0: host = "localhost" else: host = args[0] print "Querying: " + host + ":" + str(opts.port) parser = amgaMonitor.Parser() log = None (serverType, xmlData) = amgaMonitor.pollAMGA(host, opts.port) if serverType == amgaMonitor.ServerType.MDSERVER: miner = ServerMiner() elif serverType == amgaMonitor.ServerType.MDREPDAEMON: miner = RepDaemonMiner() else: raise Exception("Bad id: " + str(serverType)) while True: # print xmlData log = parser.parse(serverType, xmlData) print log miner.processLog(log) time.sleep(opts.interval) (serverType, xmlData) = amgaMonitor.pollAMGA(host, opts.port)
def main(): from optparse import OptionParser parser = OptionParser( usage="usage: %prog [options] network1 network2...", version="%prog 1.4\n"+_copyright, description=_description ) parser.add_option("-m", "--manifest", dest='createManifest', action='store_true', help="Generates the manifest file") parser.add_option("-t", "--ttl", dest='createTtls', action='store_true', help="Generates ttl files for each network") parser.add_option("-i", "--main", dest='createMain', action='store_true', help="Generates the main C++ file for the plugin library") parser.add_option("-u", "--uribase", dest='uribase', default="http://clam-project.org/examples/lv2", help="Specifies the uri base for the plugins", metavar="URIBASE") parser.add_option("-d", "--doap", dest='doapfile', help="Specifies a doapfile with additional info when generating a ttl", metavar="DOAPFILE") parser.add_option("-y", "--binary", dest='binary', help="Provides a name for the library binary (required by --ttls)", metavar="LIBBINARY") parser.add_option("-g", "--gui-binary", dest='guibinary', default=None, help="Provides a name for a ui library binary for the plugin in the ttl", metavar="UIBINARY") options, networks = parser.parse_args() names = [os.path.splitext(os.path.basename(network))[0] for network in networks] uris = [os.path.join(options.uribase,name) for name in names ] if options.createTtls: for network, uri, name in zip(networks, uris, names) : parser = make_parser() curHandler = ExporterHandler() parser.setContentHandler(curHandler) parser.parse(open(network)) curHandler.printTTL(network,uri,name,options.doapfile,options.binary,options.guibinary) return if options.createManifest: printManifest(uris,names) return if options.createMain: printCLAM_PLUGIN(networks,uris) return
def main(): usage= 'usage: %prog [options] infname' parser= OptionParser(usage=usage) parser.add_option('-o', '--output', dest='output_folder', default='patterns', help='output folder') parser.add_option('-c', '--no-cache', dest='cache', action='store_false', default=True, help='discard cache') options, args= parser.parse_args(argv[1:]) if len(args) < 1: parser.error('not enaught args') parser= parserclass('models_cache') infname= args[0] score= parser.parse(infname) score= quantize(score) output_folder_prefix= options.output_folder pat_sizes= [4] margins= range(3) f= 2 key= lambda n:('%s|%s' % (n.duration, n.is_silence),) writer= writerclass() results= {} patterns= get_score_patterns(score, pat_sizes, margins, f, key) instr, all_notes= score.notes_per_instrument.iteritems().next() output_folder= os.path.join(output_folder_prefix, infname[:-4]) if not os.path.exists(output_folder): os.mkdir(output_folder) for i, (pat, matches) in enumerate(patterns.iteritems()): pattern_folder= os.path.join(output_folder, 'pat_%s' % i) if not os.path.exists(pattern_folder): os.mkdir(pattern_folder) for j, (start, end) in enumerate(matches): notes= all_notes[start:end] notes= [n.copy() for n in notes] notes[0].start= 0 for prev, next in zip(notes, notes[1:]): next.start= prev.start+prev.duration s= score.copy() s.notes_per_instrument={instr:notes} writer.dump(s, os.path.join(pattern_folder, 'match_%s.mid' % j))
def run(): usage = "usage: %prog source dest [options]" parser = OptionParser() parser.add_option("-s", "--with-slots", dest="with_slots", action="store_true", help="generate code using __slots__") (options, args) = parser.parse_args() if len(args) != 2: parser.error("exactly two arguments required: path to directory with .proto files and path to destination package") d = args[0] od = args[1] exit_status = 0 protos = [f for f in os.listdir(d) if f.endswith(".proto")] for p in protos: sys.stdout.write(("%s..." % p).ljust(70)) sys.stdout.flush() source = open(os.path.join(d, p)).read() try: parser = make_parser() r = parser.parse(source) _, res, l = r if l != len(source): raise SyntaxError("Syntax error on line %s near %r" % ( source[:l].count('\n') + 1, source[l:l+10])) s = gen_module([m for m in res if type(m) is tuple], [m for m in res if type(m) is str], [m for m in res if type(m) is list], options.with_slots, ) open(os.path.join(od, convert_proto_name(p) + ".py"), 'wb').write(s) except: sys.stdout.write("[FAIL]\n") sys.stdout.flush() traceback.print_exc() exit_status = 1 else: sys.stdout.write("[OKAY]\n") sys.stdout.flush() return exit_status
def main(): usage= 'usage: %prog [options] infname1 [infname2 ...]' parser= OptionParser(usage=usage) parser.add_option('-o', '--output', dest='output_fname', default='cluster.out', help='output fname') parser.add_option('-c', '--no-cache', dest='cache', action='store_false', default=True, help='discard cache') parser.add_option('-k', '--resulting-dimensionality', dest='k', default=150, help='el numero de dimensiones resultantes') options, args= parser.parse_args(argv[1:]) if len(args) < 1: parser.error('not enaught args') parser= parserclass('models_cache') infnames= args outfname= options.output_fname print 'parsing scores' scores= [parser.parse(infname, cache=options.cache) for infname in infnames] nclusterss= [2] # range(2,5) npasss= [5,10, 15, 20, 25] methods= ['a', 'm'] dists= ['e', 'b', 'c', 'a', 'u', 'x', 's', 'k'] configs= list(combine(nclusterss, npasss, methods, dists)) results= {} for k in range(2,10,2): print 'k=', k concept_vectors= apply_lsa(scores, k) step= len(configs)/10 for i, (nclusters, npass, method, dist) in enumerate(configs): if (i+1)%step == 0: print '\t', ((i+1)*100)/len(configs) r= Pycluster.kcluster(concept_vectors, nclusters= nclusters, method= method, dist= dist) results[(k, nclusters, npass, method, dist)]= r f= open('clusters_results.pickle', 'w') pickle.dump(results, f, 2) f.close()
def main(): usage= 'usage: %prog [options] infname' parser= OptionParser(usage=usage) parser.add_option('-c', '--no-cache', dest='cache', action='store_false', default=True, help='discard cache') options, args= parser.parse_args(argv[1:]) if len(args) < 1: parser.error('not enaught args') parser= parserclass('models_cache') infname= args[0] score= parser.parse(infname, cache=options.cache) #import ipdb;ipdb.set_trace() notes= score.notes_per_instrument.values()[0] for n in notes: print n
def main(): usage= 'usage: %prog [options] infname outfname' parser= OptionParser(usage=usage) parser.add_option('-p', '--patch', dest='patch', type='int', help='patch to select') options, args= parser.parse_args(argv[1:]) if len(args) < 1: parser.error('not enaught args') infname= args[0] outfname= infname.replace('.mid', '.svg') parser= MidiScoreParser() score= parser.parse(infname) rank(score) graph= build_graph(score) graph.draw(outfname, prog='dot') # , args='-Gsize="4096,4096"')
def main(): usage = "vs_cmd.py PTV PATH_TO_PLUGINS" parser = OptionParser(usage) (_, args) = parser.parse_args() if len(args) != 2: parser.error(usage) ptv, path = args # Use arguments here vs.Logger_setLevel(4) vs.loadPlugins(path) parser = vs.Parser_create() if not parser.parse(ptv): raise Exception('could not parse the configuration') pano = vs.PanoDefinition_create(parser.getRoot().has("pano")) merger = vs.ImageMergerFactory_createMergerFactory( parser.getRoot().has("merger")) warper = vs.ImageWarperFactory_createWarperFactory( parser.getRoot().has("warper")) flow = vs.ImageFlowFactory_createFlowFactory(parser.getRoot().has("flow")) print("Panorama size: {}x{}".format(pano.width, pano.height)) output = parser.getRoot().has("output") first_frame = 0 last_frame = 100 input_factory = vs.DefaultReaderFactory(first_frame, last_frame) controller = vs.createController(vs.PanoDeviceDefinition(), pano, merger.object(), wraper.object(), flow.object(), input_factory) writer = vs.create(output, "test", pano.width, pano.height) device = vs.StitcherDevice() device.device = 0 stitcher = controller.createStitcher(device) stitch_output = controller.createAsyncStitchOutput( device, writer.object().getVideoWriter()) status = stitcher.stitch(stitch_output.object()) print("CAN I HAZ STITCH ? {}".format(status.ok()))
def main(): usage= 'usage: %prog [options] infname [outfname]' parser= OptionParser(usage=usage) parser.add_option('-f', '--from', dest='moment_from', help='start moment in ticks', default=0) parser.add_option('-t', '--to', dest='moment_to', help='end moment in ticks') parser.add_option('-d', '--duration', dest='duration', help='duration of the slice') parser.add_option('-c', '--no-cache', dest='cache', action='store_false', default=True, help='discard cache') options, args= parser.parse_args(argv[1:]) if len(args) < 1: parser.error('not enaught args') parser= parserclass('models_cache') moment_from= int(options.moment_from) if options.moment_to is None and options.duration is None: parser.error('falta --to y --duration') elif options.moment_to is None: moment_to= int(options.moment_from)+int(options.duration) else: moment_to= int(options.moment_to) infname= args[0] if len(args) == 1: outfname= '%s-%s-%s.mid' % (infname[:-4], moment_from, moment_to) else: outfname= args[1] print 'creating score' score= parser.parse(infname, cache=options.cache) beat_duration= float(score.tempo)/1e6 divisions= score.divisions for instrument, notes in score.notes_per_instrument.iteritems(): new_notes= [n for n in notes \ if n.start/divisions*beat_duration >= moment_from and \ (n.start + n.duration)/divisions*beat_duration< moment_to] score.notes_per_instrument[instrument]= new_notes print 'dumping' writer= writerclass() writer.dump(score, outfname)
def main(): usage = """usage: %prog [options] /path/to/dump.rdb Example : %prog --command json -k "user.*" /var/redis/6379/dump.rdb""" parser = OptionParser(usage=usage) parser.add_option("-c", "--command", dest="command", help="Command to execute. Valid commands are json or diff", metavar="FILE") parser.add_option("-f", "--file", dest="output", help="Output file", metavar="FILE") parser.add_option("-n", "--db", dest="dbs", action="append", help="Database Number. Multiple databases can be provided. If not specified, all databases will be included.") parser.add_option("-k", "--key", dest="keys", default=None, help="Keys to export. This can be a regular expression," "When picking this mode, we will dump a deep copy of the key") parser.add_option("-t", "--type", dest="types", action="append", help="""Data types to include. Possible values are string, hash, set, sortedset, list. Multiple typees can be provided. If not specified, all data types will be returned""") parser.add_option("-p", "--pos", dest="pos", help="""Position in RDB file to skip to, after generated an memory index file this can be used to speed up. The data starts reading at 9 bytes""", default = 9) parser.add_option("-m", "--max", dest="max", help=""" Read maximum number of keys, to limit search.""", default = 2e31) parser.add_option("-v", "--verbose", dest="verbose", help="""If true dump a deep copy of the data structure""", action = "store_true", default = False) parser.add_option("-q", "--quick", dest="quick", help="""If true dump a deep copy of the data structure""", action = "store_true", default = False) (options, args) = parser.parse_args() if len(args) == 0: parser.error("Redis RDB file not specified") dump_file = args[0] filters = {} if options.dbs: filters['dbs'] = [] for x in options.dbs: try: filters['dbs'].append(int(x)) except ValueError: raise Exception('Invalid database number %s' %x) if options.keys: filters['keys'] = options.keys if options.types: filters['types'] = [] for x in options.types: if not x in VALID_TYPES: raise Exception('Invalid type provided - %s. Expected one of %s' % (x, (", ".join(VALID_TYPES)))) else: filters['types'].append(x) filters['pos'] = options.pos filters['max'] = options.max # TODO : Fix this ugly if-else code callback = None if options.output: f = open(options.output, "wb") if 'diff' == options.command: callback = DiffCallback(f) elif 'json' == options.command: callback = JSONCallback(f) elif 'memory' == options.command: reporter = PrintAllKeys(f) callback = MemoryCallback(reporter, 64, options.verbose) else: raise Exception('Invalid Command %s' % options.output) else: if 'diff' == options.command: callback = DiffCallback(sys.stdout) elif 'json' == options.command: callback = JSONCallback(sys.stdout) elif 'memory' == options.command: reporter = PrintAllKeys(sys.stdout) callback = MemoryCallback(reporter, 64, options.verbose) else: raise Exception('Invalid Command %s' % options.output) start = time.clock() parser = RdbParser(callback, filters=filters, quick=options.quick) parser.parse(dump_file) end = time.clock() print "time=%s seconds" % (end-start)
parser = OptionParser() parser.add_option("-v", "--verbose", dest="verbose", help="Print verbose log to stdout", action="store_true", default=False) parser.add_option("", "--no-warnings", dest="warnings", help="Hide warning messages", action="store_false", default=True) parser.add_option("", "--no-errors", dest="errors", help="Hide error messages", action="store_false", default=True) parser.add_option("", "--modules", dest="modules", help="comma-separated list of modules to generate comments", metavar="MODS", default=",".join(rst_parser.allmodules)) (options, args) = parser.parse_args(sys.argv) options.modules = options.modules.split(",") if len(args) < 2 or len(options.modules) < 1: parser.print_help() exit(0) parser = rst_parser.RstParser(hdr_parser.CppHeaderParser()) for m in options.modules: parser.parse(m, os.path.join(selfpath, "../../" + m)) parser.printSummary() generator = JavadocGenerator(parser.definitions, options.modules) generator.verbose = options.verbose generator.show_warnings = options.warnings generator.show_errors = options.errors for path in args: folder = os.path.abspath(path) for jfile in [f for f in glob.glob(os.path.join(folder,"*.java")) if not f.endswith("-jdoc.java")]: outfile = os.path.abspath(os.path.basename(jfile).replace(".java", "-jdoc.java")) generator.document(jfile, outfile) generator.printSummary()
lines = [] end_exists = False try: for i in range(0, len(code)): if code[i].split()[1] == r'end': code = code[0:i + 1] end_exists = True break if not end_exists: raise AssertionError except Exception: print("EOF (end) expected.") sys.exit(-1) for line in code: # run the code parser.parse("\n" + line) if args.noforward == "yes": DG = DependencyGraph(lines) noper = Nope() noper.add_nops(DG.dependencies, DG.lines, forwarding=False) elif args.forward == "yes": DG = DependencyGraph(lines) noper = Nope() noper.add_nops(DG.dependencies, DG.lines, forwarding=True) elif args.schedule == "yes": DG = DependencyGraph(lines, out_of_order=True) DG.get_edges() DG.get_dependency_tree() #DG.print_dependency_graph() # comment out for debugging #DG.print_dependencies()
def main(): # get command line inputs. Use -h to list all possible arguments and default values parser = OptionParser( usage="Usage: %prog [options] <input1>.log <input2>.log ...") parser.add_option("--amp", dest="amplitude", action="store", help="amplitude (default 0.2)", default="0.2", type="float", metavar="AMPLITUDE") parser.add_option("--nproc", dest="nproc", action="store", help="number of processors (default 1)", default="1", type="int", metavar="NPROC") parser.add_option("--mem", dest="mem", action="store", help="memory (default 4GB)", default="4GB", type="string", metavar="MEM") parser.add_option( "--route", dest="route", action="store", help="calculation route (defaults to same as original file)", default=None, type="string", metavar="ROUTE") parser.add_option("-v", dest="verbose", action="store_true", help="verbose output", default=True, metavar="VERBOSE") parser.add_option("--auto", dest="auto", action="store_true", help="turn on automatic batch processing", default=False, metavar="AUTO") parser.add_option("--name", dest="suffix", action="store", help="append to file name (defaults to QRC)", default="QRC", type="string", metavar="SUFFIX") parser.add_option( "-f", "--freq", dest="freq", action="store", help="request motion along a particular frequency (cm-1)", default=None, type="float", metavar="FREQ") parser.add_option( "--freqnum", dest="freqnum", action="store", help="request motion along a particular frequency (number)", default=None, type="int", metavar="FREQNUM") #arguments for running calcs parser.add_option( "--qcoord", dest="qcoord", action="store_true", help= "request automatic single point calculation along a particular normal mode (number)", default=False, metavar="QCOORD") parser.add_option( "--nummodes", dest="nummodes", action="store", help="number of modes for automatic single point calculation", default='all', type='string', metavar="NUMMODES") (options, args) = parser.parse_args() files = [] if len(sys.argv) > 1: for elem in sys.argv[1:]: try: if os.path.splitext(elem)[1] in [".out", ".log"]: for file in glob(elem): files.append(file) except IndexError: pass for file in files: # parse compchem output with cclib & count imaginary frequencies parser = cclib.io.ccopen(file) data = parser.parse() #for i in range(1,len(data.atomcoords)): # mw_distance = mwdist(data.atomcoords[i], data.atomcoords[1], data.atomnos) # print(mw_distance) if hasattr(data, 'vibfreqs'): im_freq = len([val for val in data.vibfreqs if val < 0]) else: print('o {} has no frequency information: exiting'.format(file)) sys.exit() if not options.qcoord: if im_freq == 0 and options.auto != False: print('x {} has no imaginary frequencies: skipping'.format( file)) else: if options.freq == None and options.freqnum == None: print('o {} has {} imaginary frequencies: processing'. format(file, im_freq)) elif options.freq != None: print( 'o {} will be distorted along the frequency of {} cm-1: processing' .format(file, options.freq)) elif options.freqnum != None: print( 'o {} will be distorted along the frequency number {}: processing' .format(file, options.freqnum)) qrc = gen_qrc(file, options.amplitude, options.nproc, options.mem, options.route, options.verbose, options.suffix, options.freq, options.freqnum) #doing automatic calcualtions (single points for stability check) else: log_output = Logger("RUNIRC", 'dat', options.nummodes) if im_freq == 0: log_output.Writeonlyfile( 'o {} has no imaginary frequencies: check for stability'. format(file)) amp_base = [round(elem, 2) for elem in np.arange(0, 1, 0.1)] # amp_base_backward = [round(elem, 2) for elem in np.arange(-1,0,0.1) ] energy_base = [] root_dir = os.getcwd() parent_dir = os.getcwd() + '/' + file.split('.')[0] if not os.path.exists(parent_dir): os.makedirs(parent_dir) log_output.Writeonlyfile( 'o Entering directory {}'.format(parent_dir)) # getting energetics of the current molecule energy_base.append(data.freeenergy) #creating folders for number of normal modes if options.nummodes == 'all': freq_range = range(1, len(data.vibfreqs) + 1) else: freq_range = range(1, len(data.vibfreqs) + 1) freq_range = freq_range[:int(options.nummodes)] for num in freq_range: num_dir = parent_dir + '/' + 'num_' + str(num) if not os.path.exists(num_dir): os.makedirs(num_dir) log_output.Writeonlyfile( 'o Entering directory {}'.format(num_dir)) shutil.copyfile(root_dir + '/' + file, num_dir + '/' + file) os.chdir(num_dir) for amp in amp_base: suffix = 'num_' + str(num) + '_amp_' + str(amp).split( '.')[0] + str(amp).split('.')[1] run_irc(file, options, num, amp, freq.LEVELOFTHEORY, suffix, freq.CHARGE, freq.MULT, log_output) log_output.Writeonlyfile('o Writing to file {}'.format( file.split('.')[0] + '_' + suffix)) os.chdir(parent_dir)
# These are duplicates, only found via a contents page still # linked to via a "Back to contents" link. if re.search('day-wa-01_war0816\.htm',filename): continue if re.search('day-wa-03_war1110\.htm',filename): continue if re.search('day-wa-03_war0922\.htm',filename): continue if re.search('day-wa-03_war0805\.htm',filename): continue # This is a misnamed file, a duplicate of wa-02_wa1004.htm if re.search('day-wa-02_wa0104\.htm',filename): continue parser = Parser() parser.parse(filename) # try: # parser.parse(filename) # except Exception, e: # if verbose: print "An unhandled exception occured in parsing: "+filename # if verbose: print "The exception was: "+str(e) # traceback.print_exc(file=sys.stdout) # files_and_exceptions.append( (filename,e) ) # if verbose: print "All exceptions:" # for e in files_and_exceptions: # if verbose: print "%s: %s" % e
parser.add_option("-d", "--key-directory", default="", dest="key_directory", help="Specify a parent directory for keys") (options, args) = parser.parse_args() if len(args) < 2: parser.error("Must specify a config file (keys.conf) AND mac_permissions.xml file(s)!") logging.basicConfig(level=logging.INFO if options.verbose == True else logging.WARN) # Read the config file config = ParseConfig() config.read(args[0]) os.chdir(options.root) output_file = sys.stdout if options.output_file == "stdout" else open(options.output_file, "w") logging.info("Setting output file to: " + options.output_file) # Generate the key list key_map = config.generateKeyMap(options.target_build_variant.lower(), options.key_directory) logging.info("Generate key map:") for k in key_map: logging.info(k + " : " + str(key_map[k])) # Generate the XML file with markup replaced with keys parser = make_parser() parser.setContentHandler(ReplaceTags(key_map, output_file)) for f in args[1:]: parser.parse(f)
sys.exit(1) data_folder = args[0] output_folder = args[1] # Import the data report report = DataReport() if not(report.parse(data_folder)): print 'ERROR: Failed to parse the data report' sys.exit(1) # Parse the data written by the instrument parser = DataParser() (data_format, classification_results) = parser.parse(report.data(INSTRUMENT_NAME)) # Only keep the images from the test set classification_results = filter(lambda x: report.image(x.image_index).test_index is not None, classification_results) # Complete the data if it was saved in an old format if data_format != '1.1': label_names = [] for label in range(0, report.nbLabels()): label_names.append(report.labelName(label)) database_name = report.getExperimentSettingDatabaseName() for classification_result in classification_results:
if name == "msg": self.outFile.write("\n") else: self.outFile.write(",") def characters(self, ch): self.outFile.write(ch.strip("\n\t,")) if __name__ == "__main__": parser = OptionParser() parser.add_option("-f", "--file", dest="filename", help="write report to FILE", metavar="FILE", default="none") (options, args) = parser.parse_args() if options.filename == "none": exit() parser = xml.sax.make_parser() fileCSV = open("dump.csv", "w") handler = XML2CSV(fileCSV) parser.setContentHandler(handler) parser.parse(options.filename)
# # cleanup / preparation # cmd = 'rm -rf %s/%s' % (QASRC_DIRFN, corpusname) logging.info(cmd) os.system(cmd) cmd = 'mkdir -p %s/%s' % (QASRC_DIRFN, corpusname) logging.info(cmd) os.system(cmd) # # parse XML # parser = make_parser() parser.setContentHandler(WikiContentHandler(WIKIXML[options.lang])) article_cnt = 0 article_tot = 0 wikisize = os.path.getsize(WIKIXML[options.lang]) with open(WIKIXML[options.lang], 'rb') as wikibz2f: # with codecs.getreader('utf8')(BZ2File(WIKIXML[options.lang], 'r')) as wikif: with codecs.getreader('utf8')(BZ2File(wikibz2f)) as wikif: parser.parse(wikif)
tarball = tarfile.open(args[0], 'r') except Exception, e: print "Error: %s" % str(e) sys.exit(1) queue = Queue.Queue() #start the workers for i in range(int(options.cores)): worker = Worker(queue) worker.setDaemon(True) worker.start() #fill the queue parser=ConfigParser(args[0]) requests=parser.parse() for request in requests: if os.path.exists(request._dir) and options.force is False: print 'Directory '+request._dir+' exists already, doing nothing.' print 'You can overwrite with the --force option' continue if not os.path.exists(request._dir): os.mkdir(request._dir) commandBuilder = LocalCommandBuilder(request, options.events) #, options.batch) command = commandBuilder.build()
help="don't delete the temporary files created "\ "(useful for finding problems)", default=False) options, files = parser.parse_args(argv[1:]) if not files: print "Error: no files to process" print >> sys.stderr, __doc__ return 1 options.flags = options.gccxml_options options.verbose = not options.quiet parser = cparser.IncludeParser(options) parser.parse(files) def main(argv=None): if argv is None: argv = sys.argv try: compile_to_xml(argv) except cparser.CompilerError, detail: print >> sys.stderr, "CompilerError:", detail return 1 if __name__ == "__main__": sys.exit(main())
def main(): usage = """usage: %prog [options] /path/to/dump.rdb Example : %prog --command json -k "user.*" /var/redis/6379/dump.rdb""" parser = OptionParser(usage=usage) parser.add_option( "-c", "--command", dest="command", help="Command to execute. Valid commands are json or diff", metavar="FILE") parser.add_option("-f", "--file", dest="output", help="Output file", metavar="FILE") parser.add_option( "-n", "--db", dest="dbs", action="append", help= "Database Number. Multiple databases can be provided. If not specified, all databases will be included." ) parser.add_option("-k", "--key", dest="keys", default=None, help="Keys to export. This can be a regular expression") parser.add_option( "-t", "--type", dest="types", action="append", help= """Data types to include. Possible values are string, hash, set, sortedset, list. Multiple typees can be provided. If not specified, all data types will be returned""") (options, args) = parser.parse_args() if len(args) == 0: parser.error("Redis RDB file not specified") dump_file = args[0] filters = {} if options.dbs: filters['dbs'] = [] for x in options.dbs: try: filters['dbs'].append(int(x)) except ValueError: raise Exception('Invalid database number %s' % x) if options.keys: filters['keys'] = options.keys if options.types: filters['types'] = [] for x in options.types: if not x in VALID_TYPES: raise Exception( 'Invalid type provided - %s. Expected one of %s' % (x, (", ".join(VALID_TYPES)))) else: filters['types'].append(x) # TODO : Fix this ugly if-else code if options.output: with open(options.output, "wb") as f: if 'diff' == options.command: callback = DiffCallback(f) elif 'json' == options.command: callback = JSONCallback(f) elif 'memory' == options.command: reporter = PrintAllKeys(f) callback = MemoryCallback(reporter, 64) else: raise Exception('Invalid Command %s' % options.output) parser = RdbParser(callback) parser.parse(dump_file) else: if 'diff' == options.command: callback = DiffCallback(sys.stdout) elif 'json' == options.command: callback = JSONCallback(sys.stdout) elif 'memory' == options.command: reporter = PrintAllKeys(sys.stdout) callback = MemoryCallback(reporter, 64) else: raise Exception('Invalid Command %s' % options.output) parser = RdbParser(callback, filters=filters) parser.parse(dump_file)
def main(): usage = """usage: %prog [options] /path/to/dump.rdb Example : %prog --command json -k "user.*" /var/redis/6379/dump.rdb""" parser = OptionParser(usage=usage) parser.add_option("-c", "--command", dest="command", help="Command to execute. Valid commands are json, diff, justkeys, justkeyvals, memory and protocol", metavar="FILE") parser.add_option("-f", "--file", dest="output", help="Output file", metavar="FILE") parser.add_option("-n", "--db", dest="dbs", action="append", help="Database Number. Multiple databases can be provided. If not specified, all databases will be included.") parser.add_option("-k", "--key", dest="keys", default=None, help="Keys to export. This can be a regular expression") parser.add_option("-o", "--not-key", dest="not_keys", default=None, help="Keys Not to export. This can be a regular expression") parser.add_option("-t", "--type", dest="types", action="append", help="""Data types to include. Possible values are string, hash, set, sortedset, list. Multiple typees can be provided. If not specified, all data types will be returned""") parser.add_option("-b", "--bytes", dest="bytes", default=None, help="Limit memory output to keys greater to or equal to this value (in bytes)") parser.add_option("-l", "--largest", dest="largest", default=None, help="Limit memory output to only the top N keys (by size)") parser.add_option("-e", "--escape", dest="escape", choices=ESCAPE_CHOICES, help="Escape strings to encoding: %s (default), %s, %s, or %s." % tuple(ESCAPE_CHOICES)) (options, args) = parser.parse_args() if len(args) == 0: parser.error("Redis RDB file not specified") dump_file = args[0] filters = {} if options.dbs: filters['dbs'] = [] for x in options.dbs: try: filters['dbs'].append(int(x)) except ValueError: raise Exception('Invalid database number %s' %x) if options.keys: filters['keys'] = options.keys if options.not_keys: filters['not_keys'] = options.not_keys if options.types: filters['types'] = [] for x in options.types: if not x in VALID_TYPES: raise Exception('Invalid type provided - %s. Expected one of %s' % (x, (", ".join(VALID_TYPES)))) else: filters['types'].append(x) out_file_obj = None try: if options.output: out_file_obj = open(options.output, "wb") else: # Prefer not to depend on Python stdout implementation for writing binary. out_file_obj = os.fdopen(sys.stdout.fileno(), 'wb') try: callback = { 'diff': lambda f: DiffCallback(f, string_escape=options.escape), 'json': lambda f: JSONCallback(f, string_escape=options.escape), 'justkeys': lambda f: KeysOnlyCallback(f, string_escape=options.escape), 'justkeyvals': lambda f: KeyValsOnlyCallback(f, string_escape=options.escape), 'memory': lambda f: MemoryCallback(PrintAllKeys(f, options.bytes, options.largest), 64, string_escape=options.escape), 'protocol': lambda f: ProtocolCallback(f, string_escape=options.escape) }[options.command](out_file_obj) except: raise Exception('Invalid Command %s' % options.command) if not PYTHON_LZF_INSTALLED: eprint("WARNING: python-lzf package NOT detected. " + "Parsing dump file will be very slow unless you install it. " + "To install, run the following command:") eprint("") eprint("pip install python-lzf") eprint("") parser = RdbParser(callback, filters=filters) parser.parse(dump_file) finally: if options.output and out_file_obj is not None: out_file_obj.close()
def main(argv): #global globalX, globalY usage = "%prog [OPTIONS] INPUT_FILE OUTPUT_FILE" parser = OptionParser(usage = usage, version = "pdfa2hocr 0.2") parser.add_option("-m", "--mapping", help="mapping from PDF/A tags to hOCR elements", dest="mapping", default=None) parser.add_option("-c", "--columns", help="divide text into columns", dest="columns", default=None) parser.add_option("-p", "--pdfminer", action="store_true", help="analyze layout using pdfminer", dest="pdfminer", default=False) parser.add_option("-l", "--module", help="custom layout analyzing module", dest="module", default=None) parser.add_option("-v", "--verbose", action="store_true", help="write information about progress", dest="verbose", default=False) parser.add_option("-i", "--ignore-text-groups", action="store_true", help="ignore textgroups from pdfminer (textboxes only - simpler layout)", dest="ignore", default=False) parser.add_option("-g", "--ignore-pages", help="pages which should not be divided into columns", dest="pages", default="") parser.add_option("-f", "--font-mapping", help="mapping of PostScript font names into html style", dest="fontMap", default=None) parser.add_option("-u", "--icu", help="use ICU word segmentation algorithm", dest="icu", default=None) parser.add_option("-t", "--special-font-tags", action="store_true", help="use special font style hOCR tags", dest="tags", default=False) parser.add_option("-r", "--resolution", help="resolution of page in pixels", dest="resolution", default=None) (options, args) = parser.parse_args(argv) if len(args) != 3: #print args parser.print_help() exit() #print args #print "a", options.columns, options.ignore, options.module #fin = "C:\\Users\\to\\Documents\\Próbka2.pdf" #fin = "C:\\Users\\to\\Desktop\\linde\\Linde2edK_1_0133-0222.pdf" #fin = "C:\\Users\\to\\Desktop\\pliki\\FRiLindeJSBkor1-50.pdf" #fin = "C:\\Users\\to\\Desktop\\linde\\wyn.xml" #fout = "C:\\Users\\to\\Documents\\a.html" fin = args[1] fout = args[2] if not os.path.exists(fin): sys.stderr.write("File " + fin + " does not exist\n") exit() # TODO: E sprawdzic, czy sciezka do fout jest prawidlowa map = None if options.fontMap != None: if not os.path.exists(options.fontMap): sys.stderr.write("File " + options.fontMap + " does not exist\n") exit() map = FontMap(options.fontMap) # ladujemy z pliku mapowanie postscriptowych # nazw fontow na rodziny if options.module != None: if not os.path.exists(options.module): sys.stderr.write("File " + options.module + " does not exist\n") exit() if options.module[-3:] != ".py": sys.stderr.write("File " + options.module + " should have .py " + "extension\n") exit() if options.mapping != None: if not os.path.exists(options.mapping): sys.stderr.write("File " + options.mapping + " does not exist\n") exit() if options.resolution != None: dims = options.resolution.split("x") utils.globalX = int(dims[0]) utils.globalY = int(dims[1]) if (not str.isdigit(dims[0])) or int(dims[0]) == 0 or (not str.isdigit(dims[1])) or int(dims[1]) == 0: sys.stderr.write("Invalid parameters of -r/--resolution option") exit() if options.pdfminer: # wczytujemy PDF i analizujemy uklad strony PDFMinerem lib = XMLLib(map=map) # tworzymy obiekt udajacy TagLib (patrz TagLib i # XMLLib) lib.loadPTree(fin) # ladujemy slowniki zasobow stron parser = PDFMinerParser() (columnizer, cols) = processModuleAndColumnsOptions(options) print options.columns if columnizer != None: columnizer.setCols(cols) # ustawiamy liczbe kolumn do podzielenia if options.mapping == None and columnizer == None: # eksportujemy bez # dodatkowej analizy ukladu strony if not parser.extractHOCRFromPDFDirect(fin, lib, fout, options.ignore, options.verbose, map, options.icu, options.tags): # patrz opis metody extractHOCRFromPDFDirect sys.stderr.write("File " + fin + " is not a PDF file.\n") exit() else: hocr = HOCRExporter(None, fout, lib, mapping=options.mapping, xml=True, verbose=options.verbose, fontMap=map, icu=options.icu, tags=options.tags) # tworzymy obiekt eksportujacy do hOCR if not parser.extractHOCRFromPDF(fin, lib, hocr, columnizer, options.ignore): # patrz opis metody extractHOCRFromPDF sys.stderr.write("File " + fin + " is not a PDF file.\n") exit() else: try: lib = TagLib(map=map) # tworzymy obiekt bedacy interfejsem do pdfminera # dla kokretnego pliku if not lib.initialize(fin): # otwieramy plik PDF pdfminerem # okazalo sie, ze to jednak plik XML: # TODO: F od razu page z SAXA do hOCR if options.fontMap == None: sys.stderr.write("Warning: pdfmimer results conversion to hOCR without font map specified. In resulting hOCR fonts won't display properly in browser.\n") parser = PDFMinerParser() lib = XMLLib(map=map) # tworzymy obiekt udajacy TagLib (patrz TagLib i # XMLLib) parser.parse(fin, options.ignore, lib=lib) # wczytujemy plik XML root = parser.getResult() # pobieramy korzen drzewa struktury (columnizer, cols) = processModuleAndColumnsOptions(options) if columnizer != None: columnizer.columnize(root, cols) # analizujemy uklad strony if columnizer.isFatal(): # blad w columnizerze - nie udalo sie # nic podzielic sys.stderr.write(columnizer.getLastMessage()) sys.exit() elif columnizer.isError(): # nie udalo sie podzielic niektorych stron for m in columnizer.getMessages(): sys.stderr.write("Warning: " + m + "\n") hocr = HOCRExporter(root, fout, lib, mapping=options.mapping, xml=True, verbose=options.verbose, fontMap=map, icu=options.icu, tags=options.tags) # tworzymy obiekt konwertujacy do hOCR else: # to faktycznie plik PDF if options.columns != None: sys.stderr.write("Warning: --columns option specified for" + " export using PDF structural tags. Option" + " ignored.\n") if options.module != None: sys.stderr.write("Warning: --module option specified for" + " export using PDF structural tags. Option" + " ignored.\n") if options.ignore: sys.stderr.write("Warning: --ignore option specified for" + " export using PDF structural tags. Option" + " ignored.\n") if options.pages != "": sys.stderr.write("Warning: --ignore-pages option specified for" + " export using PDF structural tags. Option" + " ignored.\n") if not lib.hasRoot(): # nie ma StructTreeRoot sys.stderr.write("PDF file does not contain logical strucutre.\n") exit() root = lib.getRoot() hocr = HOCRExporter(root, fout, lib, mapping=options.mapping, verbose=options.verbose, fontMap=map, icu=options.icu, tags=options.tags) # tworzymy obiekt eksportujacy plik PDF except MyException: exit() hocr.export() # eksportujemy plik PDF hocr.save()
def main(): usage = """usage: %prog [options] /path/to/dump.rdb Example : %prog --command json -k "user.*" /var/redis/6379/dump.rdb""" parser = OptionParser(usage=usage) parser.add_option( "-c", "--command", dest="command", help="Command to execute. Valid commands are json or diff", metavar="FILE") parser.add_option("-f", "--file", dest="output", help="Output file", metavar="FILE") parser.add_option( "-n", "--db", dest="dbs", action="append", help= "Database Number. Multiple databases can be provided. If not specified, all databases will be included." ) parser.add_option("-k", "--key", dest="keys", default=None, help="Keys to export. This can be a regular expression") parser.add_option( "-t", "--type", dest="types", action="append", help= """Data types to include. Possible values are string, hash, set, sortedset, list. Multiple typees can be provided. If not specified, all data types will be returned""") (options, args) = parser.parse_args() if len(args) == 0: parser.error("Redis RDB file not specified") dump_file = args[0] filters = {} if options.dbs: filters['dbs'] = [] for x in options.dbs: try: filters['dbs'].append(int(x)) except ValueError: raise Exception('Invalid database number %s' % x) if options.keys: filters['keys'] = options.keys if options.types: filters['types'] = [] for x in options.types: if not x in VALID_TYPES: raise Exception( 'Invalid type provided - %s. Expected one of %s' % (x, (", ".join(VALID_TYPES)))) else: filters['types'].append(x) destination = sys.stdout if options.output: desitnaion = open(options.output, 'wb') cmds = { 'diff': DiffCallback, 'json': JSONCallback, 'memory': lambda r: MemoryCallback(r, 64) } for key, cb in cmds.items(): if key != options.command: continue with destination as f: parser = RdbParser(cb(f), filters=filters) parser.parse(dump_file) return True raise Exception('Invalid Caommand %s' % options.output)
exc_type, exc_value, exc_traceback = sys.exc_info() stdlogger.error("Erreur parsing file:" + file_name) stdlogger.error( "Error in convert_url_to_html method, stack %s" % (e)) stdlogger.error(e, exc_info=True) return '' if __name__ == "__main__": usage = "usage: %prog http://www.server.com/page.html" parser = OptionParser(usage=usage, description="Parse favori_file") parser.add_option("-d", "--debug", action="store_true", dest="debug", help="Turn on debug logging") parser.add_option("-q", "--quiet", action="store_true", dest="quiet", help="turn off all logging") parser.add_option("-o", "--output", action="store", dest="output", default="output.html", help="output file name, defaults to output.html") (options, args) = parser.parse_args() logging.basicConfig(level=logging.DEBUG if options.debug else (logging.ERROR if options.quiet else logging.INFO)) for file_name in args: stdlogger.info("Parsing file_name " + file_name) parser = FavoriteParser() parser.parse(file_name)
def main(): usage= 'usage: %prog [options] midis' parser= OptionParser(usage=usage) parser.add_option('--n-measures', dest='n_measures', default=1, type='int', help='if the partition algorithm is MEASURE, especifies the number of measures to take as a unit') parser.add_option('-o', dest='outfname', help='the output file') options, args= parser.parse_args(argv[1:]) if len(args) < 1: parser.error('not enaught args') outfname= options.outfname if outfname is None: parser.error('missing outfname') infnames= args[:] parser= MidiScoreParser() models= [] for infname in infnames: score= parser.parse(infname) if not score: import ipdb;ipdb.set_trace() # por que podria devolver None? ######### # AJUSTES DE PARSING notes= score.get_first_voice() notes= [n for n in notes if n.duration > 0] instr= score.notes_per_instrument.keys()[0] patch= instr.patch channel= instr.channel score.notes_per_instrument= {instr:notes} ######### interval_size= measure_interval_size(score, options.n_measures) algorithm= HarmonyHMM(interval_size, instrument=patch, channel=channel) algorithm.train(score) algorithm.create_model() models.append(algorithm.model) def sim(m1, m2): """ calcula H(m1|m2) """ from math import log ans= 0 for s1 in m2.state_transition: for s2, prob in m1.state_transition.get(s1, {}).iteritems(): ans+= m2.pi[s1]*prob*log(prob) return -ans from os import path def get_node_name(ticks): n_quarters= 0 while ticks >= score.divisions: ticks-= score.divisions n_quarters+= 1 if ticks > 0: f= Fraction(ticks, score.divisions) if n_quarters > 0: return "%s + %s" % (n_quarters, f) else: return repr(f) return "%s" % n_quarters for i, m in enumerate(models): m.pi= m.calc_stationationary_distr() from pygraphviz import AGraph from utils.fraction import Fraction g= AGraph(directed=True, strict=False) for n1, adj in m.state_transition.iteritems(): n1_name= get_node_name(n1) for n2, prob in adj.iteritems(): n2_name= get_node_name(n2) g.add_edge(n1_name, n2_name) e= g.get_edge(n1_name, n2_name) e.attr['label']= str(prob)[:5] model_fname= path.basename(infnames[i]).replace('mid', 'png') g.draw(model_fname, prog='dot', args='-Grankdir=LR') sims= defaultdict(dict) for i, m1 in enumerate(models): for j in xrange(i+1, len(models)): m2= models[j] sims[path.basename(infnames[i])][path.basename(infnames[j])]= sim(m1, m2) sims[path.basename(infnames[j])][path.basename(infnames[i])]= sim(m2, m1) import csv f= open(outfname, 'w') writer= csv.writer(f) head= sorted(sims) head.insert(0, "-") writer.writerow(head) for (k, row) in sorted(sims.iteritems(), key=lambda x:x[0]): row= sorted(row.iteritems(), key=lambda x:x[0]) row= [t[1] for t in row] row.insert(0, k) writer.writerow(row) f.close() return from pygraphviz import AGraph from utils.fraction import Fraction g= AGraph(directed=True, strict=False) for infname1, adj in sims.iteritems(): for infname2, sim in adj.iteritems(): if sim>0.2 and sim!=0: continue g.add_edge(infname1, infname2) e= g.get_edge(infname1, infname2) e.attr['label']= str(sim)[:5] g.draw(outfname, prog='dot')
print 'usage: ingestpipeline.py <path_to_dir_containing_set_of_jobfiles>' logging.basicConfig(format="%(asctime)-15s %(levelname)s %(message)s", level=logging.DEBUG) logger = logging.getLogger('Slave') if options.user == 'ingest': import ingest_config as config else: import ingest_config_test as config path = args[0] if os.path.isfile(path): parser = job_parser.parser(logger) job = parser.parse(path) job['filename'] = path logger.info("Parsed jobfile %s: %s" % (path, str(job))) jobPipeline = IngestPipeline(None, jobToRun, config.momClient, config.ltaClient, config.ipaddress, config.ltacpport, config.mailCommand, config.momRetry, config.ltaRetry, config.srmRetry, config.srmInit) jobPipeline.run() exit(0) if os.path.isdir(path): dirEntries = [os.path.join(path, x) for x in os.listdir(path)] jobfilepaths = [p for p in dirEntries if os.path.isfile(p)] try:
def main(): parser = OptionParser() parser.add_option("-m", "--master", action="store", dest="master", help="Master URI", metavar="MASTER", default=os.getenv("SW_MASTER")) parser.add_option("-i", "--id", action="store", dest="id", help="Job ID", metavar="ID", default="default") parser.add_option("-e", "--env", action="store_true", dest="send_env", help="Set this flag to send the current environment with the script as _env", default=False) (options, args) = parser.parse_args() if not options.master: parser.print_help() print >> sys.stderr, "Must specify master URI with --master" sys.exit(1) if len(args) != 1: parser.print_help() print >> sys.stderr, "Must specify one script file to execute, as argument" sys.exit(1) script_name = args[0] master_uri = options.master id = options.id print id, "STARTED", now_as_timestamp() parser = SWScriptParser() script = parser.parse(open(script_name, 'r').read()) print id, "FINISHED_PARSING", now_as_timestamp() if script is None: print "Script did not parse :(" exit() cont = SWContinuation(script, SimpleContext()) if options.send_env: cont.context.bind_identifier('env', os.environ) http = httplib2.Http() master_data_uri = urlparse.urljoin(master_uri, "/data/") pickled_cont = pickle.dumps(cont) (_, content) = http.request(master_data_uri, "POST", pickled_cont) cont_id = simplejson.loads(content) print id, "SUBMITTED_CONT", now_as_timestamp() #print continuation_uri master_netloc = urlparse.urlparse(master_uri).netloc task_descriptor = {'dependencies': {'_cont' : SW2_ConcreteReference(cont_id, SWNoProvenance(), len(pickled_cont), [master_netloc])}, 'handler': 'swi'} master_task_submit_uri = urlparse.urljoin(master_uri, "/job/") (_, content) = http.request(master_task_submit_uri, "POST", simplejson.dumps(task_descriptor, cls=SWReferenceJSONEncoder)) print id, "SUBMITTED_JOB", now_as_timestamp() out = simplejson.loads(content) notify_url = urlparse.urljoin(master_uri, "/job/%s/completion" % out['job_id']) job_url = urlparse.urljoin(master_uri, "/browse/job/%s" % out['job_id']) print id, "JOB_URL", job_url #print "Blocking to get final result" (_, content) = http.request(notify_url) completion_result = simplejson.loads(content, object_hook=json_decode_object_hook) if "error" in completion_result.keys(): print id, "ERROR", completion_result["error"] return None else: print id, "GOT_RESULT", now_as_timestamp() #print content return completion_result["result_ref"]
action="store_true", help="don't delete the temporary files created "\ "(useful for finding problems)", default=False) options, files = parser.parse_args(argv[1:]) if not files: print "Error: no files to process" print >> sys.stderr, __doc__ return 1 options.flags = options.gccxml_options options.verbose = not options.quiet parser = cparser.IncludeParser(options) parser.parse(files) def main(argv=None): if argv is None: argv = sys.argv try: compile_to_xml(argv) except cparser.CompilerError, detail: print >> sys.stderr, "CompilerError:", detail return 1 if __name__ == "__main__": sys.exit(main())
dest="reformat", default=False, help="run reformat scripts for the observations according to namelist") options, args = parser.parse_args() if len(args) == 0: parser.print_help() sys.exit(0) # Get command arguments. input_xml_full_path = args[0] # Parse input namelist into project_info-dictionary. Project = xml_parsers.namelistHandler() parser = xml.sax.make_parser() parser.setContentHandler(Project) parser.parse(input_xml_full_path) # Project_info is a dictionary with all info from the namelist. project_info = Project.project_info if options.reformat: if 'REFORMAT' not in project_info.keys(): error('No REFORMAT tag specified in {0}'.format(input_xml_full_path)) if len(project_info['REFORMAT']) == 0: info('No reformat script specified', 1, 1) print_header({}, options.reformat) for k, v in project_info['REFORMAT'].iteritems(): if not os.path.exists(v): error('Path {0} does not exist'.format(v)) projects.run_executable(v, project_info, 1, False, write_di=False) sys.exit(0)
# These are duplicates, only found via a contents page still # linked to via a "Back to contents" link. if re.search('day-wa-01_war0816\.htm', filename): continue if re.search('day-wa-03_war1110\.htm', filename): continue if re.search('day-wa-03_war0922\.htm', filename): continue if re.search('day-wa-03_war0805\.htm', filename): continue # This is a misnamed file, a duplicate of wa-02_wa1004.htm if re.search('day-wa-02_wa0104\.htm', filename): continue parser = Parser() parser.parse(filename) # try: # parser.parse(filename) # except Exception, e: # if verbose: print "An unhandled exception occurred in parsing: "+filename # if verbose: print "The exception was: "+str(e) # traceback.print_exc(file=sys.stdout) # files_and_exceptions.append( (filename,e) ) # if verbose: print "All exceptions:" # for e in files_and_exceptions: # if verbose: print "%s: %s" % e
def endElement(self, name): if name == "ch1" or name == "ch2" or name == "watts" or name == "data": print "no" if name == "msg": self.outFile.write("\n") else: self.outFile.write(",") def characters(self, ch): self.outFile.write( ch.strip("\n\t,") ) if __name__ == "__main__": parser = OptionParser() parser.add_option("-f", "--file", dest="filename", help="write report to FILE", metavar="FILE", default="none") (options, args) = parser.parse_args() if options.filename == "none": exit() parser = xml.sax.make_parser() fileCSV = open("dump.csv", "w") handler = XML2CSV(fileCSV) parser.setContentHandler(handler) parser.parse(options.filename)
options.osmInput = options.osmInput.rstrip("." + archiveType) print "File uncompressed: ", options.osmInput if options.kmlOutput == None: options.kmlOutput = os.path.basename(options.osmInput).rstrip(".osc") if options.kmlVersion == None: options.kmlVersion = "1" myAwareness = OSMaware(debug=False, verbose=False, ele=options.linesElevation, kml_version=options.kmlVersion) parser = make_parser() parser.setContentHandler(myAwareness) t0 = datetime.now() print "Starting parsing OSM file..." parser.parse(options.osmInput) #for userName, userStat in self.statsUsers.iteritems(): print userName, userStat print "Number of contributors (users):", len(myAwareness.statsUsers) print "Number of Nodes created, deleted or modified:", myAwareness.nodeCount print "Number of Ways created, deleted or modified:", myAwareness.wayCount print "Number of Realations created, deleted or modified:", myAwareness.relationCount if options.kmlVersion == "2": myAwareness.createKmlV2(options.kmlOutput, heightFactor=myAwareness.linesElevation) if options.kmlVersion == "1": myAwareness.createKmlV1(options.kmlOutput) if options.kmlVersion == "0": myAwareness.createKmlV0(options.kmlOutput)
def run_standalone(): from optparse import OptionParser parser = OptionParser("usage: %prog [options] source.c") parser.add_option("-o", "--output", help="write output to FILE (default stdout)", metavar="FILE") parser.add_option("-E", "--preprocess", action="store_true") parser.add_option("-I", "--include", action="append", help="include path, passed on to C preprocessor", metavar="PATH") parser.add_option("-D", "--define", action="append", help="C macro definition, passed on to C preprocessor", metavar="NAME[=VALUE]") parser.add_option("--cpp", help="C preprocessor to use", metavar="COMMAND") parser.add_option("--ast", action="store_true", help="print syntax tree, quit") (options, args) = parser.parse_args() if not args: parser.print_help() sys.exit(1) in_file = args[0] src = open(in_file, "rt").read() src = insert_parens_in_brackets(in_file, src) if options.preprocess: extra_lines = PREAMBLE + [ "# 1 \"%s\"" % in_file, ] src = "\n".join(extra_lines) + "\n" + src if options.preprocess: cpp_options = [] #cpp_options = ["-P"] if options.include: for inc_dir in options.include: cpp_options.extend(["-I", inc_dir]) if options.define: for define in options.define: cpp_options.extend(["-D", define]) src = preprocess_source(src, options.cpp, cpp_options) #print "preprocessed source in ", write_temp_file(src, ".c") parser = GnuCndParser() ast = parser.parse(src, filename=in_file) if options.ast: ast.show() return generator = GnuCGenerator() if options.output is not None: outf = open(options.output, "wt") else: outf = sys.stdout try: outf.write(generator.visit(ast)) finally: if options.output is not None: outf.close()
if not options.direct_resync: parser.error( "The option --direct-resync is required to run this module directly" ) sys.exit(1) univention.debug.init("stderr", univention.debug.NO_FLUSH, univention.debug.NO_FUNCTION) ucr = ConfigRegistry() ucr.load() univention.debug.set_level(univention.debug.LISTENER, int(ucr.get('listener/debug/level', 2))) cmd = ['/usr/bin/univention-ldapsearch', '-LLL', filter, 'objectClass'] cmd.extend(attributes) p1 = subprocess.Popen(cmd, stdout=subprocess.PIPE) (stdout, stderr) = p1.communicate() class ListenerHandler(LDIFParser): def __init__(self, input): LDIFParser.__init__(self, input) def handle(self, dn, entry): handler(dn, entry, {}, 'a') parser = ListenerHandler(StringIO.StringIO(stdout)) parser.parse() subprocess.call(['net', 'cache', 'flush'])
if status == BuildStatus.Success: pass # Parse log to save 'TestFailed' results. elif status == BuildStatus.TestFailed : logging.info("Checking build log for '%s' at %d (%s)" % (name, starttime, ctime(starttime))) try: failures = [] # Grab the build log. log, headers = urllib.urlretrieve("http://tinderbox.mozilla.org/%s/%s" % (options.tree, build['logfile'])) gz = GzipFile(log) # I need a list of lines from the build log # assured to match because we search for this above harness_type = re.match(harness_regex, logname).groups()[0] parser = parsers.get(harness_type, logparser.LogParser)() failures = parser.parse(gz) # add the failures to the database for failure in failures: # convenience variables; can probably delete test = failure['test'] text = failure['text'] reason = failure['reason'] testnames_id=GetOrInsertTest(conn,test) if HaveFailRecord(conn,buildid, reason, testnames_id): logging.info("Skipping already recorded failure '%s' in build with id '%s' with failure record '%s' " % (test, buildid, text)) else: InsertTest(conn, buildid, reason, testnames_id, text)
option_session = options.session option_cookie = options.cookie setOptions(options) else: try: f = open("grabber.conf.xml", 'r') except IOError: print "No arguments ? You need to setup the XML configuration file or using the inline arguments" print "Look at the doc to start..." sys.exit(1) parser = make_parser() conf_handler = ConfHandler() # Tell the parser to use our handler parser.setContentHandler(conf_handler) parser.parse("grabber.conf.xml") option_url = confUrl option_spider = confSpider option_sql = "sql" in confActions option_bsql = "bsql" in confActions option_xss = "xss" in confActions option_backup = "backup" in confActions option_include = "include" in confActions option_js = "javascript" in confActions option_crystal = "crystal" in confActions option_session = "session" in confActions # default to localhost ? archives_url = "http://localhost" if option_url:
def main(): parser = OptionParser() parser.add_option("-m", "--master", action="store", dest="master", help="Master URI", metavar="MASTER", default=os.getenv("SW_MASTER")) parser.add_option("-i", "--id", action="store", dest="id", help="Job ID", metavar="ID", default="default") parser.add_option( "-e", "--env", action="store_true", dest="send_env", help= "Set this flag to send the current environment with the script as _env", default=False) (options, args) = parser.parse_args() if not options.master: parser.print_help() print >> sys.stderr, "Must specify master URI with --master" sys.exit(1) if len(args) != 1: parser.print_help() print >> sys.stderr, "Must specify one script file to execute, as argument" sys.exit(1) script_name = args[0] master_uri = options.master id = options.id print id, "STARTED", now_as_timestamp() parser = SWScriptParser() script = parser.parse(open(script_name, 'r').read()) print id, "FINISHED_PARSING", now_as_timestamp() if script is None: print "Script did not parse :(" exit() cont = SWContinuation(script, SimpleContext()) if options.send_env: cont.context.bind_identifier('env', os.environ) http = httplib2.Http() master_data_uri = urlparse.urljoin(master_uri, "/data/") pickled_cont = pickle.dumps(cont) (_, content) = http.request(master_data_uri, "POST", pickled_cont) cont_id = simplejson.loads(content) out_id = 'joboutput:%s' % cont_id print id, "SUBMITTED_CONT", now_as_timestamp() #print continuation_uri master_netloc = urlparse.urlparse(master_uri).netloc task_descriptor = { 'dependencies': { '_cont': SW2_ConcreteReference(cont_id, len(pickled_cont), [master_netloc]) }, 'handler': 'swi', 'expected_outputs': [out_id] } master_task_submit_uri = urlparse.urljoin(master_uri, "/job/") (_, content) = http.request( master_task_submit_uri, "POST", simplejson.dumps(task_descriptor, cls=SWReferenceJSONEncoder)) print id, "SUBMITTED_JOB", now_as_timestamp() out = simplejson.loads(content) notify_url = urlparse.urljoin(master_uri, "/job/%s/completion" % out['job_id']) job_url = urlparse.urljoin(master_uri, "/browse/job/%s" % out['job_id']) print id, "JOB_URL", job_url #print "Blocking to get final result" (_, content) = http.request(notify_url) completion_result = simplejson.loads(content, object_hook=json_decode_object_hook) if "error" in completion_result.keys(): print id, "ERROR", completion_result["error"] return None else: print id, "GOT_RESULT", now_as_timestamp() #print content return completion_result["result_ref"]
def main(): usage = """usage: %prog [options] /path/to/dump.rdb Example : %prog --command json -k "user.*" /var/redis/6379/dump.rdb""" parser = OptionParser(usage=usage) parser.add_option( "-c", "--command", dest="command", help= "Command to execute. Valid commands are json, diff, justkeys, justkeyvals, memory and protocol", metavar="FILE") parser.add_option("-f", "--file", dest="output", help="Output file", metavar="FILE") parser.add_option( "-n", "--db", dest="dbs", action="append", help= "Database Number. Multiple databases can be provided. If not specified, all databases will be included." ) parser.add_option("-k", "--key", dest="keys", default=None, help="Keys to export. This can be a regular expression") parser.add_option( "-o", "--not-key", dest="not_keys", default=None, help="Keys Not to export. This can be a regular expression") parser.add_option( "-t", "--type", dest="types", action="append", help= """Data types to include. Possible values are string, hash, set, sortedset, list. Multiple typees can be provided. If not specified, all data types will be returned""") parser.add_option( "-b", "--bytes", dest="bytes", default=None, help= "Limit memory output to keys greater to or equal to this value (in bytes)" ) parser.add_option( "-l", "--largest", dest="largest", default=None, help="Limit memory output to only the top N keys (by size)") parser.add_option( "-e", "--escape", dest="escape", choices=ESCAPE_CHOICES, help="Escape strings to encoding: %s (default), %s, %s, or %s." % tuple(ESCAPE_CHOICES)) (options, args) = parser.parse_args() if len(args) == 0: parser.error("Redis RDB file not specified") dump_file = args[0] filters = {} if options.dbs: filters['dbs'] = [] for x in options.dbs: try: filters['dbs'].append(int(x)) except ValueError: raise Exception('Invalid database number %s' % x) if options.keys: filters['keys'] = options.keys if options.not_keys: filters['not_keys'] = options.not_keys if options.types: filters['types'] = [] for x in options.types: if not x in VALID_TYPES: raise Exception( 'Invalid type provided - %s. Expected one of %s' % (x, (", ".join(VALID_TYPES)))) else: filters['types'].append(x) out_file_obj = None try: if options.output: out_file_obj = open(options.output, "wb") else: # Prefer not to depend on Python stdout implementation for writing binary. out_file_obj = os.fdopen(sys.stdout.fileno(), 'wb') try: callback = { 'diff': lambda f: DiffCallback(f, string_escape=options.escape), 'json': lambda f: JSONCallback(f, string_escape=options.escape), 'justkeys': lambda f: KeysOnlyCallback(f, string_escape=options.escape), 'justkeyvals': lambda f: KeyValsOnlyCallback(f, string_escape=options.escape), 'memory': lambda f: MemoryCallback(PrintAllKeys(f, options.bytes, options .largest), 64, string_escape=options.escape), 'protocol': lambda f: ProtocolCallback(f, string_escape=options.escape) }[options.command](out_file_obj) except: raise Exception('Invalid Command %s' % options.command) if not PYTHON_LZF_INSTALLED: eprint( "WARNING: python-lzf package NOT detected. " + "Parsing dump file will be very slow unless you install it. " + "To install, run the following command:") eprint("") eprint("pip install python-lzf") eprint("") parser = RdbParser(callback, filters=filters) parser.parse(dump_file) finally: if options.output and out_file_obj is not None: out_file_obj.close()
) (options, args) = parser.parse_args() if not options.input: print "--mapcss parameter is required" raise SystemExit if options.name: style_name = options.name else: style_name = re.sub("\..*", "", options.input) content = open(options.input).read() parser = MapCSSParser(debug=False) mapcss = parser.parse(content) mapcss_js = mapcss.as_js() subparts_var = ", ".join( map(lambda subpart: "s_%s = {}" % subpart, subparts)) subparts_var = " var %s;" % subparts_var subparts_fill = "\n".join( map( lambda subpart: " if (Object.keys(s_%s).length) {\n style['%s'] = s_%s; }" % (subpart, subpart, subpart), subparts)) js = """ (function (MapCSS) { 'use strict'; function restyle(style, tags, zoom, type, selector) {
else: stats = FortranStaticAnalysis() for name in args: stats.scan(name) stats.analyze() stats.summary() stats.saveStateToFile() if options.prof: theme = gprof2dot.TEMPERATURE_COLORMAP if options.theme_skew: theme.skew = options.theme_skew theme.rankdir = options.rankdir fh = open(options.prof, 'rt') parser = gprof2dot.GprofParser(fh) profile = parser.parse() profile.prune(options.node_thres/100.0, options.edge_thres/100.0) root, ext = os.path.splitext(options.prof) if not options.output: options.output = "%s.dot" % root fh = open(options.output, 'wb') dot = FortranDotWriter(fh) dot.graph(profile, theme, stats, exclude, title=root) fh.close() print(options.output) root, ext = os.path.splitext(options.output) print("%s.ps" % root) elif options.isolate: print "Isolating %s" % options.isolate i = stats.isolate(options.isolate) i.summary()
def main(): if_recording = 'r' supported_parser_list = [ Parser(if_recording, 'recording file', parsers.recording_file.RecordingFileMapParser), ] of_castle_fortify = 'cf' of_grid = 'g' supported_writer_list = [ Writer(of_castle_fortify, 'Castle Fortify (see http://castlefortify.com/)', writers.castle_fortify.Writer), Writer(of_grid, 'simple grid', writers.simple.Writer), ] valid_input_formats = [parser.name for parser in supported_parser_list] valid_output_formats = [writer.name for writer in supported_writer_list] parser = OptionParser() parser.add_option('-f', '--input-file', type = 'string', dest = 'input_filename', default = '', help = 'Input filename') parser.add_option('-i', '--input-format', type = 'choice', dest = 'input_format', default = if_recording, choices = valid_input_formats, help = 'Input format: ' + ', '.join(p.name + ' - ' + p.desc for p in supported_parser_list)) parser.add_option('-o', '--output-format', type = 'choice', dest = 'output_format', default = of_grid, choices = valid_output_formats, help = 'Output format: ' + ', '.join(w.name + ' - ' + w.desc for w in supported_writer_list)) parser.add_option('-O', '--output-format-list', type = 'string', dest = 'output_formats', default = '', help = 'Output formats separated by a comma, see --output-format') (options, args) = parser.parse_args() input_format = options.input_format output_formats = [] for output_format in options.output_formats.split(','): if output_format in valid_output_formats and not output_format in output_formats: output_formats += [output_format] if len(output_formats) == 0: output_formats = [options.output_format] try: input = io.input.FileInput(options.input_filename) except: print 'Could not open: \'' + options.input_filename + '\'' return parser = [parser.parser(input) for parser in supported_parser_list if parser.name == input_format][0] converting = True while converting: parsed_map = core.map.Map() try: parsed = parser.parse(parsed_map) if parsed: writer_list = [] for format in output_formats: writer_list += [writer.writer() for writer in supported_writer_list if writer.name == format] for writer in writer_list: writer.write(parsed_map) except parsers.recording_file.ParseError as e: print 'Failed to parse file:', e.message converting = not input.eof()
dest='outfname', help='output fname, default=midi_fname.intervals') parser.add_option('-c', '--no-cache', dest='cache', action='store_false', default=True, help='discard cache') options, args= parser.parse_args(argv[1:]) if len(args) < 1: parser.error('faltan argumentos') infame= args[0] outfname= options.outfname if outfname is None: outfname= infame[:-3] + 'intervals' parser= parserclass() score= parser.parse(infame) notes= score.notes_per_instrument.itervalues().next() # que no haya dos silencios consecutivos assert not any((prev.is_silence and next.is_silence for (prev, next) in zip(notes, notes[1:]))) for prev, next in zip(notes, notes[1:]): if next.is_silence: prev.duration += next.duration notes= [n for n in notes if not n.is_silence] def f(e): if e < 0: return 'm%s' % abs(e) else: return str(e) intervals= [f(next.pitch - prev.pitch) for (prev, next) in zip(notes, notes[1:])]
def main(self, args=None): """ Main entry point for the Application. Parameters ---------- :param args: command line arguments. Returns ---------- :return: N/A Examples -------- >>> python TestVectorDirectoryParserApp.py -dir /data/asc_dir -out Files.txt """ # **************************************** # Execution information # **************************************** print(__doc__) # **************************************** # Command line argument processing # **************************************** # Python 2.4 argument processing. parser = OptionParser() # REQUIRED ARGUMENTS parser.add_option("--dir", action="store", dest="dir", help='Path to the directory to parse (required).', default=None) parser.add_option("--out", action="store", dest="out", help='Path to the output file (required).', default=None) parser.add_option("--ext", action="store", dest="ext", help='File extension to look for (required).', default='.fil') parser.add_option("-f", type="int", dest="format", help='The file output format (optional).', default=1) parser.add_option("-v", action="store_true", dest="verbose", help='Verbose debugging flag (optional).', default=False) (args, options) = parser.parse_args() # Update variables with command line parameters. directory = args.dir output_file = args.out output_format = args.format verbose = args.verbose ############################################################ # Check user supplied parameters # ############################################################ # Check the directory is valid... if directory is None: print "No valid directory supplied, exiting." sys.exit() elif not Common.dir_exists(directory): print "No valid directory supplied, exiting." sys.exit() # Check the output file is valid... if output_file is None: print "No valid output file supplied, exiting." sys.exit() elif not Common.is_path_valid(output_file): print "No valid output file supplied, exiting." sys.exit() # Check the output file is valid... if args.ext is None: print "No valid file extension supplied, exiting." sys.exit() else: extension = [args.ext] if output_format < 1 or output_format > 2: print "You must supply a valid output format via the -f flag." print "1 - CSV." print "2 - JSON (not implemented yet)." sys.exit() ############################################################ # Start parsing the directory # ############################################################ print "\tSearching: ", directory # Used to measure feature generation time. start = datetime.datetime.now() parser = TestVectorDirectoryParser() parser.parse(directory, extension, output_file, output_format) # Finally get the time that the procedure finished. end = datetime.datetime.now() ############################################################ # Summarise outcome # ############################################################ print "\tFinished parsing" print "\tExecution time: ", str(end - start) print "Done."
def main(): usage = """usage: %prog [options] /path/to/dump.rdb Example : %prog --command json -k "user.*" /var/redis/6379/dump.rdb""" parser = OptionParser(usage=usage) parser.add_option("-c", "--command", dest="command", help="Command to execute. Valid commands are json, diff, and protocol", metavar="FILE") parser.add_option("-f", "--file", dest="output", help="Output file", metavar="FILE") parser.add_option("-n", "--db", dest="dbs", action="append", help="Database Number. Multiple databases can be provided. If not specified, all databases will be included.") parser.add_option("-k", "--key", dest="keys", default=None, help="Keys to export. This can be a regular expression") parser.add_option("-t", "--type", dest="types", action="append", help="""Data types to include. Possible values are string, hash, set, sortedset, list. Multiple typees can be provided. If not specified, all data types will be returned""") (options, args) = parser.parse_args() if len(args) == 0: parser.error("Redis RDB file not specified") dump_file = args[0] filters = {} if options.dbs: filters['dbs'] = [] for x in options.dbs: try: filters['dbs'].append(int(x)) except ValueError: raise Exception('Invalid database number %s' %x) if options.keys: filters['keys'] = options.keys if options.types: filters['types'] = [] for x in options.types: if not x in VALID_TYPES: raise Exception('Invalid type provided - %s. Expected one of %s' % (x, (", ".join(VALID_TYPES)))) else: filters['types'].append(x) # TODO : Fix this ugly if-else code if options.output: with open(options.output, "wb") as f: if 'diff' == options.command: callback = DiffCallback(f) elif 'json' == options.command: callback = JSONCallback(f) elif 'csv' == options.command: callback = CSVCallback(f) elif 'memory' == options.command: reporter = PrintAllKeys(f) callback = MemoryCallback(reporter, 64) elif 'protocol' == options.command: callback = ProtocolCallback(f) else: raise Exception('Invalid Command %s' % options.command) parser = RdbParser(callback) parser.parse(dump_file) else: if 'diff' == options.command: callback = DiffCallback(sys.stdout) elif 'json' == options.command: callback = JSONCallback(sys.stdout) elif 'csv' == options.command: callback = CSVCallback(sys.stdout) elif 'memory' == options.command: reporter = PrintAllKeys(sys.stdout) callback = MemoryCallback(reporter, 64) elif 'protocol' == options.command: callback = ProtocolCallback(sys.stdout) else: raise Exception('Invalid Command %s' % options.command) parser = RdbParser(callback, filters=filters) parser.parse(dump_file)
parser.add_option("-s", "--output-sprite", dest="sprite", help="Filename of generated CSS sprite. If not specified, [stylename].png will be uesed") (options, args) = parser.parse_args() if not options.input: print "--mapcss parameter is required" raise SystemExit style_name = re.sub("\..*", "", options.input) content = open(options.input).read() parser = MapCSSParser(debug=False) mapcss = parser.parse(content) mapcss_js = mapcss.as_js() subparts_var = "\n".join(map(lambda subpart: " var s_%s = {};" % subpart, subparts)) subparts_fill = "\n".join(map(lambda subpart: " if (s_%s) {\n style['%s'] = s_%s;\n }" % (subpart, subpart, subpart), subparts)) js = """ (function(MapCSS) { function restyle(tags, zoom, type, selector) { %s %s var style = {}; %s return style; } """ % (subparts_var, mapcss_js, subparts_fill)
def weblab_start(directory): parser = OptionParser(usage="%prog create DIR [options]") parser.add_option("-m", "--machine", dest="machine", default=None, metavar="MACHINE", help = "If there is more than one machine in the configuration, which one should be started.") parser.add_option("-l", "--list-machines", dest="list_machines", action='store_true', default=False, help = "List machines.") parser.add_option("-s", "--script", dest="script", default=None, metavar="SCRIPT", help = "If the runner option is not available, which script should be used.") options, args = parser.parse_args() check_dir_exists(directory, parser) def on_dir(directory, configuration_files): db_conf = DbConfiguration(configuration_files) regular_url = db_conf.build_url() coord_url = db_conf.build_coord_url() upgrader = DbUpgrader(regular_url, coord_url) return upgrader.check_updated() if not run_with_config(directory, on_dir): print >> sys.stderr, "Error: WebLab-Deusto instance outdated! You may have updated WebLab-Deusto recently. Run: weblab-admin.py upgrade %s" % directory sys.exit(-1) old_cwd = os.getcwd() os.chdir(directory) try: if options.script: # If a script is provided, ignore the rest if os.path.exists(options.script): execfile(options.script) elif os.path.exists(os.path.join(old_cwd, options.script)): execfile(os.path.join(old_cwd, options.script)) else: print >> sys.stderr, "Provided script %s does not exist" % options.script sys.exit(-1) else: parser = GlobalParser() global_configuration = parser.parse('.') if options.list_machines: for machine in global_configuration.machines: print ' - %s' % machine sys.exit(0) machine_name = options.machine if machine_name is None: if len(global_configuration.machines) == 1: machine_name = global_configuration.machines.keys()[0] else: print >> sys.stderr, "System has more than one machine (see -l). Please detail which machine you want to start with the -m option." sys.exit(-1) if not machine_name in global_configuration.machines: print >> sys.stderr, "Error: %s machine does not exist. Use -l to see the list of existing machines." % machine_name sys.exit(-1) machine_config = global_configuration.machines[machine_name] if machine_config.runner is None: if os.path.exists('run.py'): execfile('run.py') else: print >> sys.stderr, "No runner was specified, and run.py was not available. Please the -s argument to specify the script or add the <runner file='run.py'/> option in %s." % machine_name sys.exit(-1) else: if os.path.exists(machine_config.runner): execfile(machine_config.runner) else: print >> sys.stderr, "Misconfigured system. Machine %s points to %s which does not exist." % (machine_name, os.path.abspath(machine_config.runner)) sys.exit(-1) finally: os.chdir(old_cwd)
def main(): #################### args parsing ############ help = "#################Usage:##################\n dsTemplater -i <input XML> \n " help = help + "Example 1: dsTemplater -i dstemplate.xml \n " help = help + "Example 2: dsTemplater -i examples/dstemplate.60lo0FUTURE.xml \n" #print usage basedir = os.environ.get('BASEDIR') if(basedir is None): print "Warning: BASEDIR environment variable not set" parser = OptionParser(usage=help) parser.add_option("-i", "--file", dest="uinput", help="pass location of XML template", metavar="FILE") parser.add_option("-f", "--force",action="store_true",default=False, help="Force override existing output. Default is set to FALSE") #parser.add_option("-v", "--version",action="store_true", dest="version",default="v20120422", help="Assign version for downscaled data. Default is set to v20120422") (options, args) = parser.parse_args() verOpt = True #default forOpt = True #default inter = 'on' #for debugging if (inter == 'off'): uinput = 'dstemplate.xml' dversion = "v20120422" force = False if os.path.exists(uinput): parser = xml.sax.make_parser( ) handler = temphandler.TempHandler( ) parser.setContentHandler(handler) parser.parse(uinput) if (inter == 'on'): for opts,vals in options.__dict__.items(): # print opts,vals if(opts == 'uinput'): uinput = vals if os.path.exists(uinput): print "XML input:",uinput #parser = xml.sax.make_parser( ) #handler = temphandler.TempHandler( ) #parser.setContentHandler(handler) #parser.parse(uinput) else: print "Please pass a valid input XML filename with the -i argument and try again. See -h for syntax. Quitting now.." sys.exit() if(opts == 'force'): if (vals == True): forOpt = False force = vals ######### call listVars() ############################################################# output_grid,kfold,lone,region,fut_train_start_time,fut_train_end_time,file_j_range,hist_file_start_time,hist_file_end_time,hist_train_start_time,hist_train_end_time,lats,late,lons,late, basedir,method,target_file_start_time,target_file_end_time,target_train_start_time,target_train_end_time,spat_mask,fut_file_start_time,fut_file_end_time,predictor,target,params,outdir,dversion,dexper,target_scenario,target_model,target_freq,hist_scenario,hist_model,hist_freq,fut_scenario,fut_model,fut_freq,hist_pred_dir,fut_pred_dir,target_dir,expconfig,target_time_window,hist_time_window,fut_time_window,tstamp,ds_region,target_ver,auxcustom= listVars(uinput,basedir,force) ######### call script creators.. ####################################################### ############################### 1 ############################################################### # make.code.tmax.sh 1 748 756 /vftmp/Aparna.Radhakrishnan/pid15769 outdir 1979 2008 tasmax # If the runscript directory already exists, please quit #/home/a1r/gitlab/cew/fudge2014///scripts/tasmax/35txa-CDFt-A00X01K02 scriptdir = basedir+"/scripts/"+target+"/"+expconfig if(os.path.exists(scriptdir)): print "Warning: script directory already exists.",scriptdir #print "ERROR: Directory already exists. Clean up and try again please:",scriptdir script1Loc = basedir+"/utils/bin/create_runcode" make_code_cmd = script1Loc+" "+str(predictor)+" "+str(target)+" "+str(output_grid)+" "+str(spat_mask)+" "+str(region) make_code_cmd = make_code_cmd+" "+str(file_j_range)+" "+str(lons)+" "+str(lats)+" "+str(late) make_code_cmd = make_code_cmd+" "+str(hist_file_start_time)+" "+str(hist_file_end_time)+" "+str(hist_train_start_time)+" "+str(hist_train_end_time)+" "+str(hist_scenario)+" "+str(hist_model)+" "+hist_freq+" "+str(hist_pred_dir)+" "+str(hist_time_window) make_code_cmd = make_code_cmd+" "+str(fut_file_start_time)+" "+str(fut_file_end_time)+" "+str(fut_train_start_time)+" "+str(fut_train_end_time)+" "+str(fut_scenario)+" "+str(fut_model)+" "+fut_freq+" "+str(fut_pred_dir)+" "+str(fut_time_window) make_code_cmd = make_code_cmd+" "+str(target_file_start_time)+" "+str(target_file_end_time)+" "+str(target_train_start_time)+" "+str(target_train_end_time)+" "+str(target_scenario)+" "+str(target_model)+" "+target_freq+" "+str(target_dir)+" "+str(target_time_window) make_code_cmd = make_code_cmd+" "+str(method)+" "+str(expconfig)+" "+str(kfold)+" "+str(outdir)+" "+str(tstamp)+" "+'na'+" "+basedir+" "+str(lone) print "Step 1: R code starters generation ..in progress" #p = subprocess.Popen('tcsh -c "'+make_code_cmd+'"',shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE) params_new = '"'+str(params)+'\"' make_code_cmd = make_code_cmd +" "+params_new+" "+"'"+str(ds_region)+"'" make_code_cmd = make_code_cmd+" "+str(auxcustom) #cprint make_code_cmd #p = subprocess.Popen(make_code_cmd +" "+params_new,shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE) p = subprocess.Popen(make_code_cmd,shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE) output, errors = p.communicate() if(p.returncode != 0): print "Step1:!!!! FAILED !!!!, please contact developer." print output, errors sys.exit(0) #cprint output, errors #cprint "----Log-----" #cprint output,errors ############################################################################################### print "1- completed\n" ############################### 2 ################################################################ #target_time_window,hist_time_window,fut_time_window script2Loc = basedir+"/utils/bin/"+"create_runscript" create_runscript_cmd = script2Loc+" "+str(lons)+" "+str(lone)+" "+str(expconfig)+" "+str(basedir)+" "+target+" "+method+" "+target_dir+" "+hist_pred_dir+" "+fut_pred_dir+" "+outdir+" "+str(file_j_range)+" "+tstamp+" "+str(target_file_start_time)+" "+str(target_file_end_time)+" "+str(hist_file_start_time)+" "+str(hist_file_end_time)+" "+str(fut_file_start_time)+" "+str(fut_file_end_time)+" "+str(spat_mask)+" "+str(region)+" "+auxcustom+" "+target_time_window+" "+hist_time_window+" "+fut_time_window print "Step 2: Individual Runscript generation: \n"+create_runscript_cmd p1 = subprocess.Popen('tcsh -c "'+create_runscript_cmd+'"',shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE) print "Step 2: Individual runscript creation.. in progress" output1, errors1 = p1.communicate() #cprint output1,errors1 print "2- completed\n" if(p1.returncode != 0): print "Step2:!!!! FAILED !!!!, please contact developer." print output1, errors1 sys.exit(0) #print output1, errors1 #c print errors1 ###################################### 3 ################################################################ script3Loc = basedir+"/utils/bin/"+"create_master_runscript" create_master_cmd= script3Loc+" "+str(lons)+" "+str(lone)+" "+str(predictor)+" "+method+" "+basedir+" "+expconfig+" "+file_j_range+" "+tstamp+" "+str(ppn) print "Step 3: --------------MASTER SCRIPT GENERATION-----------------------"#+create_master_cmd p2 = subprocess.Popen('tcsh -c "'+create_master_cmd+'"',shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE) #cprint create_master_cmd #print "Create master script .. in progress" output2, errors2 = p2.communicate() ####### if(p2.returncode != 0): print "Step3:!!!! FAILED !!!!, please contact developer." print output2, errors2 sys.exit(0) print output2, errors2 print "3- completed" ##################################### 4 ############################################ # copy xml to configs dir cdir = basedir+"/"+"/scripts/"+predictor+"/"+expconfig+"/config/" try: os.mkdir(cdir) except: print "Unable to create dir. Dir may exist already", cdir shutil.copy2(uinput, cdir) print "Config XML saved in ",cdir print "----See readMe in fudge2014 directory for the next steps----"
def main(): #################### args parsing ############ help = "#################Usage:(run from AN nodes)##################\n dsTemplater -i <input XML> \n " help = help + "Example 1: expergen -i dstemplate.xml \n " help = help + "Example 2: expergen -i examples/dstemplate.60lo0FUTURE.xml \n" #print usage basedir = os.environ.get('BASEDIR') if(basedir is None): print "Warning: BASEDIR environment variable not set" ##### get version BRANCH info ###################### branch = os.environ.get('BRANCH') if(branch is None): print "Warning: BRANCH env variable not set. BRANCH will be set to undefined" branch = "undefined" #################################################### parser = OptionParser(usage=help) parser.add_option("-i", "--file", dest="uinput", help="pass location of XML template", metavar="FILE") #since we now have an XML tag parser.add_option("-f", "--force",action="store_true",default=False, help="Force override existing output. Default is set to FALSE") parser.add_option("--msub", "--msub",action="store_true",default=False, help="Automatically submit the master runscripts using msub.Default is set to FALSE") #parser.add_option("-v", "--version",action="store_true", dest="version",default="v20120422", help="Assign version for downscaled data. Default is set to v20120422") (options, args) = parser.parse_args() verOpt = True #default msub = False forOpt = True #default inter = 'on' #for debugging #########if platform is not PAN, disable expergen at this time 11/14/2014 ############### system,node,release,version,machine = os.uname() if(node.startswith('pp')): print "Running on PP(PAN) node", node if(node.startswith('an')): print "Running on AN(PAN) node", node else: print "\033[1;41mERROR code -5: Running on a workstation not tested for end-to-end runs yet. Please login to analysis nodes to run expergen. \033[1;m",node sys.exit(-5) ######################################################################################### if (inter == 'off'): uinput = 'dstemplate.xml' dversion = "v20120422" force = False if os.path.exists(uinput): parser = xml.sax.make_parser( ) handler = temphandler.TempHandler( ) parser.setContentHandler(handler) parser.parse(uinput) if (inter == 'on'): for opts,vals in options.__dict__.items(): # print opts,vals if(opts == 'uinput'): uinput = vals if os.path.exists(uinput): print "XML input:",uinput #parser = xml.sax.make_parser( ) #handler = temphandler.TempHandler( ) #parser.setContentHandler(handler) #parser.parse(uinput) else: print "Please pass a valid input XML filename with the -i argument and try again. See -h for syntax. Quitting now.." sys.exit() if(opts == 'msub'): if (vals == True): msub = True else: msub = False msub = vals ######### call listVars() ############################################################# output_grid,kfold,lone,region,fut_train_start_time,fut_train_end_time,file_j_range,hist_file_start_time,hist_file_end_time,hist_train_start_time,hist_train_end_time,lats,late,lons,late, basedir,method,target_file_start_time,target_file_end_time,target_train_start_time,target_train_end_time,spat_mask,fut_file_start_time,fut_file_end_time,predictor,target,params,outdir,dversion,dexper,target_scenario,target_model,target_freq,hist_scenario,hist_model,hist_freq,fut_scenario,fut_model,fut_freq,hist_pred_dir,fut_pred_dir,target_dir,expconfig,target_time_window,hist_time_window,fut_time_window,tstamp,ds_region,target_ver,auxcustom,qc_mask,qc_varname,qc_type,adjust_out,sbase,pr_opts,masklists= listVars(uinput,basedir,msub) ######### call script creators.. ####################################################### ############################### 1 ############################################################### # make.code.tmax.sh 1 748 756 /vftmp/Aparna.Radhakrishnan/pid15769 outdir 1979 2008 tasmax # If the runscript directory already exists, please quit #/home/a1r/gitlab/cew/fudge2014///scripts/tasmax/35txa-CDFt-A00X01K02 scriptdir = [sbase+"/master",sbase+"/runcode",sbase+"/runscript"] for sd in scriptdir: if(os.path.exists(sd)): if os.listdir(sd): if (overwrite == False): print '\033[1;41mERROR: Scripts Directory already exists. Clean up using cleanup_script and try again please -or- (Use <ifpreexist>erase</ifpreexist> if output/scripts already exist and you want to override this and let expergen run the cleanup_script for you; Use <ifpreexist>move</ifpreexist> to move existing output)\033[1;m',sd print "\033[1;41mERROR code -6: script directory already exists.Check --\033[1;m",scriptdir sys.exit(-6) if (overwrite == True): print "\033[1;43mWarning: Scripts Directory already exists. But, since <ifpreexist> has different settings, cleanup utility will handle this\033[1;m " print "Now invoking cleanup utility..........." cleaner_script = basedir+"/utils/bin/"+"cleanup_script.csh" if(preexist_glob == 'erase'): print "Lets erase it ........... ifpreexist" cleaner_cmd = cleaner_script+" d "+uinput else: if(preexist_glob != 'move') & (preexist_glob != 'exit'): print "CHECK ifpreexist settings, quitting now" sys.exit(1) print "Lets move it............... if preexist" cleaner_cmd = cleaner_script+" m "+uinput print "cleaner_cmd" print cleaner_cmd pclean = subprocess.Popen(cleaner_cmd,shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE) #check return code output0, errors0 = pclean.communicate() if(pclean.returncode != 0): print "033[1;41mCleaner Step:!!!! FAILED !!!!, please contact developer.033[1;m" print output0, errors0 sys.exit(1) else: print output0,errors0 print "continue expergen run...................." break else: print "scriptdir "+sd+" does not exist. Looks like a clean slate " #print "hist_freq"+hist_freq script1Loc = basedir+"/utils/bin/create_runcode" make_code_cmd = script1Loc+" "+str(predictor)+" "+str(target)+" "+str(output_grid)+" "+str(spat_mask)+" "+str(region) make_code_cmd = make_code_cmd+" "+str(file_j_range)+" "+str(lons)+" "+str(lats)+" "+str(late) make_code_cmd = make_code_cmd+" "+str(hist_file_start_time)+" "+str(hist_file_end_time)+" "+str(hist_train_start_time)+" "+str(hist_train_end_time)+" "+str(hist_scenario)+" "+str(hist_model)+" "+hist_freq+" "+str(hist_pred_dir)+" "+str(hist_time_window) make_code_cmd = make_code_cmd+" "+str(fut_file_start_time)+" "+str(fut_file_end_time)+" "+str(fut_train_start_time)+" "+str(fut_train_end_time)+" "+str(fut_scenario)+" "+str(fut_model)+" "+fut_freq+" "+str(fut_pred_dir)+" "+str(fut_time_window) make_code_cmd = make_code_cmd+" "+str(target_file_start_time)+" "+str(target_file_end_time)+" "+str(target_train_start_time)+" "+str(target_train_end_time)+" "+str(target_scenario)+" "+str(target_model)+" "+target_freq+" "+str(target_dir)+" "+str(target_time_window) make_code_cmd = make_code_cmd+" "+str(method)+" "+str(expconfig)+" "+str(kfold)+" "+str(outdir)+" "+str(tstamp)+" "+'na'+" "+basedir+" "+str(lone) print "Step 1: R code starters generation ..in progress" #p = subprocess.Popen('tcsh -c "'+make_code_cmd+'"',shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE) params_new = '"'+str(params)+'\"' params_pr_opts = '"'+str(pr_opts)+'\"' make_code_cmd = make_code_cmd +" "+params_new+" "+"'"+str(ds_region)+"'" make_code_cmd = make_code_cmd+" "+str(auxcustom)+" "+str(qc_mask)+" "+str(qc_varname)+" "+str(qc_type)+" "+str(adjust_out)+" "+str(sbase)+" "+str(params_pr_opts)+" "+str(branch)+" "+'"'+str(masklists)+'"' #cprint make_code_cmd #p = subprocess.Popen(make_code_cmd +" "+params_new,shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE) p = subprocess.Popen(make_code_cmd,shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE) output, errors = p.communicate() if(p.returncode != 0): print "Step1:!!!! FAILED !!!!, please contact developer." print output, errors sys.exit(0) #cprint output, errors #cprint "----Log-----" #cprint output,errors ############################################################################################### print "1- completed\n" #print "debug............msub turned ",msub ############################### 2 ################################################################ #target_time_window,hist_time_window,fut_time_window script2Loc = basedir+"/utils/bin/"+"create_runscript" create_runscript_cmd = script2Loc+" "+str(lons)+" "+str(lone)+" "+str(expconfig)+" "+str(basedir)+" "+target+" "+method+" "+target_dir+" "+hist_pred_dir+" "+fut_pred_dir+" "+outdir+" "+str(file_j_range)+" "+tstamp+" "+str(target_file_start_time)+" "+str(target_file_end_time)+" "+str(hist_file_start_time)+" "+str(hist_file_end_time)+" "+str(fut_file_start_time)+" "+str(fut_file_end_time)+" "+str(spat_mask)+" "+str(region)+" "+auxcustom+" "+target_time_window+" "+hist_time_window+" "+fut_time_window+" "+sbase print "Step 2: Individual Runscript generation: \n"+create_runscript_cmd p1 = subprocess.Popen('tcsh -c "'+create_runscript_cmd+'"',shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE) print "Step 2: Individual runscript creation.. in progress" output1, errors1 = p1.communicate() #cprint output1,errors1 print "2- completed\n" if(p1.returncode != 0): print "Step2:!!!! FAILED !!!!, please contact developer." print output1, errors1 sys.exit(0) #print output1, errors1 #c print errors1 ###################################### 3 ################################################################ script3Loc = basedir+"/utils/bin/"+"create_master_runscript" create_master_cmd= script3Loc+" "+str(lons)+" "+str(lone)+" "+str(predictor)+" "+method+" "+sbase+" "+expconfig+" "+file_j_range+" "+tstamp+" "+str(ppn)+" "+str(msub) print "Step 3: --------------MASTER SCRIPT GENERATION-----------------------"#+create_master_cmd p2 = subprocess.Popen('tcsh -c "'+create_master_cmd+'"',shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE) #cprint create_master_cmd print "Create master script .. in progress" output2, errors2 = p2.communicate() ####### if(p2.returncode != 0): print "Step3:!!!! FAILED !!!!, please contact developer." print output2, errors2 sys.exit(0) print output2, errors2 print "3- completed" ##################################### 4 ############################################ # copy xml to configs dir cdir = sbase+"/config/" try: os.mkdir(cdir) except: print "Unable to create dir. Dir may exist already", cdir shutil.copy2(uinput, cdir) print "Config XML saved in ",cdir print "RunScripts will be saved under:",sbase print "----See readMe in fudge2014 directory for the next steps----" print "Use submit_job to submit scripts" ############### crte ppscript ################# dev = "off" print(sbase+"/postProc/aux/") if not os.path.exists(sbase+"/postProc/aux/"): os.makedirs(sbase+"/postProc/aux/") if (dev == "off"): tsuffix = "" ppbase = sbase+"/postProc/aux/"+"/postProc_source" else: tsuffix = "_"+tstamp ppbase = sbase+"/postProc/aux/"+"/postProc_source"+tstamp try: ppfile = open(ppbase, 'w') #check if qc_mask is relevant if(qc_mask != 'off'): pp_cmnd = "python $BASEDIR/bin/postProc -i "+os.path.abspath(uinput)+" -v "+target+","+target+"_qcmask\n" else: pp_cmnd = "python $BASEDIR/bin/postProc -i "+os.path.abspath(uinput)+" -v "+target+"\n" ppfile.write(pp_cmnd) ppfile.close() except: print "Unable to create postProc command file. You may want to check your settings." #c print create_pp_cmd if(os.path.exists(ppbase)): ######################### write postProc_job to be used ############################ ppLoc = basedir+"/utils/bin/"+"create_postProc" create_pp_cmd= ppLoc+" "+ppbase+" "+sbase+" "+basedir+" "+tstamp+" "+branch print "Step 4: --------------PP postProc SCRIPT GENERATION-----------------------" p4 = subprocess.Popen('tcsh -c "'+create_pp_cmd+'"',shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE) output4, error4 = p4.communicate() if(p4.returncode != 0): print "Step4:!!!! FAILED !!!!, please contact developer." print output4, error4 sys.exit(-4) print output4, error4 print "4- completed" print "NOTE: postProc will succeed only if you're running the model for the full downscaled region. (it will fail if you're running downscaling for a single slice for example)" print "----------------------------------------" print "\033[1;42mPlease use this script to run post post-processing (or msub this script), postProc when downscaling jobs are complete \033[1;m",sbase+"postProc/postProc_command"+tsuffix try: NEMSemail = os.environ["NEMSemail"] print "msub -m ae -M "+os.environ.get('NEMSemail')+" "+sbase+"postProc/postProc_command"+tsuffix except KeyError: print "NEMSemail not set. Please use your email for notification in the following msub command i.e msub -m ae -M <email> script " print "msub "+sbase+"postProc/postProc_command"+tsuffix else: print "postProc_command cannot be created. postProc_source does not exist" ################ step 5 fudgeList invocation ############################################## slogloc = sbase+"/"+"experiment_info.txt" fcmd = "python "+basedir+"/bin/fudgeList.py -f -i "+uinput+" -o "+slogloc f = subprocess.Popen(fcmd, stdout=subprocess.PIPE, shell=True) out, err = f.communicate() #print "fudgeList out", out #if err is not None: # print "fudgeList err", err print "Summary Log File: ", slogloc