def main(): from optparse import OptionParser parser = OptionParser( usage="usage: %prog [options] network1 network2...", version="%prog 1.4\n"+_copyright, description=_description ) parser.add_option("-m", "--manifest", dest='createManifest', action='store_true', help="Generates the manifest file") parser.add_option("-t", "--ttl", dest='createTtls', action='store_true', help="Generates ttl files for each network") parser.add_option("-i", "--main", dest='createMain', action='store_true', help="Generates the main C++ file for the plugin library") parser.add_option("-u", "--uribase", dest='uribase', default="http://clam-project.org/examples/lv2", help="Specifies the uri base for the plugins", metavar="URIBASE") parser.add_option("-d", "--doap", dest='doapfile', help="Specifies a doapfile with additional info when generating a ttl", metavar="DOAPFILE") parser.add_option("-y", "--binary", dest='binary', help="Provides a name for the library binary (required by --ttls)", metavar="LIBBINARY") parser.add_option("-g", "--gui-binary", dest='guibinary', default=None, help="Provides a name for a ui library binary for the plugin in the ttl", metavar="UIBINARY") options, networks = parser.parse_args() names = [os.path.splitext(os.path.basename(network))[0] for network in networks] uris = [os.path.join(options.uribase,name) for name in names ] if options.createTtls: for network, uri, name in zip(networks, uris, names) : parser = make_parser() curHandler = ExporterHandler() parser.setContentHandler(curHandler) parser.parse(open(network)) curHandler.printTTL(network,uri,name,options.doapfile,options.binary,options.guibinary) return if options.createManifest: printManifest(uris,names) return if options.createMain: printCLAM_PLUGIN(networks,uris) return
def main(): #################### args parsing ############ help = "#################Usage:##################\n dsTemplater -i <input XML> \n " help = help + "Example 1: dsTemplater -i dstemplate.xml \n " help = help + "Example 2: dsTemplater -i examples/dstemplate.60lo0FUTURE.xml \n" #print usage basedir = os.environ.get('BASEDIR') if(basedir is None): print "Warning: BASEDIR environment variable not set" parser = OptionParser(usage=help) parser.add_option("-i", "--file", dest="uinput", help="pass location of XML template", metavar="FILE") parser.add_option("-f", "--force",action="store_true",default=False, help="Force override existing output. Default is set to FALSE") #parser.add_option("-v", "--version",action="store_true", dest="version",default="v20120422", help="Assign version for downscaled data. Default is set to v20120422") (options, args) = parser.parse_args() verOpt = True #default forOpt = True #default inter = 'on' #for debugging if (inter == 'off'): uinput = 'dstemplate.xml' dversion = "v20120422" force = False if os.path.exists(uinput): parser = xml.sax.make_parser( ) handler = temphandler.TempHandler( ) parser.setContentHandler(handler) parser.parse(uinput) if (inter == 'on'): for opts,vals in options.__dict__.items(): # print opts,vals if(opts == 'uinput'): uinput = vals if os.path.exists(uinput): print "XML input:",uinput #parser = xml.sax.make_parser( ) #handler = temphandler.TempHandler( ) #parser.setContentHandler(handler) #parser.parse(uinput) else: print "Please pass a valid input XML filename with the -i argument and try again. See -h for syntax. Quitting now.." sys.exit() if(opts == 'force'): if (vals == True): forOpt = False force = vals ######### call listVars() ############################################################# output_grid,kfold,lone,region,fut_train_start_time,fut_train_end_time,file_j_range,hist_file_start_time,hist_file_end_time,hist_train_start_time,hist_train_end_time,lats,late,lons,late, basedir,method,target_file_start_time,target_file_end_time,target_train_start_time,target_train_end_time,spat_mask,fut_file_start_time,fut_file_end_time,predictor,target,params,outdir,dversion,dexper,target_scenario,target_model,target_freq,hist_scenario,hist_model,hist_freq,fut_scenario,fut_model,fut_freq,hist_pred_dir,fut_pred_dir,target_dir,expconfig,target_time_window,hist_time_window,fut_time_window,tstamp,ds_region,target_ver,auxcustom= listVars(uinput,basedir,force) ######### call script creators.. ####################################################### ############################### 1 ############################################################### # make.code.tmax.sh 1 748 756 /vftmp/Aparna.Radhakrishnan/pid15769 outdir 1979 2008 tasmax # If the runscript directory already exists, please quit #/home/a1r/gitlab/cew/fudge2014///scripts/tasmax/35txa-CDFt-A00X01K02 scriptdir = basedir+"/scripts/"+target+"/"+expconfig if(os.path.exists(scriptdir)): print "Warning: script directory already exists.",scriptdir #print "ERROR: Directory already exists. Clean up and try again please:",scriptdir script1Loc = basedir+"/utils/bin/create_runcode" make_code_cmd = script1Loc+" "+str(predictor)+" "+str(target)+" "+str(output_grid)+" "+str(spat_mask)+" "+str(region) make_code_cmd = make_code_cmd+" "+str(file_j_range)+" "+str(lons)+" "+str(lats)+" "+str(late) make_code_cmd = make_code_cmd+" "+str(hist_file_start_time)+" "+str(hist_file_end_time)+" "+str(hist_train_start_time)+" "+str(hist_train_end_time)+" "+str(hist_scenario)+" "+str(hist_model)+" "+hist_freq+" "+str(hist_pred_dir)+" "+str(hist_time_window) make_code_cmd = make_code_cmd+" "+str(fut_file_start_time)+" "+str(fut_file_end_time)+" "+str(fut_train_start_time)+" "+str(fut_train_end_time)+" "+str(fut_scenario)+" "+str(fut_model)+" "+fut_freq+" "+str(fut_pred_dir)+" "+str(fut_time_window) make_code_cmd = make_code_cmd+" "+str(target_file_start_time)+" "+str(target_file_end_time)+" "+str(target_train_start_time)+" "+str(target_train_end_time)+" "+str(target_scenario)+" "+str(target_model)+" "+target_freq+" "+str(target_dir)+" "+str(target_time_window) make_code_cmd = make_code_cmd+" "+str(method)+" "+str(expconfig)+" "+str(kfold)+" "+str(outdir)+" "+str(tstamp)+" "+'na'+" "+basedir+" "+str(lone) print "Step 1: R code starters generation ..in progress" #p = subprocess.Popen('tcsh -c "'+make_code_cmd+'"',shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE) params_new = '"'+str(params)+'\"' make_code_cmd = make_code_cmd +" "+params_new+" "+"'"+str(ds_region)+"'" make_code_cmd = make_code_cmd+" "+str(auxcustom) #cprint make_code_cmd #p = subprocess.Popen(make_code_cmd +" "+params_new,shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE) p = subprocess.Popen(make_code_cmd,shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE) output, errors = p.communicate() if(p.returncode != 0): print "Step1:!!!! FAILED !!!!, please contact developer." print output, errors sys.exit(0) #cprint output, errors #cprint "----Log-----" #cprint output,errors ############################################################################################### print "1- completed\n" ############################### 2 ################################################################ #target_time_window,hist_time_window,fut_time_window script2Loc = basedir+"/utils/bin/"+"create_runscript" create_runscript_cmd = script2Loc+" "+str(lons)+" "+str(lone)+" "+str(expconfig)+" "+str(basedir)+" "+target+" "+method+" "+target_dir+" "+hist_pred_dir+" "+fut_pred_dir+" "+outdir+" "+str(file_j_range)+" "+tstamp+" "+str(target_file_start_time)+" "+str(target_file_end_time)+" "+str(hist_file_start_time)+" "+str(hist_file_end_time)+" "+str(fut_file_start_time)+" "+str(fut_file_end_time)+" "+str(spat_mask)+" "+str(region)+" "+auxcustom+" "+target_time_window+" "+hist_time_window+" "+fut_time_window print "Step 2: Individual Runscript generation: \n"+create_runscript_cmd p1 = subprocess.Popen('tcsh -c "'+create_runscript_cmd+'"',shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE) print "Step 2: Individual runscript creation.. in progress" output1, errors1 = p1.communicate() #cprint output1,errors1 print "2- completed\n" if(p1.returncode != 0): print "Step2:!!!! FAILED !!!!, please contact developer." print output1, errors1 sys.exit(0) #print output1, errors1 #c print errors1 ###################################### 3 ################################################################ script3Loc = basedir+"/utils/bin/"+"create_master_runscript" create_master_cmd= script3Loc+" "+str(lons)+" "+str(lone)+" "+str(predictor)+" "+method+" "+basedir+" "+expconfig+" "+file_j_range+" "+tstamp+" "+str(ppn) print "Step 3: --------------MASTER SCRIPT GENERATION-----------------------"#+create_master_cmd p2 = subprocess.Popen('tcsh -c "'+create_master_cmd+'"',shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE) #cprint create_master_cmd #print "Create master script .. in progress" output2, errors2 = p2.communicate() ####### if(p2.returncode != 0): print "Step3:!!!! FAILED !!!!, please contact developer." print output2, errors2 sys.exit(0) print output2, errors2 print "3- completed" ##################################### 4 ############################################ # copy xml to configs dir cdir = basedir+"/"+"/scripts/"+predictor+"/"+expconfig+"/config/" try: os.mkdir(cdir) except: print "Unable to create dir. Dir may exist already", cdir shutil.copy2(uinput, cdir) print "Config XML saved in ",cdir print "----See readMe in fudge2014 directory for the next steps----"
def endElement(self, name): if name == "ch1" or name == "ch2" or name == "watts" or name == "data": print "no" if name == "msg": self.outFile.write("\n") else: self.outFile.write(",") def characters(self, ch): self.outFile.write( ch.strip("\n\t,") ) if __name__ == "__main__": parser = OptionParser() parser.add_option("-f", "--file", dest="filename", help="write report to FILE", metavar="FILE", default="none") (options, args) = parser.parse_args() if options.filename == "none": exit() parser = xml.sax.make_parser() fileCSV = open("dump.csv", "w") handler = XML2CSV(fileCSV) parser.setContentHandler(handler) parser.parse(options.filename)
option_crystal = options.crystal option_session = options.session option_cookie = options.cookie setOptions(options) else: try: f = open("grabber.conf.xml", 'r') except IOError: print "No arguments ? You need to setup the XML configuration file or using the inline arguments" print "Look at the doc to start..." sys.exit(1) parser = make_parser() conf_handler = ConfHandler() # Tell the parser to use our handler parser.setContentHandler(conf_handler) parser.parse("grabber.conf.xml") option_url = confUrl option_spider = confSpider option_sql = "sql" in confActions option_bsql = "bsql" in confActions option_xss = "xss" in confActions option_backup = "backup" in confActions option_include = "include" in confActions option_js = "javascript" in confActions option_crystal = "crystal" in confActions option_session = "session" in confActions # default to localhost ? archives_url = "http://localhost"
parser.add_option("-d", "--key-directory", default="", dest="key_directory", help="Specify a parent directory for keys") (options, args) = parser.parse_args() if len(args) < 2: parser.error("Must specify a config file (keys.conf) AND mac_permissions.xml file(s)!") logging.basicConfig(level=logging.INFO if options.verbose == True else logging.WARN) # Read the config file config = ParseConfig() config.read(args[0]) os.chdir(options.root) output_file = sys.stdout if options.output_file == "stdout" else open(options.output_file, "w") logging.info("Setting output file to: " + options.output_file) # Generate the key list key_map = config.generateKeyMap(options.target_build_variant.lower(), options.key_directory) logging.info("Generate key map:") for k in key_map: logging.info(k + " : " + str(key_map[k])) # Generate the XML file with markup replaced with keys parser = make_parser() parser.setContentHandler(ReplaceTags(key_map, output_file)) for f in args[1:]: parser.parse(f)
action="store_true", dest="reformat", default=False, help="run reformat scripts for the observations according to namelist") options, args = parser.parse_args() if len(args) == 0: parser.print_help() sys.exit(0) # Get command arguments. input_xml_full_path = args[0] # Parse input namelist into project_info-dictionary. Project = xml_parsers.namelistHandler() parser = xml.sax.make_parser() parser.setContentHandler(Project) parser.parse(input_xml_full_path) # Project_info is a dictionary with all info from the namelist. project_info = Project.project_info if options.reformat: if 'REFORMAT' not in project_info.keys(): error('No REFORMAT tag specified in {0}'.format(input_xml_full_path)) if len(project_info['REFORMAT']) == 0: info('No reformat script specified', 1, 1) print_header({}, options.reformat) for k, v in project_info['REFORMAT'].iteritems(): if not os.path.exists(v): error('Path {0} does not exist'.format(v)) projects.run_executable(v, project_info, 1, False, write_di=False)
def main(): #################### args parsing ############ help = "#################Usage:(run from AN nodes)##################\n dsTemplater -i <input XML> \n " help = help + "Example 1: expergen -i dstemplate.xml \n " help = help + "Example 2: expergen -i examples/dstemplate.60lo0FUTURE.xml \n" #print usage basedir = os.environ.get('BASEDIR') if(basedir is None): print "Warning: BASEDIR environment variable not set" ##### get version BRANCH info ###################### branch = os.environ.get('BRANCH') if(branch is None): print "Warning: BRANCH env variable not set. BRANCH will be set to undefined" branch = "undefined" #################################################### parser = OptionParser(usage=help) parser.add_option("-i", "--file", dest="uinput", help="pass location of XML template", metavar="FILE") #since we now have an XML tag parser.add_option("-f", "--force",action="store_true",default=False, help="Force override existing output. Default is set to FALSE") parser.add_option("--msub", "--msub",action="store_true",default=False, help="Automatically submit the master runscripts using msub.Default is set to FALSE") #parser.add_option("-v", "--version",action="store_true", dest="version",default="v20120422", help="Assign version for downscaled data. Default is set to v20120422") (options, args) = parser.parse_args() verOpt = True #default msub = False forOpt = True #default inter = 'on' #for debugging #########if platform is not PAN, disable expergen at this time 11/14/2014 ############### system,node,release,version,machine = os.uname() if(node.startswith('pp')): print "Running on PP(PAN) node", node if(node.startswith('an')): print "Running on AN(PAN) node", node else: print "\033[1;41mERROR code -5: Running on a workstation not tested for end-to-end runs yet. Please login to analysis nodes to run expergen. \033[1;m",node sys.exit(-5) ######################################################################################### if (inter == 'off'): uinput = 'dstemplate.xml' dversion = "v20120422" force = False if os.path.exists(uinput): parser = xml.sax.make_parser( ) handler = temphandler.TempHandler( ) parser.setContentHandler(handler) parser.parse(uinput) if (inter == 'on'): for opts,vals in options.__dict__.items(): # print opts,vals if(opts == 'uinput'): uinput = vals if os.path.exists(uinput): print "XML input:",uinput #parser = xml.sax.make_parser( ) #handler = temphandler.TempHandler( ) #parser.setContentHandler(handler) #parser.parse(uinput) else: print "Please pass a valid input XML filename with the -i argument and try again. See -h for syntax. Quitting now.." sys.exit() if(opts == 'msub'): if (vals == True): msub = True else: msub = False msub = vals ######### call listVars() ############################################################# output_grid,kfold,lone,region,fut_train_start_time,fut_train_end_time,file_j_range,hist_file_start_time,hist_file_end_time,hist_train_start_time,hist_train_end_time,lats,late,lons,late, basedir,method,target_file_start_time,target_file_end_time,target_train_start_time,target_train_end_time,spat_mask,fut_file_start_time,fut_file_end_time,predictor,target,params,outdir,dversion,dexper,target_scenario,target_model,target_freq,hist_scenario,hist_model,hist_freq,fut_scenario,fut_model,fut_freq,hist_pred_dir,fut_pred_dir,target_dir,expconfig,target_time_window,hist_time_window,fut_time_window,tstamp,ds_region,target_ver,auxcustom,qc_mask,qc_varname,qc_type,adjust_out,sbase,pr_opts,masklists= listVars(uinput,basedir,msub) ######### call script creators.. ####################################################### ############################### 1 ############################################################### # make.code.tmax.sh 1 748 756 /vftmp/Aparna.Radhakrishnan/pid15769 outdir 1979 2008 tasmax # If the runscript directory already exists, please quit #/home/a1r/gitlab/cew/fudge2014///scripts/tasmax/35txa-CDFt-A00X01K02 scriptdir = [sbase+"/master",sbase+"/runcode",sbase+"/runscript"] for sd in scriptdir: if(os.path.exists(sd)): if os.listdir(sd): if (overwrite == False): print '\033[1;41mERROR: Scripts Directory already exists. Clean up using cleanup_script and try again please -or- (Use <ifpreexist>erase</ifpreexist> if output/scripts already exist and you want to override this and let expergen run the cleanup_script for you; Use <ifpreexist>move</ifpreexist> to move existing output)\033[1;m',sd print "\033[1;41mERROR code -6: script directory already exists.Check --\033[1;m",scriptdir sys.exit(-6) if (overwrite == True): print "\033[1;43mWarning: Scripts Directory already exists. But, since <ifpreexist> has different settings, cleanup utility will handle this\033[1;m " print "Now invoking cleanup utility..........." cleaner_script = basedir+"/utils/bin/"+"cleanup_script.csh" if(preexist_glob == 'erase'): print "Lets erase it ........... ifpreexist" cleaner_cmd = cleaner_script+" d "+uinput else: if(preexist_glob != 'move') & (preexist_glob != 'exit'): print "CHECK ifpreexist settings, quitting now" sys.exit(1) print "Lets move it............... if preexist" cleaner_cmd = cleaner_script+" m "+uinput print "cleaner_cmd" print cleaner_cmd pclean = subprocess.Popen(cleaner_cmd,shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE) #check return code output0, errors0 = pclean.communicate() if(pclean.returncode != 0): print "033[1;41mCleaner Step:!!!! FAILED !!!!, please contact developer.033[1;m" print output0, errors0 sys.exit(1) else: print output0,errors0 print "continue expergen run...................." break else: print "scriptdir "+sd+" does not exist. Looks like a clean slate " #print "hist_freq"+hist_freq script1Loc = basedir+"/utils/bin/create_runcode" make_code_cmd = script1Loc+" "+str(predictor)+" "+str(target)+" "+str(output_grid)+" "+str(spat_mask)+" "+str(region) make_code_cmd = make_code_cmd+" "+str(file_j_range)+" "+str(lons)+" "+str(lats)+" "+str(late) make_code_cmd = make_code_cmd+" "+str(hist_file_start_time)+" "+str(hist_file_end_time)+" "+str(hist_train_start_time)+" "+str(hist_train_end_time)+" "+str(hist_scenario)+" "+str(hist_model)+" "+hist_freq+" "+str(hist_pred_dir)+" "+str(hist_time_window) make_code_cmd = make_code_cmd+" "+str(fut_file_start_time)+" "+str(fut_file_end_time)+" "+str(fut_train_start_time)+" "+str(fut_train_end_time)+" "+str(fut_scenario)+" "+str(fut_model)+" "+fut_freq+" "+str(fut_pred_dir)+" "+str(fut_time_window) make_code_cmd = make_code_cmd+" "+str(target_file_start_time)+" "+str(target_file_end_time)+" "+str(target_train_start_time)+" "+str(target_train_end_time)+" "+str(target_scenario)+" "+str(target_model)+" "+target_freq+" "+str(target_dir)+" "+str(target_time_window) make_code_cmd = make_code_cmd+" "+str(method)+" "+str(expconfig)+" "+str(kfold)+" "+str(outdir)+" "+str(tstamp)+" "+'na'+" "+basedir+" "+str(lone) print "Step 1: R code starters generation ..in progress" #p = subprocess.Popen('tcsh -c "'+make_code_cmd+'"',shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE) params_new = '"'+str(params)+'\"' params_pr_opts = '"'+str(pr_opts)+'\"' make_code_cmd = make_code_cmd +" "+params_new+" "+"'"+str(ds_region)+"'" make_code_cmd = make_code_cmd+" "+str(auxcustom)+" "+str(qc_mask)+" "+str(qc_varname)+" "+str(qc_type)+" "+str(adjust_out)+" "+str(sbase)+" "+str(params_pr_opts)+" "+str(branch)+" "+'"'+str(masklists)+'"' #cprint make_code_cmd #p = subprocess.Popen(make_code_cmd +" "+params_new,shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE) p = subprocess.Popen(make_code_cmd,shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE) output, errors = p.communicate() if(p.returncode != 0): print "Step1:!!!! FAILED !!!!, please contact developer." print output, errors sys.exit(0) #cprint output, errors #cprint "----Log-----" #cprint output,errors ############################################################################################### print "1- completed\n" #print "debug............msub turned ",msub ############################### 2 ################################################################ #target_time_window,hist_time_window,fut_time_window script2Loc = basedir+"/utils/bin/"+"create_runscript" create_runscript_cmd = script2Loc+" "+str(lons)+" "+str(lone)+" "+str(expconfig)+" "+str(basedir)+" "+target+" "+method+" "+target_dir+" "+hist_pred_dir+" "+fut_pred_dir+" "+outdir+" "+str(file_j_range)+" "+tstamp+" "+str(target_file_start_time)+" "+str(target_file_end_time)+" "+str(hist_file_start_time)+" "+str(hist_file_end_time)+" "+str(fut_file_start_time)+" "+str(fut_file_end_time)+" "+str(spat_mask)+" "+str(region)+" "+auxcustom+" "+target_time_window+" "+hist_time_window+" "+fut_time_window+" "+sbase print "Step 2: Individual Runscript generation: \n"+create_runscript_cmd p1 = subprocess.Popen('tcsh -c "'+create_runscript_cmd+'"',shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE) print "Step 2: Individual runscript creation.. in progress" output1, errors1 = p1.communicate() #cprint output1,errors1 print "2- completed\n" if(p1.returncode != 0): print "Step2:!!!! FAILED !!!!, please contact developer." print output1, errors1 sys.exit(0) #print output1, errors1 #c print errors1 ###################################### 3 ################################################################ script3Loc = basedir+"/utils/bin/"+"create_master_runscript" create_master_cmd= script3Loc+" "+str(lons)+" "+str(lone)+" "+str(predictor)+" "+method+" "+sbase+" "+expconfig+" "+file_j_range+" "+tstamp+" "+str(ppn)+" "+str(msub) print "Step 3: --------------MASTER SCRIPT GENERATION-----------------------"#+create_master_cmd p2 = subprocess.Popen('tcsh -c "'+create_master_cmd+'"',shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE) #cprint create_master_cmd print "Create master script .. in progress" output2, errors2 = p2.communicate() ####### if(p2.returncode != 0): print "Step3:!!!! FAILED !!!!, please contact developer." print output2, errors2 sys.exit(0) print output2, errors2 print "3- completed" ##################################### 4 ############################################ # copy xml to configs dir cdir = sbase+"/config/" try: os.mkdir(cdir) except: print "Unable to create dir. Dir may exist already", cdir shutil.copy2(uinput, cdir) print "Config XML saved in ",cdir print "RunScripts will be saved under:",sbase print "----See readMe in fudge2014 directory for the next steps----" print "Use submit_job to submit scripts" ############### crte ppscript ################# dev = "off" print(sbase+"/postProc/aux/") if not os.path.exists(sbase+"/postProc/aux/"): os.makedirs(sbase+"/postProc/aux/") if (dev == "off"): tsuffix = "" ppbase = sbase+"/postProc/aux/"+"/postProc_source" else: tsuffix = "_"+tstamp ppbase = sbase+"/postProc/aux/"+"/postProc_source"+tstamp try: ppfile = open(ppbase, 'w') #check if qc_mask is relevant if(qc_mask != 'off'): pp_cmnd = "python $BASEDIR/bin/postProc -i "+os.path.abspath(uinput)+" -v "+target+","+target+"_qcmask\n" else: pp_cmnd = "python $BASEDIR/bin/postProc -i "+os.path.abspath(uinput)+" -v "+target+"\n" ppfile.write(pp_cmnd) ppfile.close() except: print "Unable to create postProc command file. You may want to check your settings." #c print create_pp_cmd if(os.path.exists(ppbase)): ######################### write postProc_job to be used ############################ ppLoc = basedir+"/utils/bin/"+"create_postProc" create_pp_cmd= ppLoc+" "+ppbase+" "+sbase+" "+basedir+" "+tstamp+" "+branch print "Step 4: --------------PP postProc SCRIPT GENERATION-----------------------" p4 = subprocess.Popen('tcsh -c "'+create_pp_cmd+'"',shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE) output4, error4 = p4.communicate() if(p4.returncode != 0): print "Step4:!!!! FAILED !!!!, please contact developer." print output4, error4 sys.exit(-4) print output4, error4 print "4- completed" print "NOTE: postProc will succeed only if you're running the model for the full downscaled region. (it will fail if you're running downscaling for a single slice for example)" print "----------------------------------------" print "\033[1;42mPlease use this script to run post post-processing (or msub this script), postProc when downscaling jobs are complete \033[1;m",sbase+"postProc/postProc_command"+tsuffix try: NEMSemail = os.environ["NEMSemail"] print "msub -m ae -M "+os.environ.get('NEMSemail')+" "+sbase+"postProc/postProc_command"+tsuffix except KeyError: print "NEMSemail not set. Please use your email for notification in the following msub command i.e msub -m ae -M <email> script " print "msub "+sbase+"postProc/postProc_command"+tsuffix else: print "postProc_command cannot be created. postProc_source does not exist" ################ step 5 fudgeList invocation ############################################## slogloc = sbase+"/"+"experiment_info.txt" fcmd = "python "+basedir+"/bin/fudgeList.py -f -i "+uinput+" -o "+slogloc f = subprocess.Popen(fcmd, stdout=subprocess.PIPE, shell=True) out, err = f.communicate() #print "fudgeList out", out #if err is not None: # print "fudgeList err", err print "Summary Log File: ", slogloc
# # cleanup / preparation # cmd = 'rm -rf %s/%s' % (QASRC_DIRFN, corpusname) logging.info(cmd) os.system(cmd) cmd = 'mkdir -p %s/%s' % (QASRC_DIRFN, corpusname) logging.info(cmd) os.system(cmd) # # parse XML # parser = make_parser() parser.setContentHandler(WikiContentHandler(WIKIXML[options.lang])) article_cnt = 0 article_tot = 0 wikisize = os.path.getsize(WIKIXML[options.lang]) with open(WIKIXML[options.lang], 'rb') as wikibz2f: # with codecs.getreader('utf8')(BZ2File(WIKIXML[options.lang], 'r')) as wikif: with codecs.getreader('utf8')(BZ2File(wikibz2f)) as wikif: parser.parse(wikif)
if name == "msg": self.outFile.write("\n") else: self.outFile.write(",") def characters(self, ch): self.outFile.write(ch.strip("\n\t,")) if __name__ == "__main__": parser = OptionParser() parser.add_option("-f", "--file", dest="filename", help="write report to FILE", metavar="FILE", default="none") (options, args) = parser.parse_args() if options.filename == "none": exit() parser = xml.sax.make_parser() fileCSV = open("dump.csv", "w") handler = XML2CSV(fileCSV) parser.setContentHandler(handler) parser.parse(options.filename)
elif archiveType == "bz2" or archiveType == "BZ2": os.system('bzip2 -d "%s"' % options.osmInput) options.osmInput = options.osmInput.rstrip("." + archiveType) print "File uncompressed: ", options.osmInput if options.kmlOutput == None: options.kmlOutput = os.path.basename(options.osmInput).rstrip(".osc") if options.kmlVersion == None: options.kmlVersion = "1" myAwareness = OSMaware(debug=False, verbose=False, ele=options.linesElevation, kml_version=options.kmlVersion) parser = make_parser() parser.setContentHandler(myAwareness) t0 = datetime.now() print "Starting parsing OSM file..." parser.parse(options.osmInput) #for userName, userStat in self.statsUsers.iteritems(): print userName, userStat print "Number of contributors (users):", len(myAwareness.statsUsers) print "Number of Nodes created, deleted or modified:", myAwareness.nodeCount print "Number of Ways created, deleted or modified:", myAwareness.wayCount print "Number of Realations created, deleted or modified:", myAwareness.relationCount if options.kmlVersion == "2": myAwareness.createKmlV2(options.kmlOutput, heightFactor=myAwareness.linesElevation) if options.kmlVersion == "1": myAwareness.createKmlV1(options.kmlOutput)
if archiveType == "gz" or archiveType == "GZ": os.system('gunzip "%s"' % options.osmInput) elif archiveType == "bz2" or archiveType == "BZ2": os.system('bzip2 -d "%s"' % options.osmInput) options.osmInput = options.osmInput.rstrip("." + archiveType) print "File uncompressed: ", options.osmInput if options.kmlOutput == None: options.kmlOutput = os.path.basename(options.osmInput).rstrip(".osc") if options.kmlVersion == None: options.kmlVersion = "1" myAwareness = OSMaware(debug=False, verbose=False, ele=options.linesElevation, kml_version=options.kmlVersion) parser = make_parser() parser.setContentHandler(myAwareness) t0 = datetime.now() print "Starting parsing OSM file..." parser.parse(options.osmInput) # for userName, userStat in self.statsUsers.iteritems(): print userName, userStat print "Number of contributors (users):", len(myAwareness.statsUsers) print "Number of Nodes created, deleted or modified:", myAwareness.nodeCount print "Number of Ways created, deleted or modified:", myAwareness.wayCount print "Number of Realations created, deleted or modified:", myAwareness.relationCount if options.kmlVersion == "2": myAwareness.createKmlV2(options.kmlOutput, heightFactor=myAwareness.linesElevation) if options.kmlVersion == "1": myAwareness.createKmlV1(options.kmlOutput) if options.kmlVersion == "0":
option_include = options.include option_spider = options.spider option_js = options.javascript option_crystal = options.crystal option_session = options.session else: try: f = open("grabber.conf.xml", 'r') except IOError: print "No arguments ? You need to setup the XML configuration file or using the inline arguments" print "Look at the doc to start..." sys.exit(1) parser = make_parser() conf_handler = ConfHandler() # Tell the parser to use our handler parser.setContentHandler(conf_handler) parser.parse("grabber.conf.xml") option_url = confUrl option_spider = confSpider option_sql = "sql" in confActions option_bsql = "bsql" in confActions option_xss = "xss" in confActions option_backup = "backup" in confActions option_include= "include" in confActions option_js = "javascript" in confActions option_crystal= "crystal" in confActions option_session= "session" in confActions # default to localhost ? archives_url = "http://localhost"
questionsByIdFilename = 'questionsById.pickle' historyByIdFilename = 'historyById.pickle' userAnswersByParentId = {} answersByParentId = {} questionsById = {} historyById = {} usersById = {} print "scan 1: find answers by userid", options.userid, "in", options.posts if options.temp: if os.path.exists(userAnswersByParentIdFilename): userAnswersByParentId = load(userAnswersByParentIdFilename) if len(userAnswersByParentId) == 0: ush = UserAnswersStreamHandler(options.userid, userAnswersByParentId) parser.setContentHandler(ush) with open(options.posts) as f: parser.parse(f) if options.temp: save(userAnswersByParentIdFilename, userAnswersByParentId) print "found", len(userAnswersByParentId), "answers by userid", options.userid if len(userAnswersByParentId) > 0: print "scan 2: get questions for answers in", options.posts if options.temp: if os.path.exists(answersByParentIdFilename) and os.path.exists(questionsByIdFilename): answersByParentId = load(answersByParentIdFilename) questionsById = load(questionsByIdFilename)