def main():
	from optparse import OptionParser
	parser = OptionParser(
		usage="usage: %prog [options] network1 network2...",
		version="%prog 1.4\n"+_copyright,
		description=_description
		)
	parser.add_option("-m", "--manifest", dest='createManifest', action='store_true',
		help="Generates the manifest file")
	parser.add_option("-t", "--ttl", dest='createTtls', action='store_true',
		help="Generates ttl files for each network")
	parser.add_option("-i", "--main", dest='createMain', action='store_true',
		help="Generates the main C++ file for the plugin library")
	parser.add_option("-u", "--uribase", dest='uribase', default="http://clam-project.org/examples/lv2",
		help="Specifies the uri base for the plugins", metavar="URIBASE")
	parser.add_option("-d", "--doap", dest='doapfile',
		help="Specifies a doapfile with additional info when generating a ttl", metavar="DOAPFILE")
	parser.add_option("-y", "--binary", dest='binary',
		help="Provides a name for the library binary (required by --ttls)", metavar="LIBBINARY")
	parser.add_option("-g", "--gui-binary", dest='guibinary', default=None,
		help="Provides a name for a ui library binary for the plugin in the ttl", metavar="UIBINARY")

	options, networks = parser.parse_args()

	names = [os.path.splitext(os.path.basename(network))[0] for network in networks]
	uris  = [os.path.join(options.uribase,name) for name in names ]

	if options.createTtls:
		for network, uri, name in zip(networks, uris, names) :
			parser = make_parser()   
			curHandler = ExporterHandler()
			parser.setContentHandler(curHandler)
			parser.parse(open(network))
			curHandler.printTTL(network,uri,name,options.doapfile,options.binary,options.guibinary)
			return

	if options.createManifest:
		printManifest(uris,names)
		return

	if options.createMain:
		printCLAM_PLUGIN(networks,uris)
		return
Example #2
0
def main():
    #################### args parsing ############
        help = "#################Usage:##################\n dsTemplater -i <input XML> \n "
        help = help + "Example 1: dsTemplater -i dstemplate.xml \n "
        help = help + "Example 2: dsTemplater -i examples/dstemplate.60lo0FUTURE.xml \n"
        #print usage
	basedir = os.environ.get('BASEDIR')
        if(basedir is None):
	   print "Warning: BASEDIR environment variable not set"	
        parser = OptionParser(usage=help)
        parser.add_option("-i", "--file", dest="uinput",
        help="pass location of XML template", metavar="FILE")
        parser.add_option("-f", "--force",action="store_true",default=False, help="Force override existing output. Default is set to FALSE")
        #parser.add_option("-v", "--version",action="store_true", dest="version",default="v20120422", help="Assign version for downscaled data. Default is set to v20120422")        
        
        (options, args) = parser.parse_args()
        verOpt = True #default 
        forOpt = True #default
        inter = 'on' #for debugging
        if (inter == 'off'):
		uinput = 'dstemplate.xml'
	        dversion = "v20120422"
                force = False
                if os.path.exists(uinput):
                                        parser = xml.sax.make_parser(  )
                                        handler = temphandler.TempHandler(  )
                                        parser.setContentHandler(handler)
                                        parser.parse(uinput)
        if (inter == 'on'):  
        	for opts,vals in options.__dict__.items():
           	# print opts,vals
            		if(opts == 'uinput'):
                		uinput = vals
                		if os.path.exists(uinput):
				        print "XML input:",uinput
                    			#parser = xml.sax.make_parser(  )
                    			#handler = temphandler.TempHandler(  )
                    			#parser.setContentHandler(handler)
                    			#parser.parse(uinput)
                		else:
                    			print "Please pass a valid input XML filename with the -i argument and try again. See -h for syntax. Quitting now.."
                    			sys.exit()
            		if(opts == 'force'):
                		if (vals == True):
                    			forOpt = False
                		force = vals 
        #########  call listVars() #############################################################
	output_grid,kfold,lone,region,fut_train_start_time,fut_train_end_time,file_j_range,hist_file_start_time,hist_file_end_time,hist_train_start_time,hist_train_end_time,lats,late,lons,late, basedir,method,target_file_start_time,target_file_end_time,target_train_start_time,target_train_end_time,spat_mask,fut_file_start_time,fut_file_end_time,predictor,target,params,outdir,dversion,dexper,target_scenario,target_model,target_freq,hist_scenario,hist_model,hist_freq,fut_scenario,fut_model,fut_freq,hist_pred_dir,fut_pred_dir,target_dir,expconfig,target_time_window,hist_time_window,fut_time_window,tstamp,ds_region,target_ver,auxcustom= listVars(uinput,basedir,force)
        ######### call script creators..  #######################################################
        ############################### 1 ###############################################################
        #  make.code.tmax.sh 1 748 756 /vftmp/Aparna.Radhakrishnan/pid15769 outdir 1979 2008 tasmax

	# If the runscript directory already exists, please quit
	#/home/a1r/gitlab/cew/fudge2014///scripts/tasmax/35txa-CDFt-A00X01K02
	scriptdir = basedir+"/scripts/"+target+"/"+expconfig

	if(os.path.exists(scriptdir)):
		print "Warning: script directory already exists.",scriptdir
		#print "ERROR: Directory already exists. Clean up and try again please:",scriptdir 
        script1Loc = basedir+"/utils/bin/create_runcode"
        make_code_cmd = script1Loc+" "+str(predictor)+" "+str(target)+" "+str(output_grid)+" "+str(spat_mask)+" "+str(region)
        make_code_cmd = make_code_cmd+" "+str(file_j_range)+" "+str(lons)+" "+str(lats)+" "+str(late)
	make_code_cmd = make_code_cmd+" "+str(hist_file_start_time)+" "+str(hist_file_end_time)+" "+str(hist_train_start_time)+" "+str(hist_train_end_time)+" "+str(hist_scenario)+" "+str(hist_model)+" "+hist_freq+" "+str(hist_pred_dir)+" "+str(hist_time_window)
        make_code_cmd = make_code_cmd+" "+str(fut_file_start_time)+" "+str(fut_file_end_time)+" "+str(fut_train_start_time)+" "+str(fut_train_end_time)+" "+str(fut_scenario)+" "+str(fut_model)+" "+fut_freq+" "+str(fut_pred_dir)+" "+str(fut_time_window)
        make_code_cmd = make_code_cmd+" "+str(target_file_start_time)+" "+str(target_file_end_time)+" "+str(target_train_start_time)+" "+str(target_train_end_time)+" "+str(target_scenario)+" "+str(target_model)+" "+target_freq+" "+str(target_dir)+" "+str(target_time_window)
        make_code_cmd = make_code_cmd+" "+str(method)+" "+str(expconfig)+" "+str(kfold)+" "+str(outdir)+" "+str(tstamp)+" "+'na'+" "+basedir+" "+str(lone)

        print "Step 1: R code starters generation ..in progress"
        #p = subprocess.Popen('tcsh -c "'+make_code_cmd+'"',shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE)
        
        params_new =  '"'+str(params)+'\"'
        make_code_cmd = make_code_cmd +" "+params_new+" "+"'"+str(ds_region)+"'"
        make_code_cmd = make_code_cmd+" "+str(auxcustom)
	#cprint make_code_cmd
        #p = subprocess.Popen(make_code_cmd +" "+params_new,shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE)
        p = subprocess.Popen(make_code_cmd,shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE)
        output, errors = p.communicate() 
        if(p.returncode != 0):
         print "Step1:!!!! FAILED !!!!, please contact developer."
         print output, errors
         sys.exit(0)
        #cprint output, errors
	#cprint "----Log-----"
        #cprint output,errors
        ###############################################################################################
        print "1- completed\n"
        ############################### 2 ################################################################
	#target_time_window,hist_time_window,fut_time_window
        script2Loc = basedir+"/utils/bin/"+"create_runscript"
        create_runscript_cmd = script2Loc+" "+str(lons)+" "+str(lone)+" "+str(expconfig)+" "+str(basedir)+" "+target+" "+method+" "+target_dir+" "+hist_pred_dir+" "+fut_pred_dir+" "+outdir+" "+str(file_j_range)+" "+tstamp+" "+str(target_file_start_time)+" "+str(target_file_end_time)+" "+str(hist_file_start_time)+" "+str(hist_file_end_time)+" "+str(fut_file_start_time)+" "+str(fut_file_end_time)+" "+str(spat_mask)+" "+str(region)+" "+auxcustom+" "+target_time_window+" "+hist_time_window+" "+fut_time_window
        print "Step 2: Individual Runscript generation: \n"+create_runscript_cmd
        p1 = subprocess.Popen('tcsh -c "'+create_runscript_cmd+'"',shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE)
        print "Step 2: Individual runscript  creation.. in progress"
        output1, errors1 = p1.communicate()
	#cprint output1,errors1
        print "2- completed\n"
        if(p1.returncode != 0):
         print "Step2:!!!! FAILED !!!!, please contact developer."
         print output1, errors1
         sys.exit(0)
        #print output1, errors1
        #c print errors1
        
        ###################################### 3 ################################################################
        script3Loc = basedir+"/utils/bin/"+"create_master_runscript"
        create_master_cmd= script3Loc+" "+str(lons)+" "+str(lone)+" "+str(predictor)+" "+method+" "+basedir+" "+expconfig+" "+file_j_range+" "+tstamp+" "+str(ppn)
        print "Step 3: --------------MASTER SCRIPT GENERATION-----------------------"#+create_master_cmd
        p2 = subprocess.Popen('tcsh -c "'+create_master_cmd+'"',shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE)
        #cprint create_master_cmd
        #print "Create master script .. in progress"
        output2, errors2 = p2.communicate()
        #######
        if(p2.returncode != 0):
         print "Step3:!!!! FAILED !!!!, please contact developer."
         print output2, errors2
         sys.exit(0)
        print output2, errors2
        print "3- completed"
	##################################### 4 ############################################
        # copy xml to configs dir 
        cdir = basedir+"/"+"/scripts/"+predictor+"/"+expconfig+"/config/"
        try:
	  os.mkdir(cdir)
	except:
	  print "Unable to create dir. Dir may exist already", cdir 
	shutil.copy2(uinput, cdir)
        print "Config XML saved in ",cdir 
	print "----See readMe in fudge2014 directory for the next steps----"
	
	def endElement(self, name):
		if name == "ch1" or name == "ch2" or name == "watts" or name == "data":
			print "no"
		if name == "msg":
			self.outFile.write("\n")
		else:
			self.outFile.write(",")
	
	def characters(self, ch):
		self.outFile.write( ch.strip("\n\t,") )


if __name__ == "__main__":

    parser = OptionParser()
    parser.add_option("-f", "--file", dest="filename",
                      help="write report to FILE", metavar="FILE", default="none")

    (options, args) = parser.parse_args()
    
    if options.filename == "none":
        exit()

    parser = xml.sax.make_parser()
    
    fileCSV = open("dump.csv", "w")
    handler = XML2CSV(fileCSV)
    parser.setContentHandler(handler)
    parser.parse(options.filename)
Example #4
0
        option_crystal = options.crystal
        option_session = options.session
        option_cookie = options.cookie

        setOptions(options)
    else:
        try:
            f = open("grabber.conf.xml", 'r')
        except IOError:
            print "No arguments ? You need to setup the XML configuration file or using the inline arguments"
            print "Look at the doc to start..."
            sys.exit(1)
        parser = make_parser()
        conf_handler = ConfHandler()
        # Tell the parser to use our handler
        parser.setContentHandler(conf_handler)
        parser.parse("grabber.conf.xml")

        option_url = confUrl
        option_spider = confSpider
        option_sql = "sql" in confActions
        option_bsql = "bsql" in confActions
        option_xss = "xss" in confActions
        option_backup = "backup" in confActions
        option_include = "include" in confActions
        option_js = "javascript" in confActions
        option_crystal = "crystal" in confActions
        option_session = "session" in confActions

    # default to localhost ?
    archives_url = "http://localhost"
Example #5
0
    parser.add_option("-d", "--key-directory", default="", dest="key_directory",
                      help="Specify a parent directory for keys")

    (options, args) = parser.parse_args()

    if len(args) < 2:
        parser.error("Must specify a config file (keys.conf) AND mac_permissions.xml file(s)!")

    logging.basicConfig(level=logging.INFO if options.verbose == True else logging.WARN)

    # Read the config file
    config = ParseConfig()
    config.read(args[0])

    os.chdir(options.root)

    output_file = sys.stdout if options.output_file == "stdout" else open(options.output_file, "w")
    logging.info("Setting output file to: " + options.output_file)

    # Generate the key list
    key_map = config.generateKeyMap(options.target_build_variant.lower(), options.key_directory)
    logging.info("Generate key map:")
    for k in key_map:
        logging.info(k + " : " + str(key_map[k]))
    # Generate the XML file with markup replaced with keys
    parser = make_parser()
    parser.setContentHandler(ReplaceTags(key_map, output_file))
    for f in args[1:]:
        parser.parse(f)
Example #6
0
    action="store_true",
    dest="reformat",
    default=False,
    help="run reformat scripts for the observations according to namelist")
options, args = parser.parse_args()
if len(args) == 0:
    parser.print_help()
    sys.exit(0)

# Get command arguments.
input_xml_full_path = args[0]

# Parse input namelist into project_info-dictionary.
Project = xml_parsers.namelistHandler()
parser = xml.sax.make_parser()
parser.setContentHandler(Project)
parser.parse(input_xml_full_path)

# Project_info is a dictionary with all info from the namelist.
project_info = Project.project_info

if options.reformat:
    if 'REFORMAT' not in project_info.keys():
        error('No REFORMAT tag specified in {0}'.format(input_xml_full_path))
    if len(project_info['REFORMAT']) == 0:
        info('No reformat script specified', 1, 1)
    print_header({}, options.reformat)
    for k, v in project_info['REFORMAT'].iteritems():
        if not os.path.exists(v):
            error('Path {0} does not exist'.format(v))
        projects.run_executable(v, project_info, 1, False, write_di=False)
Example #7
0
def main():
    #################### args parsing ############
        help = "#################Usage:(run from AN nodes)##################\n dsTemplater -i <input XML> \n "
        help = help + "Example 1: expergen -i dstemplate.xml \n "
        help = help + "Example 2: expergen -i examples/dstemplate.60lo0FUTURE.xml \n"
        #print usage
	basedir = os.environ.get('BASEDIR')
        if(basedir is None):
	   print "Warning: BASEDIR environment variable not set"	
	##### get version BRANCH info ######################
        branch = os.environ.get('BRANCH')
        if(branch is None):
           print "Warning: BRANCH env variable not set. BRANCH will be set to undefined"
           branch = "undefined"
	####################################################
        parser = OptionParser(usage=help)
        parser.add_option("-i", "--file", dest="uinput",
        help="pass location of XML template", metavar="FILE")
      #since we now have an XML tag  parser.add_option("-f", "--force",action="store_true",default=False, help="Force override existing output. Default is set to FALSE")
        parser.add_option("--msub", "--msub",action="store_true",default=False, help="Automatically submit the master runscripts using msub.Default is set to FALSE")
        #parser.add_option("-v", "--version",action="store_true", dest="version",default="v20120422", help="Assign version for downscaled data. Default is set to v20120422")        
        
        (options, args) = parser.parse_args()
        verOpt = True #default
	msub = False 
        forOpt = True #default
        inter = 'on' #for debugging
#########if platform is not PAN, disable expergen at this time 11/14/2014 ###############
        system,node,release,version,machine = os.uname()
        if(node.startswith('pp')):
                print "Running on PP(PAN) node", node
        if(node.startswith('an')):
                print "Running on AN(PAN) node", node
        else:
                 print "\033[1;41mERROR code -5: Running on a workstation not tested for end-to-end runs yet. Please login to analysis nodes to run expergen. \033[1;m",node
		 sys.exit(-5)

   
######################################################################################### 
        if (inter == 'off'):
		uinput = 'dstemplate.xml'
	        dversion = "v20120422"
                force = False
                if os.path.exists(uinput):
                                        parser = xml.sax.make_parser(  )
                                        handler = temphandler.TempHandler(  )
                                        parser.setContentHandler(handler)
                                        parser.parse(uinput)
        if (inter == 'on'):  
        	for opts,vals in options.__dict__.items():
           	# print opts,vals
            		if(opts == 'uinput'):
                		uinput = vals
                		if os.path.exists(uinput):
				        print "XML input:",uinput
                    			#parser = xml.sax.make_parser(  )
                    			#handler = temphandler.TempHandler(  )
                    			#parser.setContentHandler(handler)
                    			#parser.parse(uinput)
                		else:
                    			print "Please pass a valid input XML filename with the -i argument and try again. See -h for syntax. Quitting now.."
                    			sys.exit()
            		if(opts == 'msub'):
                		if (vals == True):
                    			msub = True 
				else:
					msub = False
                		msub = vals 
        #########  call listVars() #############################################################
	output_grid,kfold,lone,region,fut_train_start_time,fut_train_end_time,file_j_range,hist_file_start_time,hist_file_end_time,hist_train_start_time,hist_train_end_time,lats,late,lons,late, basedir,method,target_file_start_time,target_file_end_time,target_train_start_time,target_train_end_time,spat_mask,fut_file_start_time,fut_file_end_time,predictor,target,params,outdir,dversion,dexper,target_scenario,target_model,target_freq,hist_scenario,hist_model,hist_freq,fut_scenario,fut_model,fut_freq,hist_pred_dir,fut_pred_dir,target_dir,expconfig,target_time_window,hist_time_window,fut_time_window,tstamp,ds_region,target_ver,auxcustom,qc_mask,qc_varname,qc_type,adjust_out,sbase,pr_opts,masklists= listVars(uinput,basedir,msub)
        ######### call script creators..  #######################################################
        ############################### 1 ###############################################################
        #  make.code.tmax.sh 1 748 756 /vftmp/Aparna.Radhakrishnan/pid15769 outdir 1979 2008 tasmax

	# If the runscript directory already exists, please quit
	#/home/a1r/gitlab/cew/fudge2014///scripts/tasmax/35txa-CDFt-A00X01K02
	scriptdir = [sbase+"/master",sbase+"/runcode",sbase+"/runscript"]
        for sd in scriptdir:
	    if(os.path.exists(sd)):
       		 if os.listdir(sd):
		    if (overwrite == False):    
			print '\033[1;41mERROR: Scripts Directory already exists. Clean up using cleanup_script and try again please -or- (Use <ifpreexist>erase</ifpreexist> if output/scripts already exist and you want to override this and let expergen run the cleanup_script for you; Use <ifpreexist>move</ifpreexist> to move existing output)\033[1;m',sd
                        print "\033[1;41mERROR code -6: script directory already exists.Check --\033[1;m",scriptdir
                	sys.exit(-6)
		    if (overwrite == True):
			print "\033[1;43mWarning: Scripts Directory already exists. But, since <ifpreexist> has different settings, cleanup utility will handle this\033[1;m "
			print "Now invoking cleanup utility..........."
		        cleaner_script = basedir+"/utils/bin/"+"cleanup_script.csh"
			if(preexist_glob == 'erase'):
				print "Lets erase it ........... ifpreexist"
		        	cleaner_cmd = cleaner_script+" d "+uinput 
			else: 
				if(preexist_glob != 'move') & (preexist_glob != 'exit'):
					print "CHECK ifpreexist settings, quitting now"
					sys.exit(1)
				print "Lets move it............... if preexist" 
                               	cleaner_cmd = cleaner_script+" m "+uinput
	                print "cleaner_cmd"
			print cleaner_cmd		
	                pclean = subprocess.Popen(cleaner_cmd,shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE)
			#check return code
        		output0, errors0 = pclean.communicate()
        		if(pclean.returncode != 0):
         			print "033[1;41mCleaner Step:!!!! FAILED !!!!, please contact developer.033[1;m"
         			print output0, errors0
        			sys.exit(1)
			else:
				print output0,errors0
			print "continue expergen run...................." 
			break 
	    else:
		   print "scriptdir "+sd+" does not exist. Looks like a clean slate "
        #print  "hist_freq"+hist_freq
        script1Loc = basedir+"/utils/bin/create_runcode"
        make_code_cmd = script1Loc+" "+str(predictor)+" "+str(target)+" "+str(output_grid)+" "+str(spat_mask)+" "+str(region)
        make_code_cmd = make_code_cmd+" "+str(file_j_range)+" "+str(lons)+" "+str(lats)+" "+str(late)
	make_code_cmd = make_code_cmd+" "+str(hist_file_start_time)+" "+str(hist_file_end_time)+" "+str(hist_train_start_time)+" "+str(hist_train_end_time)+" "+str(hist_scenario)+" "+str(hist_model)+" "+hist_freq+" "+str(hist_pred_dir)+" "+str(hist_time_window)
        make_code_cmd = make_code_cmd+" "+str(fut_file_start_time)+" "+str(fut_file_end_time)+" "+str(fut_train_start_time)+" "+str(fut_train_end_time)+" "+str(fut_scenario)+" "+str(fut_model)+" "+fut_freq+" "+str(fut_pred_dir)+" "+str(fut_time_window)
        make_code_cmd = make_code_cmd+" "+str(target_file_start_time)+" "+str(target_file_end_time)+" "+str(target_train_start_time)+" "+str(target_train_end_time)+" "+str(target_scenario)+" "+str(target_model)+" "+target_freq+" "+str(target_dir)+" "+str(target_time_window)
        make_code_cmd = make_code_cmd+" "+str(method)+" "+str(expconfig)+" "+str(kfold)+" "+str(outdir)+" "+str(tstamp)+" "+'na'+" "+basedir+" "+str(lone)

        print "Step 1: R code starters generation ..in progress"
        #p = subprocess.Popen('tcsh -c "'+make_code_cmd+'"',shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE)
        
        params_new =  '"'+str(params)+'\"'
	params_pr_opts = '"'+str(pr_opts)+'\"'
        make_code_cmd = make_code_cmd +" "+params_new+" "+"'"+str(ds_region)+"'"
        make_code_cmd = make_code_cmd+" "+str(auxcustom)+" "+str(qc_mask)+" "+str(qc_varname)+" "+str(qc_type)+" "+str(adjust_out)+" "+str(sbase)+" "+str(params_pr_opts)+" "+str(branch)+" "+'"'+str(masklists)+'"' 
	#cprint make_code_cmd
        #p = subprocess.Popen(make_code_cmd +" "+params_new,shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE)
        p = subprocess.Popen(make_code_cmd,shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE)
        output, errors = p.communicate() 
        if(p.returncode != 0):
         print "Step1:!!!! FAILED !!!!, please contact developer."
         print output, errors
         sys.exit(0)
        #cprint output, errors
	#cprint "----Log-----"
        #cprint output,errors
        ###############################################################################################
        print "1- completed\n"
        #print "debug............msub turned ",msub
        ############################### 2 ################################################################
	#target_time_window,hist_time_window,fut_time_window
        script2Loc = basedir+"/utils/bin/"+"create_runscript"
        create_runscript_cmd = script2Loc+" "+str(lons)+" "+str(lone)+" "+str(expconfig)+" "+str(basedir)+" "+target+" "+method+" "+target_dir+" "+hist_pred_dir+" "+fut_pred_dir+" "+outdir+" "+str(file_j_range)+" "+tstamp+" "+str(target_file_start_time)+" "+str(target_file_end_time)+" "+str(hist_file_start_time)+" "+str(hist_file_end_time)+" "+str(fut_file_start_time)+" "+str(fut_file_end_time)+" "+str(spat_mask)+" "+str(region)+" "+auxcustom+" "+target_time_window+" "+hist_time_window+" "+fut_time_window+" "+sbase
        print "Step 2: Individual Runscript generation: \n"+create_runscript_cmd
        p1 = subprocess.Popen('tcsh -c "'+create_runscript_cmd+'"',shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE)
        print "Step 2: Individual runscript  creation.. in progress"
        output1, errors1 = p1.communicate()
	#cprint output1,errors1
        print "2- completed\n"
        if(p1.returncode != 0):
         print "Step2:!!!! FAILED !!!!, please contact developer."
         print output1, errors1
         sys.exit(0)
        #print output1, errors1
        #c print errors1
        
        ###################################### 3 ################################################################
        script3Loc = basedir+"/utils/bin/"+"create_master_runscript"
        create_master_cmd= script3Loc+" "+str(lons)+" "+str(lone)+" "+str(predictor)+" "+method+" "+sbase+" "+expconfig+" "+file_j_range+" "+tstamp+" "+str(ppn)+" "+str(msub)
        print "Step 3: --------------MASTER SCRIPT GENERATION-----------------------"#+create_master_cmd
        p2 = subprocess.Popen('tcsh -c "'+create_master_cmd+'"',shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE)
        #cprint create_master_cmd
        print "Create master script .. in progress"
        output2, errors2 = p2.communicate()
        #######
        if(p2.returncode != 0):
         print "Step3:!!!! FAILED !!!!, please contact developer."
         print output2, errors2
         sys.exit(0)
        print output2, errors2
        print "3- completed"
	##################################### 4 ############################################
        # copy xml to configs dir 
        cdir = sbase+"/config/"
        try:
	  os.mkdir(cdir)
	except:
	  print "Unable to create dir. Dir may exist already", cdir 
	shutil.copy2(uinput, cdir)
        print "Config XML saved in ",cdir 
        print "RunScripts will be saved under:",sbase
	print "----See readMe in fudge2014 directory for the next steps----"
	print "Use submit_job to submit scripts"

############### crte ppscript #################
	dev = "off" 
        print(sbase+"/postProc/aux/")
        if not os.path.exists(sbase+"/postProc/aux/"):
        	os.makedirs(sbase+"/postProc/aux/")
        if (dev == "off"):
		tsuffix = ""
        	ppbase = sbase+"/postProc/aux/"+"/postProc_source"
	else:
		tsuffix = "_"+tstamp
                ppbase = sbase+"/postProc/aux/"+"/postProc_source"+tstamp
	try:
  		ppfile = open(ppbase, 'w')
#check if qc_mask is relevant  
		if(qc_mask != 'off'):
  			pp_cmnd = "python $BASEDIR/bin/postProc -i "+os.path.abspath(uinput)+" -v "+target+","+target+"_qcmask\n"
	 	else:
                        pp_cmnd = "python $BASEDIR/bin/postProc -i "+os.path.abspath(uinput)+" -v "+target+"\n"
  		ppfile.write(pp_cmnd)
  		ppfile.close()
	except:
  		print "Unable to create postProc command file. You may want to check your settings."
	#c	print create_pp_cmd
        if(os.path.exists(ppbase)):
######################### write postProc_job to be used ############################
                ppLoc = basedir+"/utils/bin/"+"create_postProc"
                create_pp_cmd= ppLoc+" "+ppbase+" "+sbase+"  "+basedir+" "+tstamp+" "+branch
                print "Step 4: --------------PP postProc SCRIPT GENERATION-----------------------"
                p4 = subprocess.Popen('tcsh -c "'+create_pp_cmd+'"',shell=True,stdout=PIPE,stdin=PIPE, stderr=PIPE)
                output4, error4 = p4.communicate()
                if(p4.returncode != 0):
                        print "Step4:!!!! FAILED !!!!, please contact developer."
                        print output4, error4
                        sys.exit(-4)
                print output4, error4
                print "4- completed"
                print "NOTE: postProc will succeed only if you're running the model for the full downscaled region. (it will fail if you're running downscaling for a single slice for example)"
                print "----------------------------------------"
		print "\033[1;42mPlease use this script to run post post-processing (or msub this script), postProc when downscaling jobs are complete \033[1;m",sbase+"postProc/postProc_command"+tsuffix
                try:  
   			NEMSemail = os.environ["NEMSemail"]
                        print "msub -m ae -M "+os.environ.get('NEMSemail')+" "+sbase+"postProc/postProc_command"+tsuffix
		except KeyError: 
   			print "NEMSemail not set. Please use your email for notification in the following msub command i.e msub -m ae -M <email> script " 
			print "msub "+sbase+"postProc/postProc_command"+tsuffix
	else:
		print "postProc_command cannot be created. postProc_source does not exist"
################ step 5 fudgeList invocation ##############################################
        slogloc = sbase+"/"+"experiment_info.txt"
        fcmd = "python "+basedir+"/bin/fudgeList.py -f -i "+uinput+" -o "+slogloc
        f = subprocess.Popen(fcmd, stdout=subprocess.PIPE, shell=True)
        out, err = f.communicate()
        #print "fudgeList out", out
        #if err is not None:
        #       print "fudgeList err", err
        print "Summary Log File: ", slogloc
Example #8
0
#
# cleanup / preparation
#

cmd = 'rm -rf %s/%s' % (QASRC_DIRFN, corpusname)
logging.info(cmd)
os.system(cmd)

cmd = 'mkdir -p %s/%s' % (QASRC_DIRFN, corpusname)
logging.info(cmd)
os.system(cmd)

#
# parse XML
#

parser = make_parser()
parser.setContentHandler(WikiContentHandler(WIKIXML[options.lang]))

article_cnt = 0
article_tot = 0

wikisize = os.path.getsize(WIKIXML[options.lang])

with open(WIKIXML[options.lang], 'rb') as wikibz2f:

    # with codecs.getreader('utf8')(BZ2File(WIKIXML[options.lang], 'r')) as wikif:
    with codecs.getreader('utf8')(BZ2File(wikibz2f)) as wikif:

        parser.parse(wikif)
Example #9
0
    action="store_true",
    dest="reformat",
    default=False,
    help="run reformat scripts for the observations according to namelist")
options, args = parser.parse_args()
if len(args) == 0:
    parser.print_help()
    sys.exit(0)

# Get command arguments.
input_xml_full_path = args[0]

# Parse input namelist into project_info-dictionary.
Project = xml_parsers.namelistHandler()
parser = xml.sax.make_parser()
parser.setContentHandler(Project)
parser.parse(input_xml_full_path)

# Project_info is a dictionary with all info from the namelist.
project_info = Project.project_info

if options.reformat:
    if 'REFORMAT' not in project_info.keys():
        error('No REFORMAT tag specified in {0}'.format(input_xml_full_path))
    if len(project_info['REFORMAT']) == 0:
        info('No reformat script specified', 1, 1)
    print_header({}, options.reformat)
    for k, v in project_info['REFORMAT'].iteritems():
        if not os.path.exists(v):
            error('Path {0} does not exist'.format(v))
        projects.run_executable(v, project_info, 1, False, write_di=False)
Example #10
0
        if name == "msg":
            self.outFile.write("\n")
        else:
            self.outFile.write(",")

    def characters(self, ch):
        self.outFile.write(ch.strip("\n\t,"))


if __name__ == "__main__":

    parser = OptionParser()
    parser.add_option("-f",
                      "--file",
                      dest="filename",
                      help="write report to FILE",
                      metavar="FILE",
                      default="none")

    (options, args) = parser.parse_args()

    if options.filename == "none":
        exit()

    parser = xml.sax.make_parser()

    fileCSV = open("dump.csv", "w")
    handler = XML2CSV(fileCSV)
    parser.setContentHandler(handler)
    parser.parse(options.filename)
Example #11
0
            elif archiveType == "bz2" or archiveType == "BZ2":
                os.system('bzip2 -d "%s"' % options.osmInput)

        options.osmInput = options.osmInput.rstrip("." + archiveType)
        print "File uncompressed: ", options.osmInput

    if options.kmlOutput == None:
        options.kmlOutput = os.path.basename(options.osmInput).rstrip(".osc")
    if options.kmlVersion == None: options.kmlVersion = "1"

    myAwareness = OSMaware(debug=False,
                           verbose=False,
                           ele=options.linesElevation,
                           kml_version=options.kmlVersion)
    parser = make_parser()
    parser.setContentHandler(myAwareness)
    t0 = datetime.now()
    print "Starting parsing OSM file..."
    parser.parse(options.osmInput)

    #for userName, userStat in self.statsUsers.iteritems(): print userName, userStat
    print "Number of contributors (users):", len(myAwareness.statsUsers)
    print "Number of Nodes created, deleted or modified:", myAwareness.nodeCount
    print "Number of Ways created, deleted or modified:", myAwareness.wayCount
    print "Number of Realations created, deleted or modified:", myAwareness.relationCount

    if options.kmlVersion == "2":
        myAwareness.createKmlV2(options.kmlOutput,
                                heightFactor=myAwareness.linesElevation)
    if options.kmlVersion == "1":
        myAwareness.createKmlV1(options.kmlOutput)
Example #12
0
            if archiveType == "gz" or archiveType == "GZ":
                os.system('gunzip "%s"' % options.osmInput)
            elif archiveType == "bz2" or archiveType == "BZ2":
                os.system('bzip2 -d "%s"' % options.osmInput)

        options.osmInput = options.osmInput.rstrip("." + archiveType)
        print "File uncompressed: ", options.osmInput

    if options.kmlOutput == None:
        options.kmlOutput = os.path.basename(options.osmInput).rstrip(".osc")
    if options.kmlVersion == None:
        options.kmlVersion = "1"

    myAwareness = OSMaware(debug=False, verbose=False, ele=options.linesElevation, kml_version=options.kmlVersion)
    parser = make_parser()
    parser.setContentHandler(myAwareness)
    t0 = datetime.now()
    print "Starting parsing OSM file..."
    parser.parse(options.osmInput)

    # for userName, userStat in self.statsUsers.iteritems(): print userName, userStat
    print "Number of contributors (users):", len(myAwareness.statsUsers)
    print "Number of Nodes created, deleted or modified:", myAwareness.nodeCount
    print "Number of Ways created, deleted or modified:", myAwareness.wayCount
    print "Number of Realations created, deleted or modified:", myAwareness.relationCount

    if options.kmlVersion == "2":
        myAwareness.createKmlV2(options.kmlOutput, heightFactor=myAwareness.linesElevation)
    if options.kmlVersion == "1":
        myAwareness.createKmlV1(options.kmlOutput)
    if options.kmlVersion == "0":
Example #13
0
		option_include = options.include
		option_spider = options.spider
		option_js = options.javascript
		option_crystal = options.crystal
		option_session = options.session
	else:
		try:
			f = open("grabber.conf.xml", 'r')
		except IOError:
			print "No arguments ? You need to setup the XML configuration file or using the inline arguments"
			print "Look at the doc to start..."
			sys.exit(1)
		parser = make_parser()
		conf_handler = ConfHandler()
		# Tell the parser to use our handler
		parser.setContentHandler(conf_handler)
		parser.parse("grabber.conf.xml")

		option_url    = confUrl
		option_spider = confSpider
		option_sql    = "sql" in confActions
		option_bsql   = "bsql" in confActions
		option_xss    = "xss" in confActions
		option_backup = "backup" in confActions
		option_include= "include" in confActions
		option_js     = "javascript" in confActions
		option_crystal= "crystal" in confActions
		option_session= "session" in confActions

	# default to localhost ?
	archives_url = "http://localhost"
Example #14
0
    questionsByIdFilename = 'questionsById.pickle'
    historyByIdFilename = 'historyById.pickle'
    userAnswersByParentId = {}
    answersByParentId = {}
    questionsById = {}
    historyById = {}
    usersById = {}

    print "scan 1: find answers by userid", options.userid, "in", options.posts
    if options.temp:
      if os.path.exists(userAnswersByParentIdFilename):
        userAnswersByParentId = load(userAnswersByParentIdFilename)

    if len(userAnswersByParentId) == 0:
      ush = UserAnswersStreamHandler(options.userid, userAnswersByParentId)
      parser.setContentHandler(ush)
      with open(options.posts) as f:
          parser.parse(f)

      if options.temp:
        save(userAnswersByParentIdFilename, userAnswersByParentId)

    print "found", len(userAnswersByParentId), "answers by userid", options.userid

    if len(userAnswersByParentId) > 0:

        print "scan 2: get questions for answers in", options.posts
        if options.temp:
          if os.path.exists(answersByParentIdFilename) and os.path.exists(questionsByIdFilename):
            answersByParentId = load(answersByParentIdFilename)
            questionsById = load(questionsByIdFilename)