def scanCAS(cas): keylist = [] casLines = getFileContent(cas) # ~~ clean comments core = [] for i in range(len(casLines)): line = casLines[i].replace('"""',"'''").replace('"',"'") proc = re.match(key_comment,line+'/') line = proc.group('before').strip() + ' ' proc = re.match(emptyline,line) if not proc: core.append(line) casStream = (' '.join(core)).replace(' ',' ') # ~~ clean values to keep only the keys while casStream != '': # ~~ non-key proc = re.match(key_none,casStream) if proc: casStream = proc.group('after') continue # ~~ key proc = re.match(key_equals,casStream) if not proc: print '... hmmm, did not see this one coming ...' break kw = proc.group('key').strip() casStream = proc.group('after') # still hold the separator # ~~ val proc = re.match(val_equals,casStream) if not proc: print 'no value to keyword ',kw sys.exit() val = [] while proc: val.append(proc.group('val').replace("'",'')) casStream = proc.group('after') # still hold the separator proc = re.match(val_equals,casStream) keylist.append([kw,val]) # ~~ sort out the groups, starting with 'NOM' keywords = {} while keylist != []: keywords.update({keylist[0][0]:keylist[0][1]}) keylist.pop(0) return keywords
def processLIT(cas,iFiles,TMPDir): # ~~ copy input files ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ for k in cas.keys(): if iFiles.has_key(k): cref = cas[k][0] if not path.isfile(cref): print '... file does not exist ',cref return False crun = path.join(TMPDir,iFiles[k].split(';')[1]) if iFiles[k].split(';')[3] == 'ASC': putFileContent(crun,getFileContent(cref)+['']) print ' copying: ', path.basename(cref) else: shutil.copy2(cref,crun) print ' copying: ', path.basename(cref) return True
def getNextFile(self) : # ~~ always 1st file in the list ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ while 1: if self.leftFileNames == []: return False frf = self.leftFileNames.pop(0) frd = path.dirname(frf) gbd = path.join(path.dirname(frd),self.linguist) if not path.isdir(gbd) : mkdir(gbd) self.outFileName = path.join(gbd,path.basename(frf)) if path.isfile(path.splitext(self.outFileName)[0]+'.noe'): frf = path.splitext(self.outFileName)[0]+'.noe' break if not path.isfile(self.outFileName): break self.doneLines = [] self.leftLines = getFileContent(frf) if path.splitext(frf)[1] == '.noe': remove(frf) return True
cfg = parseConfig_DoxygenTELEMAC(cfgname)[cfgname] # ~~ Scans all source files to build a relation database ~~ print '\n\nScanning the source code\n\ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n' fic,mdl,sbt,fct,prg,dep,all = scanSources(cfgname,cfg,False) # ~~ Scans all source files to update Doxygen ~~~~~~~~~~~~~~~~ for mod in fic.keys(): print '\nWriting out ' + mod.upper() + ' source code following standrad template format\n\ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' for ifile in fic[mod].keys(): # ~~ Reads the content of the source file ~~~~~~~~~~~ ilines = getFileContent(ifile) # ~~ Process Lines ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # ===================================================== found = False olines = [] for iline in ilines: oline = iline.rstrip() if oline != '': if iline[0].upper() == 'C': oline = '!' + oline[1:] found = True else: oline = '!' found = True proc = re.match(f77continu2,oline)
#!/usr/bin/python3 import re import sys import utils compound = sys.argv[1] compoundsFileName = 'output/insert_compound.sql' #read file data = utils.getFileContent(compound) names = utils.getSectionString("NAME", data) names = names.replace("'", "''") formula = utils.getSectionString("FORMULA", data) mass = utils.getSectionString("EXACT_MASS", data) compoundsFile = open(compoundsFileName, 'a') compoundsFile.write( "INSERT INTO raw_compound (id, name, formula, mass) VALUES ('" + compound + "' , '" + names + "' , '" + formula + "' , '" + mass + "');\n") compoundsFile.close()
#!/usr/bin/python3 import re import sys import utils #print ('Number of arguments:', len(sys.argv), 'arguments.', str(sys.argv)) if len(sys.argv) != 2: print('You provided ', (len(sys.argv) - 1), ' arguments. \n Please run parse_module.py <module_name>') sys.exit(2) pathway = sys.argv[1] pathwayFileName = 'output/insert_pathway.sql' #read file data = utils.getFileContent(pathway) name = utils.getSectionString("NAME", data) description = utils.getSectionString("DESCRIPTION", data) pathwayClass = utils.getSectionString("CLASS", data) pathwayFile = open(pathwayFileName, 'a') pathwayFile.write( "INSERT INTO raw_pathway (id, name, description, class) VALUES ('" + pathway + "' , '" + name + "' , '" + description + "' , '" + pathwayClass + "');\n") pathwayFile.close()
#!/usr/bin/python3 import re import sys import utils reaction = sys.argv[1] outputFileName = 'output/insert_reaction_enzyme.sql' #read file data = utils.getFileContent(reaction) enzymes = utils.getSectionArray("ENZYME", data, None) outputFile = open(outputFileName, 'a') for enzyme in enzymes: outputFile.write( "INSERT INTO raw_reaction_enzyme (reaction, enzyme) VALUES ('" + reaction + "' , '" + enzyme + "' );\n") outputFile.close() name = utils.getSectionString("NAME", data) name = name.replace("'", "") definition = utils.getSectionString("DEFINITION", data) definition = definition.replace("'", "") reactionFileName = 'output/insert_reaction.sql' outputFile = open(reactionFileName, 'a') for enzyme in enzymes: outputFile.write( "INSERT INTO raw_reaction (id, name, definition) VALUES ( '" + reaction + "' , '" + name + "' , '" + definition + "' );\n")
def scanDICO(dicoFile): keylist = [] dicoLines = getFileContent(dicoFile) # ~~ buddle continuations (long strings) and remove comments and empty lines core = []; i = -1 while i < len(dicoLines) - 1: i = i + 1; line = '' l = dicoLines[i].strip() #proc = re.match(key_comment,l) #if proc: l = proc.group('before').strip() + ' ' if l.strip()[0:1] == '/' : continue proc = re.match(emptyline,l) if proc: continue proc = re.match(key_none,l) if proc: continue proc = re.match(entryquote,l) line = proc.group('before') l = proc.group('after') while l != '': if l[0:1] == '"': proc = re.match(exitdquote,l+' ') if proc: line = line + "'" + proc.group('before').replace("'",'"') + "'" proc = re.match(entryquote,proc.group('after').strip()) line = line + proc.group('before') l = proc.group('after').strip() print '>',l else: i = i + 1 l = l.strip() + ' ' + dicoLines[i].strip() elif l[0:1] == "'": proc = re.match(exitsquote,l+' ') if proc: line = line + "'" + proc.group('before').replace("'",'"') + "'" proc = re.match(entryquote,proc.group('after').strip()) line = line + proc.group('before') l = proc.group('after').strip() else: i = i + 1 l = l.strip() + ' ' + dicoLines[i].strip() core.append(line) dicoStream = (' '.join(core)).replace(' ',' ').replace('""','"') # ~~ clean values to keep only the keys while dicoStream != '': # ~~ non-key proc = re.match(key_none,dicoStream) if proc: dicoStream = proc.group('after') continue # ~~ key proc = re.match(key_equals,dicoStream) if not proc: break kw = proc.group('key').strip() if kw not in dicokeys: print 'unknown key ',kw sys.exit() dicoStream = proc.group('after') # still hold the separator # ~~ val proc = re.match(val_equals,dicoStream) if not proc: print 'no value to keyword ',kw sys.exit() val = [] while proc: if proc.group('val')[0] == "'": val.append(proc.group('val')[1:len(proc.group('val'))-1]) else: val.append(proc.group('val')) dicoStream = proc.group('after') # still hold the separator proc = re.match(val_equals,dicoStream) keylist.append([kw,val]) # ~~ sort out the groups, starting with 'NOM' dico = {'FR':{},'GB':{},'DICO':dicoFile}; keywords = {} while keylist != []: if keylist[0][0] != 'NOM' and keylist[1][0] != 'NOM1': print 'could not read NOM or NOM1 from ',keylist[0][1] sys.exit() dico['FR'].update({keylist[0][1][0]:keylist[1][1][0]}) dico['GB'].update({keylist[1][1][0]:keylist[0][1][0]}) key = keylist[0][1][0] words = {'NOM':keylist[0][1]}; keylist.pop(0) while keylist != []: if keylist[0][0] == 'NOM': break words.update({keylist[0][0]:keylist[0][1]}) keylist.pop(0) keywords.update({key:words}) return dico,keywords
def translateCAS(cas,frgb): keyLines = [] casLines = getFileContent(cas) # ~~ split comments core = [] for i in range(len(casLines)): casLines[i] = casLines[i].replace('"""',"'''").replace('"',"'") proc = re.match(key_comment,casLines[i]+'/') head = proc.group('before').strip() tail = proc.group('after').rstrip('/') p = re.match(key_none,head+' ') if p: keyLines.insert(0,p.group('key')) head = '' # /!\ here you forget about proc.group('after') if head != '': core.append(head) keyLines.append(head) # /!\ here you forget about tail elif tail != '': keyLines.append(tail) casStream = ' '.join(core) ik = 0; frLines = []; gbLines = [] # ~~ clean values to keep only the keys while casStream != '': # ~~ key proc = re.match(key_equals,casStream) if not proc: print '... hmmm, did not see this one coming ...' break kw = proc.group('key').strip() casStream = proc.group('after') # still hold the separator while 1: p = re.match(re.compile(key_word%(kw),re.I),keyLines[ik]) if not p: frLines.append(keyLines[ik]) gbLines.append(keyLines[ik]) ik = ik + 1 else: keyLines[ik] = p.group('after') if kw in frgb['GB'].keys(): frline = frgb['GB'][kw] gbline = kw if kw in frgb['FR'].keys(): frline = kw gbline = frgb['FR'][kw] break # ~~ val proc = re.match(val_equals,casStream) if not proc: print 'no value to keyword ',kw sys.exit() val = [] while proc: val.append(proc.group('val')) #.replace("'",'')) casStream = proc.group('after') # still hold the separator while 1: p = re.match(re.compile(val_word%(proc.group('val')),re.I),keyLines[ik]) if not p: print '... could not get the values for ',kw sys.exit() keyLines[ik] = p.group('after') if keyLines[ik] == '': ik = ik + 1 break proc = re.match(val_equals,casStream) # in FRENCH for i in range(len(val)): if val[i] in ['YES','Y','TRUE']: val[i] = 'OUI' if val[i] in ['NO','N','FALSE']: val[i] = 'NON' # not more than 72 characters if len(' '+frline+' : '+';'.join(val)) < 73: frline = ' ' + frline + ' : ' + ';'.join(val) else: frline = ' ' + frline + ' :\n' if len(';'.join(val)) < 70: frline = frline + ' ' + ';'.join(val) else: frline = frline + ' ' + ';\n '.join(val) # in ENGLISH for i in range(len(val)): if val[i] in ['OUI','O','VRAI']: val[i] = 'TRUE' if val[i] in ['NON','N','FAUX']: val[i] = 'FALSE' # not more than 72 characters if len(' '+gbline+' : '+';'.join(val)) < 73: gbline = ' ' + gbline + ' : ' + ';'.join(val) else: gbline = ' ' + gbline + ' :\n' if len(';'.join(val)) < 70: gbline = gbline + ' ' + ';'.join(val) else: gbline = gbline + ' ' + ';\n '.join(val) # final append frLines.append(frline) gbLines.append(gbline) # ~~ print FR and GB versions of the CAS file putFileContent(cas+'.fr',frLines) putFileContent(cas+'.gb',gbLines) return
#print ('Number of arguments:', len(sys.argv), 'arguments.', str(sys.argv)) if len(sys.argv) != 2: print('You provided ', (len(sys.argv) - 1), ' arguments. \n Please run parse_module.py <module_name>') sys.exit(2) module = sys.argv[1] moduleReactionFileName = 'output/insert_module_reaction.sql' reactionCompoundsFileName = 'output/insert_reaction_compound.sql' reactionOrderFileName = 'output/insert_reaction_order.sql' #read file data = utils.getFileContent(module) def parseReactions(lineReactions): moduleReactionsFile = open(moduleReactionFileName, 'a') reactionCompoundsFile = open(reactionCompoundsFileName, 'a') reactionOrderFile = open(reactionOrderFileName, 'a') reactions = [] for line in lineReactions: # line looks like R05605 C04442 -> C00022 + C00118 tokens = line.split() lineReactions = re.split('\+|,', tokens[0]) type = 'INPUT' for i in range(1, len(tokens)):
def runCAS(cfgName,cfg,codeName,casFile,options): # ~~~~ Read the DICO File ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ dicoFile = path.join(path.join(cfg['MODULES'][codeName]['path'],'lib'),codeName+cfg['TELVER']+'.dico') frgb,dico = scanDICO(dicoFile) iFS,oFS = getIOFilesSubmit(frgb,dico) # ~~ Read the principal CAS File ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ if not path.exists(casFile): print '... inexistent CAS file: ',casFile return None # /!\ should you stop or carry on ? cas,lang = processCAS(casFile,frgb) if not checkConsistency(cas,dico,frgb,cfg): print '... inconsistent CAS file: ',casFile print ' +> you may be using an inappropriate configuration:',cfgName print ' +> or may be wishing for scalar mode while using parallel' return None # /!\ should you stop or carry on ? # ~~ Handling Directories ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ CASDir = path.dirname(casFile) TMPDir = processTMP(casFile) # ~~ Read the included CAS File ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ cplages,defaut = getKeyWord('COUPLING WITH',cas,dico,frgb) #/!\ having done the loop this way it will not check for DELWAQ COUPLAGE = {} for cplage in cplages: for mod in cfg['MODULES'].keys(): if mod in cplage.lower(): # ~~~~ Extract the CAS File name ~~~~~~~~~~~~~~~~~~~~~~~ casFilePlage,defaut = getKeyWord(mod.upper()+' STEERING FILE',cas,dico,frgb) if casFilePlage == []: casFilePlage = defaut casFilePlage = path.join(CASDir,casFilePlage[0]) if not path.isfile(casFilePlage): print '... missing coupling CAS file for',mod,': ',casFilePlage return None # /!\ should you stop or carry on ? # ~~~~ Read the DICO File ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ dicoFilePlage = path.join(path.join(cfg['MODULES'][mod]['path'],'lib'),mod+cfg['TELVER']+'.dico') frgbPlage,dicoPlage = scanDICO(dicoFilePlage) iFSPlage,oFSPlage = getIOFilesSubmit(frgbPlage,dicoPlage) # ~~ Read the coupled CAS File ~~~~~~~~~~~~~~~~~~~~~~~~~ casPlage,lang = processCAS(casFilePlage,frgbPlage) if not checkConsistency(casPlage,dicoPlage,frgbPlage,cfg): print '... inconsistent CAS file: ',casFilePlage return None # /!\ should you stop or carry on ? COUPLAGE.update({mod:{}}) COUPLAGE[mod].update({'cas':casPlage,'frgb':frgbPlage,'iFS':iFSPlage,'oFS':oFSPlage,'dico':dicoPlage}) # ~~ Handling sortie file ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ sortiefile = None if options.sortieFile: # define the filename (basename) of the sortie file sortiefile = path.basename(TMPDir)+'.sortie' # ~~ Handling all input files ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # >>> Placing yourself where the CAS File is chdir(CASDir) # >>> Copy INPUT files into TMPDir if not processLIT(cas,iFS,TMPDir): sys.exit() for mod in COUPLAGE.keys(): if not processLIT(COUPLAGE[mod]['cas'],COUPLAGE[mod]['iFS'],TMPDir): sys.exit() # >>> Placing yourself into the TMPDir chdir(TMPDir) # >>> Creating LNG file processCONFIG(lang) # ~~ Handling Executable ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # >>> Names for the executable set #> names within TMPDir f90File = iFS['FICHIER FORTRAN'].split(';')[1] #> aggregation of PRINCI files for mod in COUPLAGE.keys(): f90FilePlage = COUPLAGE[mod]['iFS']['FICHIER FORTRAN'].split(';')[1] if path.isfile(f90FilePlage): putFileContent(f90File,getFileContent(f90File)+['']+getFileContent(f90FilePlage)) remove(f90FilePlage) objFile = path.splitext(f90File)[0] + cfg['SYSTEM']['SFX_OBJ'] #> default executable name exeFile = path.join(path.join(cfg['MODULES'][codeName]['path'],cfgName),codeName+cfg['TELVER']+cfg['SYSTEM']['SFX_EXE']) #> user defined executable name useFile = exeFile value,defaut = getKeyWord('FICHIER FORTRAN',cas,dico,frgb) if value != []: useFile = path.join(CASDir,path.splitext(value[0])[0]+cfg['SYSTEM']['SFX_EXE']) if path.exists(useFile) and cfg['REBUILD'] > 0: remove(useFile) #> default command line compilation and linkage if not path.exists(path.join(path.join(cfg['MODULES'][codeName]['path'],cfgName),codeName+cfg['TELVER']+'.cmdo')): print '\nNot able to find your OBJECT command line: ' + path.join(cfgName,codeName+cfg['TELVER']+'.cmdo') + '\n' print ' ... you have to compile this module at least: ' print ' +> ',codeName sys.exit() objCmd = getFileContent(path.join(path.join(cfg['MODULES'][codeName]['path'],cfgName),codeName+cfg['TELVER']+'.cmdo'))[0] if not path.exists(path.join(path.join(cfg['MODULES'][codeName]['path'],cfgName),codeName+cfg['TELVER']+'.cmdx')): print '\nNot able to find your OBJECT command line: ' + path.join(cfgName,codeName+cfg['TELVER']+'.cmdx') + '\n' print ' ... you have to compile this module at least: ' print ' +> ',codeName sys.exit() exeCmd = getFileContent(path.join(path.join(cfg['MODULES'][codeName]['path'],cfgName),codeName+cfg['TELVER']+'.cmdx'))[0] # >>> Compiling the executable if required if not processExecutable(useFile,objFile,f90File,objCmd,exeCmd,CASDir): sys.exit() # >>> Rename executable because of firewall issues ~~~~~~~~~~~~~~ runCmd = path.join(TMPDir,'out_'+path.basename(useFile)) shutil.move(path.basename(useFile),runCmd) if not options.compileonly: # ~~ Handling the parallelisation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ncsize = getNCSIZE(cas,dico,frgb) if ncsize > 1: # >>> MPI configuration # ~~> Executable /!\ you need one if ncsize > 1 mpiCmd = '' if cfg.has_key('MPI'): if cfg['MPI'].has_key('EXEC'): mpiCmd = cfg['MPI']['EXEC'] if mpiCmd == '': print '... I do not know how to run MPI, can you help ?' return None # /!\ should you stop or carry on ? # ~~> Assign the mpi_telemac.conf hosts = '' if cfg.has_key('MPI'): if cfg['MPI'].has_key('HOSTS'): hosts = cfg['MPI']['HOSTS'] # ~~> MPI Command line mpiCmd = mpiCmd.replace('<wdir>','-wdir '+TMPDir) # /!\ Make sure TMPDir works in UNC convention mpiCmd = mpiCmd.replace('<ncsize>','-n '+str(ncsize)) mpiCmd = mpiCmd.replace('<hosts>',hosts) mpiCmd = mpiCmd.replace('<exename>',runCmd) runCmd = mpiCmd # >>> Parallel tools # ~~> Default path PARDir = path.join(cfg['MODULES']['parallel']['path'],cfgName) # ~~> User path if cfg.has_key('PARALLEL'): if cfg['PARALLEL'].has_key('PATH'): PARDir = cfg['PARALLEL']['PATH'].replace('<root>',cfg['TELDIR']).replace('<config>',path.join(cfg['MODULES']['parallel']['path'],cfgName)) # ~~> Creating PARA file and the mpi_telemac.conf processPARALLEL(ncsize,TMPDir+sep) # /!\ Make sure TMPDir works in UNC convention # >>> Running the partionning if ncsize > 1: # ~~> PARTEL Executable exeCmd = path.join(PARDir,'partel'+cfg['SYSTEM']['SFX_EXE']) # ~~> Run PARTEL CONLIM = getCONLIM(cas,iFS) # no check on existence runPartition(exeCmd,cas,CONLIM,iFS,ncsize) for mod in COUPLAGE.keys(): CONLIM = getCONLIM(COUPLAGE[mod]['cas'],COUPLAGE[mod]['iFS']) runPartition(exeCmd,COUPLAGE[mod]['cas'],CONLIM,COUPLAGE[mod]['iFS'],ncsize) # >>> Running the Executable ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ print runCmd if not runCode(runCmd,sortiefile): sys.exit() # >>> Handling the recollection ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ if ncsize > 1: # ~~> GRETEL Executable exeCmd = path.join(PARDir,'gretel_autop'+cfg['SYSTEM']['SFX_EXE']) # ~~> Run GRETEL GLOGEO = getGLOGEO(cas,iFS) # no check on existence runRecollection(exeCmd,cas,GLOGEO,oFS,ncsize) for mod in COUPLAGE.keys(): GLOGEO = getGLOGEO(COUPLAGE[mod]['cas'],COUPLAGE[mod]['iFS']) runRecollection(exeCmd,COUPLAGE[mod]['cas'],GLOGEO,COUPLAGE[mod]['oFS'],ncsize) # ~~ Handling all output files ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ if not processECR(cas,oFS,CASDir,TMPDir,sortiefile,ncsize): sys.exit() for mod in COUPLAGE.keys(): if not processECR(COUPLAGE[mod]['cas'],COUPLAGE[mod]['oFS'],CASDir,TMPDir,'',ncsize): sys.exit() # ~~ Handling Directories ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ chdir(CASDir) if options.tmpdirectory or options.compileonly: removeDirectories(TMPDir) return sortiefile
#!/usr/bin/python3 import re import sys import utils enzyme = sys.argv[1] outputFileName = 'output/insert_enzyme.sql' #read file data = utils.getFileContent(enzyme) names = utils.getSectionString("NAME", data) names = names.replace("'", "''") #escape single quote for postgres outputFile = open(outputFileName, 'a') outputFile.write("INSERT INTO raw_enzyme (id, name) VALUES ('" + enzyme + "' , '" + names + "');\n") outputFile.close()