def processFile(domainName, user, edbName, filePath): logger=logging.getLogger('loadRUSedb.processFile') dmn=domain(domainName) edb=Edb(domainName,user,edbName) if not dmn.edbUserExist(user): logger.error("Directory for user '"+user+"' does not exist in edb directory for domain '"+domain+"'") sys.exit() if not edb.exists(): edb.create() logger.info("EDB '"+edb.name+"' for user '"+user+"' in domain '"+dmn.name+"' created") if filePath[-3:]==".gz": utilities.gunzip(filePath) filePath=filePath[:-3] filePath=filePath[:-5] #removing .data from the file name #converting from dos-format to linux utilities.dos2linux(filePath+".asc") utilities.dos2linux(filePath+".data") #create grid object gridname=path.basename(filePath) grid=Egrid(dmn.name,user,edb.name,gridname) grid.load(filePath) #load grid into edb
def main(): #-----------Setting up and unsing option parser----------------------- parser=OptionParser(usage= usage, version=version) parser.add_option("-u",'--user', action="store",dest="user", help="Name of target edb user") parser.add_option("-e","--edb", action="store",dest="edb", help="Name of target edb") parser.add_option("-v","--viewports", action="store",dest="viewports", help="Comma-separated list of area id's to be cut out, default is all") parser.add_option("-y","--year", action="store",dest="year", help="Cut out for given year") parser.add_option("-l", "--loglevel", action="store",dest="loglevel",default=2, help="Sets the loglevel (0-3 where 3=full logging)") parser.add_option("-s","--suffix", action="store",dest="suffix",default="v1", help="Sets suffix to names of generated edb's to support version management, default is 'v1'") (options, args) = parser.parse_args() #--------------------Init logger----------------------- rootLogger = logger.RootLogger(level=options.loglevel) global log log = rootLogger.getLogger(sys.argv[0]) #-----------------Validating options------------------- if options.user is None: log.error("Need to specify -u <user>") return 1 if options.edb is None: log.error("Need to specify -e <edb>") return 1 if options.year is None: log.error("Need to specify -y <year>") return 1 if len(options.suffix)>4: log.error("Limit your suffix length to 4 characters") return 1 if len(options.year)!=4: log.error("Year should be given with four digits") return 1 dmn=Domain() viewports=[] if options.viewports is not None: viewportIds=options.viewports.split(",") else: viewportIds=dmn.listViewports() for vpId in viewportIds: vp=ViewPort() vp.read(path.join(dmn.wndPath(),"modell.par"),vpId) viewports.append(vp) edb=Edb(dmn,options.user,options.edb) log.info("Reading sourcedb...") sourcedb=Sourcedb(edb) sourcedb.read() log.info("Reading emfacdb...") emfacdb=Emfacdb(edb) emfacdb.read() log.info("Reading subdb...") subdb=Subdb(edb) subdb.read() edbDotRsrc=edb.rsrcPath() for vpInd,vp in enumerate(viewports): targetEdbName=vp.code+"_"+options.year+"_"+options.suffix tEdb=Edb(dmn,options.user,targetEdbName) if tEdb.exists(): log.info("Edb %s already exists, remove first to update" %targetEdbName) continue tEdb.create(edbRsrc=edbDotRsrc) log.info("Created empty edb %s" %targetEdbName) subdb.setEdb(tEdb) subdb.write() log.info("Wrote searchkeys") emfacdb.setEdb(tEdb) emfacdb.write() log.info("Wrote emfacdb") tSourcedb=Sourcedb(tEdb) log.info("Cutting out viewport %s (%i/%i)" %(vp.code,vpInd+1,len(viewports))) for srcInd,src in enumerate(sourcedb.sources): if includeShip(src,vp.code,src["Y1"],options.year): log.debug("Ship %i/%i included in %s" %(srcInd+1,len(sourcedb.sources),tEdb.name)) tSourcedb.sources.append(src) tSourcedb.write() log.info("Wrote exatracted sources to %s" %tEdb.name) tEdb.setDesc("This edb has been extracted from %s under user %s, " %(edb.name,edb.user)+ "and includes all ships that have visited the map area %s (%s) during %s\n" %(vp.code,vp.name,options.year)) log.info("Finished!") return 0
def extractByGeocode(argDict): """Extracts sources by geocode and saves to edb's for other users specified in a file with users, and optionally edb's, specified for each geocode""" edb = argDict["edb"] mapping = argDict["edbMapping"] codeLevel = argDict["codeLevel"] codeIndex = argDict["codeIndex"] # Processing grid sources for grd in argDict["grids"]: if not match(grd.par["NAME"].val, argDict["filters"]): continue # dict with (user,edb) as key and a list of gc to be included as value subEdbs = {} allFound = True for gc in mapping: # The mapping can contain only users for each geocode, in that case # The users are stored as values and the original edb name is used # as default # If both user and edb name is given, the values are given in a # list for each gc if isinstance(mapping[gc], list): tmp_user = mapping[gc][0] tmp_edb = mapping[gc][1] else: tmp_user = mapping[gc] tmp_edb = edb.name subEdb = Edb(argDict["domain"], tmp_user, tmp_edb) if (tmp_user, tmp_edb) in subEdbs: subEdbs[(tmp_user, tmp_edb)].append(gc) else: subEdbs[(tmp_user, tmp_edb)] = [gc] # Check if edb exist, and if so if grid exist # (to save time when updating) if subEdb.exists(): if grd.name not in subEdb.listGrids(): allFound = False continue else: allFound = False if allFound: log.info( "Splitted grid " + "%s already exist for all" % grd.name + " geocodes, remove to update") continue log.debug("Reading grid: %s" % grd.name) grd.readData() for user_edb, gcList in subEdbs.iteritems(): subEdb = Edb(argDict["domain"], user_edb[0], user_edb[1]) subEdb.rsrc = edb.rsrc # extract data from grid that maches any of the geocodes in gcList subGrid = grd.sliceByGeoCode( codeLevel - 1, map(int, gcList), subEdb ) if subGrid is not None: if not subEdb.exists(): subEdb.create(edbRsrc=edb.rsrcPath()) log.debug("Created edb for user %s" % subEdb.user) log.debug( "Writing grid: %s to edb %s for user %s" % ( grd.name, subEdb.name, subEdb.user) ) subGrid.load() if argDict["sources"] is not None: sub_source_writers = {} # subCompanydbs = {} # subFacilitydbs = {} # Read substance groups subgrpdb = Subgrpdb(edb) subgrpdb.read() facilitydb = Facilitydb(edb) facilitydb.read() companydb = Companydb(edb) companydb.read() for src in argDict["sources"]: if not match(src.NAME, argDict["filters"]): continue # gc = int(src.GEOCODE[codeIndex - 1][codeLevel - 1]) gc = int(src.GEOCODE[codeIndex - 1].split(".")[codeLevel - 1]) if src.PX is not None and src.PX != 0: facility = facilitydb.index[(src.PX, src.PY)] company = companydb.index[facility["COMPANY"]] # If no mapping is provided for a gc it is ignored if gc not in mapping: continue # create sub-edb and corresponding db:s if gc not in sub_source_writers: tmp_user = mapping[gc] subEdb = Edb(argDict["domain"], tmp_user, edb.name) # If a sub-edb does not exist, # create a new one with edb.rsrc from original edb if not subEdb.exists(): subEdb.create(edbRsrc=edb.rsrcPath()) sub_source_writers[gc] = ModelWriter( SourceStream(subEdb, mode='w') ) # subCompanydbs[gc] = Companydb(subEdb) # subFacilitydbs[gc] = Facilitydb(subEdb) # Write substance groups to subEdb subgrpdb.setEdb(subEdb) log.info("Writing subgrpdb for user %s" % subEdb.user) subgrpdb.write() # subCompanydbs[gc].append(company, force=True) # subFacilitydbs[gc].append(facility,force=True) sub_source_writers[gc].write(src) # write all sources for gc in sub_source_writers: # log.info("Writing companies to edb for gc %i" % gc) # subCompanydbs[gc].write() # log.info("Writing facilities to edb for gc %i" % gc) # subFacilitydbs[gc].write() log.info("Writing sources to edb for gc %i" % gc) return argDict