def processFile(domainName, user, edbName, filePath):
logger=logging.getLogger('loadRUSedb.processFile')
dmn=domain(domainName)
edb=Edb(domainName,user,edbName)
if not dmn.edbUserExist(user):
logger.error("Directory for user '"+user+"' does not exist in edb directory for domain '"+domain+"'")
sys.exit()
if not edb.exists():
edb.create()
logger.info("EDB '"+edb.name+"' for user '"+user+"' in domain '"+dmn.name+"' created")
if filePath[-3:]==".gz":
utilities.gunzip(filePath)
filePath=filePath[:-3]
filePath=filePath[:-5] #removing .data from the file name
#converting from dos-format to linux
utilities.dos2linux(filePath+".asc")
utilities.dos2linux(filePath+".data")
#create grid object
gridname=path.basename(filePath)
grid=Egrid(dmn.name,user,edb.name,gridname)
grid.load(filePath) #load grid into edb
def main():
#-----------Setting up and unsing option parser-----------------------
parser=OptionParser(usage= usage, version=version)
parser.add_option("-u",'--user',
action="store",dest="user",
help="Name of target edb user")
parser.add_option("-e","--edb",
action="store",dest="edb",
help="Name of target edb")
parser.add_option("-v","--viewports",
action="store",dest="viewports",
help="Comma-separated list of area id's to be cut out, default is all")
parser.add_option("-y","--year",
action="store",dest="year",
help="Cut out for given year")
parser.add_option("-l", "--loglevel",
action="store",dest="loglevel",default=2,
help="Sets the loglevel (0-3 where 3=full logging)")
parser.add_option("-s","--suffix",
action="store",dest="suffix",default="v1",
help="Sets suffix to names of generated edb's to support version management, default is 'v1'")
(options, args) = parser.parse_args()
#--------------------Init logger-----------------------
rootLogger = logger.RootLogger(level=options.loglevel)
global log
log = rootLogger.getLogger(sys.argv[0])
#-----------------Validating options-------------------
if options.user is None:
log.error("Need to specify -u <user>")
return 1
if options.edb is None:
log.error("Need to specify -e <edb>")
return 1
if options.year is None:
log.error("Need to specify -y <year>")
return 1
if len(options.suffix)>4:
log.error("Limit your suffix length to 4 characters")
return 1
if len(options.year)!=4:
log.error("Year should be given with four digits")
return 1
dmn=Domain()
viewports=[]
if options.viewports is not None:
viewportIds=options.viewports.split(",")
else:
viewportIds=dmn.listViewports()
for vpId in viewportIds:
vp=ViewPort()
vp.read(path.join(dmn.wndPath(),"modell.par"),vpId)
viewports.append(vp)
edb=Edb(dmn,options.user,options.edb)
log.info("Reading sourcedb...")
sourcedb=Sourcedb(edb)
sourcedb.read()
log.info("Reading emfacdb...")
emfacdb=Emfacdb(edb)
emfacdb.read()
log.info("Reading subdb...")
subdb=Subdb(edb)
subdb.read()
edbDotRsrc=edb.rsrcPath()
for vpInd,vp in enumerate(viewports):
targetEdbName=vp.code+"_"+options.year+"_"+options.suffix
tEdb=Edb(dmn,options.user,targetEdbName)
if tEdb.exists():
log.info("Edb %s already exists, remove first to update" %targetEdbName)
continue
tEdb.create(edbRsrc=edbDotRsrc)
log.info("Created empty edb %s" %targetEdbName)
subdb.setEdb(tEdb)
subdb.write()
log.info("Wrote searchkeys")
emfacdb.setEdb(tEdb)
emfacdb.write()
log.info("Wrote emfacdb")
tSourcedb=Sourcedb(tEdb)
log.info("Cutting out viewport %s (%i/%i)" %(vp.code,vpInd+1,len(viewports)))
for srcInd,src in enumerate(sourcedb.sources):
if includeShip(src,vp.code,src["Y1"],options.year):
log.debug("Ship %i/%i included in %s" %(srcInd+1,len(sourcedb.sources),tEdb.name))
tSourcedb.sources.append(src)
tSourcedb.write()
log.info("Wrote exatracted sources to %s" %tEdb.name)
tEdb.setDesc("This edb has been extracted from %s under user %s, " %(edb.name,edb.user)+
"and includes all ships that have visited the map area %s (%s) during %s\n" %(vp.code,vp.name,options.year))
log.info("Finished!")
return 0
def extractByGeocode(argDict):
"""Extracts sources by geocode and saves to edb's for other users specified
in a file with users, and optionally edb's, specified for each geocode"""
edb = argDict["edb"]
mapping = argDict["edbMapping"]
codeLevel = argDict["codeLevel"]
codeIndex = argDict["codeIndex"]
# Processing grid sources
for grd in argDict["grids"]:
if not match(grd.par["NAME"].val, argDict["filters"]):
continue
# dict with (user,edb) as key and a list of gc to be included as value
subEdbs = {}
allFound = True
for gc in mapping:
# The mapping can contain only users for each geocode, in that case
# The users are stored as values and the original edb name is used
# as default
# If both user and edb name is given, the values are given in a
# list for each gc
if isinstance(mapping[gc], list):
tmp_user = mapping[gc][0]
tmp_edb = mapping[gc][1]
else:
tmp_user = mapping[gc]
tmp_edb = edb.name
subEdb = Edb(argDict["domain"], tmp_user, tmp_edb)
if (tmp_user, tmp_edb) in subEdbs:
subEdbs[(tmp_user, tmp_edb)].append(gc)
else:
subEdbs[(tmp_user, tmp_edb)] = [gc]
# Check if edb exist, and if so if grid exist
# (to save time when updating)
if subEdb.exists():
if grd.name not in subEdb.listGrids():
allFound = False
continue
else:
allFound = False
if allFound:
log.info(
"Splitted grid " +
"%s already exist for all" % grd.name +
" geocodes, remove to update")
continue
log.debug("Reading grid: %s" % grd.name)
grd.readData()
for user_edb, gcList in subEdbs.iteritems():
subEdb = Edb(argDict["domain"], user_edb[0], user_edb[1])
subEdb.rsrc = edb.rsrc
# extract data from grid that maches any of the geocodes in gcList
subGrid = grd.sliceByGeoCode(
codeLevel - 1,
map(int, gcList),
subEdb
)
if subGrid is not None:
if not subEdb.exists():
subEdb.create(edbRsrc=edb.rsrcPath())
log.debug("Created edb for user %s" % subEdb.user)
log.debug(
"Writing grid: %s to edb %s for user %s" % (
grd.name,
subEdb.name,
subEdb.user)
)
subGrid.load()
if argDict["sources"] is not None:
sub_source_writers = {}
# subCompanydbs = {}
# subFacilitydbs = {}
# Read substance groups
subgrpdb = Subgrpdb(edb)
subgrpdb.read()
facilitydb = Facilitydb(edb)
facilitydb.read()
companydb = Companydb(edb)
companydb.read()
for src in argDict["sources"]:
if not match(src.NAME, argDict["filters"]):
continue
# gc = int(src.GEOCODE[codeIndex - 1][codeLevel - 1])
gc = int(src.GEOCODE[codeIndex - 1].split(".")[codeLevel - 1])
if src.PX is not None and src.PX != 0:
facility = facilitydb.index[(src.PX, src.PY)]
company = companydb.index[facility["COMPANY"]]
# If no mapping is provided for a gc it is ignored
if gc not in mapping:
continue
# create sub-edb and corresponding db:s
if gc not in sub_source_writers:
tmp_user = mapping[gc]
subEdb = Edb(argDict["domain"], tmp_user, edb.name)
# If a sub-edb does not exist,
# create a new one with edb.rsrc from original edb
if not subEdb.exists():
subEdb.create(edbRsrc=edb.rsrcPath())
sub_source_writers[gc] = ModelWriter(
SourceStream(subEdb, mode='w')
)
# subCompanydbs[gc] = Companydb(subEdb)
# subFacilitydbs[gc] = Facilitydb(subEdb)
# Write substance groups to subEdb
subgrpdb.setEdb(subEdb)
log.info("Writing subgrpdb for user %s" % subEdb.user)
subgrpdb.write()
# subCompanydbs[gc].append(company, force=True)
# subFacilitydbs[gc].append(facility,force=True)
sub_source_writers[gc].write(src)
# write all sources
for gc in sub_source_writers:
# log.info("Writing companies to edb for gc %i" % gc)
# subCompanydbs[gc].write()
# log.info("Writing facilities to edb for gc %i" % gc)
# subFacilitydbs[gc].write()
log.info("Writing sources to edb for gc %i" % gc)
return argDict
