Example #1
0
def main():
    #-----------Setting up and unsing option parser-----------------------
    parser=OptionParser(usage= usage, version=version)
    
    parser.add_option("-u",'--user',
                      action="store",dest="user",
                      help="Name of target edb user")

    parser.add_option("-e","--edb",
                      action="store",dest="edb",
                      help="Name of target edb")

    parser.add_option("-i","--infile",
                      action="store",dest="infile",
                      help="Input csv file")
    
 #   parser.add_option("-y","--year",
 #                     action="store",dest="year",
 #                     help="Only store sources for given year")
    
    parser.add_option("-v",dest='loglevel',
                      action="store_const",default=get_loglevel(),
                      help="produce verbose output")

    parser.add_option("-t", "--template",
                      action="store",dest="cf",default=None,
                      help="Generate default controlfile")

    parser.add_option("-o", "--outfile",
                      action="store",dest="outfile",default=None,
                      help="Name of outfiles (without extension)")

    parser.add_option("-d","--delimiter",
                      action="store",dest="delimiter",default="\t",
                      help="Delimiter used in csv-file")

    parser.add_option("-c","--filterCol",
                      action="store",dest="filterCol",
                      help="Header of column to use as filter")

    parser.add_option("-f","--filterVal",
                      action="store",dest="filterVal",
                      help="Value to use in filter")


#    parser.add_option("-g", "--geocodeRasterDir",
#                      action="store",dest="geocodeRasterDir",default=None,
#                      help="Directory with geocode rasters")
    


    (options, args) = parser.parse_args()
    
    #--------------------Init logger-----------------------
#     rootLogger = logger.RootLogger(level=options.loglevel)
    logging.basicConfig(
            format='%(levelname)s:%(name)s: %(message)s',
            level=options.loglevel,
    )
    global log
#     log = rootLogger.getLogger(sys.argv[0])
    log = logging.getLogger(parser.prog)

    #-----------------Validating options-------------------

    if options.cf is not None:
        generateCf(path.abspath(options.cf),controlFileTemplate)
        log.info("Wrote default controlfile")
        return 1

    if options.user is None:
        log.error("Need to specify -u <user>")
        return 1
    if options.edb is None:
        log.error("Need to specify -e <edb>")
        return 1
#    if options.year is None:
#        log.error("Need to specify -y <year>")
#        return 1
#    if len(options.year)!=4:
#        log.error("Year should be given with four digits")
#        return 1


    if len(args)!=1:
        log.error("Controlfile should be given as argument")
        return 1

    dmn=Domain()
    edb=Edb(dmn,options.user,options.edb)
    if not edb.exists():
        log.error("Edb %s does not exist" %options.edb)
        return 1

    log.info("Parsing controlfile")

    cf=ControlFile(args[0])
    cdbPars=re.compile("companydb\.par\.(\w*?):").findall(cf.content)
    fdbPars=re.compile("facilitydb\.par\.(\w*?):").findall(cf.content)
    sdbPars=re.compile("sourcedb\.par\.(\w*?):").findall(cf.content)
    substEmisNr=re.compile("sourcedb\.subst_emis\.([0-9]*)\.emis").findall(cf.content)
    subgrpEmisNr=re.compile("sourcedb\.subgrp_emis\.([0-9]*)\.emis").findall(cf.content)

    cdbCols={}
    cdbDefaults={}
    for par in cdbPars:
        cdbCols[par]=cf.findString("companydb.par.%s:" %par)
        cdbDefaults[par]=cf.findString("companydb.par.%s.default:" %par,
                                       optional=True,default=None)
    fdbCols={}
    fdbDefaults={}
    for par in fdbPars:
        fdbCols[par]=cf.findString("facilitydb.par.%s:" %par)
        fdbDefaults[par]=cf.findString("facilitydb.par.%s.default:" %par,
                                       optional=True,default=None)

    sdbCols={}
    sdbDefaults={}
    for par in sdbPars:
        sdbCols[par]=cf.findString("sourcedb.par.%s:" %par)
        sdbDefaults[par]=cf.findString("sourcedb.par.%s.default:" %par,
                                       optional=True,default=None)

    substEmisCols={}
    substEmisDefaults={}
    if substEmisNr is not None:
        for emisNr in substEmisNr:
            cols={}
            defaults={}
            emisPars=re.compile("sourcedb\.subst_emis\.%s\.(\w*?):" %(emisNr)).findall(cf.content)
            emisDefaultPars=re.compile(
                "sourcedb\.subst_emis\.%s\.(\w*?)\.default:" %(emisNr)).findall(cf.content)
            if emisPars is not None:
                for par in emisPars:            
                    cols[par]=cf.findString("sourcedb.subst_emis.%s.%s:" %(emisNr,par))        
            if emisDefaultPars is not None:        
                for par in emisDefaultPars:    
                    defaults[par]=cf.findString("sourcedb.subst_emis.%s.%s.default:" %(emisNr,par),
                                                optional=True,default=None)
            substEmisCols[emisNr]=cols
            substEmisDefaults[emisNr]=defaults

    subgrpEmisCols={}
    subgrpEmisDefaults={}
    if subgrpEmisNr is not None:
        for emisNr in subgrpEmisNr:
            cols={}
            defaults={}
            emisPars=re.compile("sourcedb\.subgrp_emis\.%s\.(\w*?):" %(emisNr)).findall(cf.content)
            emisDefaultPars=re.compile(
                "sourcedb\.subgrp_emis\.%s\.(\w*?)\.default:" %(emisNr)).findall(cf.content)
            if emisPars is not None:
                for par in emisPars:            
                    cols[par]=cf.findString("sourcedb.subgrp_emis.%s.%s:" %(emisNr,par))        
            if emisDefaultPars is not None:        
                for par in emisDefaultPars:    
                    defaults[par]=cf.findString("sourcedb.subgrp_emis.%s.%s.default:" %(emisNr,par),
                                                optional=True,default=None)
            subgrpEmisCols[emisNr]=cols
            subgrpEmisDefaults[emisNr]=defaults
        

    log.info("Reading subdb...")
    subdb=Subdb(edb)
    subdb.read()

    log.info("Reading companydb...")
    companydb=Companydb(edb)
    companydb.read()

    log.info("Reading sourcedb...")
#     source_stream = SourceStream(edb, 'w')
    source_stream = open(options.outfile, 'w')
    source_writer = ModelWriter(source_stream,encoding="HP Roman8")


    log.info("Reading facilitydb...")
    facilitydb=Facilitydb(edb)
    facilitydb.read()


    log.info("Reading subgrpdb")
    subgrpdb=Subgrpdb(edb)

    subgrpdb.read()

    log.info("Reading edb.rsrc")
    rsrc=Rsrc(edb.rsrcPath())
    
    acCodeTables=[]
    for i in range(rsrc.numberOfCodeTrees("ac")):
        acCodeTables.append(CodeTable(rsrc.path,codeType="ac",codeIndex=i+1))

    gcCodeTables=[]
    for i in range(rsrc.numberOfCodeTrees("gc")):
        gcCodeTables.append(CodeTable(rsrc.path,codeType="gc",codeIndex=i+1))
        

    geocodeRasters=[]
    rast1=Raster()
    rast1.read("/usr/airviro/data/geo/topdown/dynamicRasters/dynamic__GEOCODE__1.txt")
    rast2=Raster()
    rast2.read("/usr/airviro/data/geo/topdown/dynamicRasters/dynamic__GEOCODE__2.txt")
    geocodeRasters.append(rast1)
    geocodeRasters.append(rast2)

    log.info("Reading csv-file")
    table=DataTable()
    table.read(options.infile,delimiter=options.delimiter,encoding="ISO-8859-15")

    if options.filterCol is not None:
        if options.filterCol not in table.colIndex:
            log.error("Filter column header not found in table")
            sys.exit(1)

    invalid=False
    nFiltered=0
    nRows=0

    log.info("Processing rows")
    for rowInd,row in enumerate(table.data):
        nRows+=1
        if options.filterCol is not None:
            filterVal=row[table.colIndex[options.filterCol]]
            if options.filterVal!=str(filterVal):
                nFiltered+=1
                continue

        comp = Company()
        for par in comp.parOrder:
            val=cdbDefaults.get(par,None)
            if par in cdbCols:
                colId=cdbCols[par]
                try:
                    tableVal=row[table.colIndex[colId]]
                except KeyError:
                    log.error(
                        "No column with header %s, columns: %s" %(
                            colId,str(table.listIds())))
                if tableVal is not None:
                    val = tableVal
            if val is not None:
                #Too long names are truncated
                if par=="NAME" and len(val)>45:
                    val=val[:45]
                comp[par]=val

        fac = Facility()
        for par in fac.parOrder:
            val=fdbDefaults.get(par,None)
            if par in fdbCols:
                colId=fdbCols[par]
                tableVal=row[table.colIndex[colId]]
                if tableVal is not None:
                    val = tableVal
            if val is not None:
                #Too long names are truncated
                if par=="NAME" and len(val)>45:
                    val=val[:45]
                fac[par]=val

        src = Source()       
        for par in ["X1", "Y1","X2","Y2",
                    "PX","PY","NAME","INFO","INFO2","DATE","CHANGED",
                    "CHIMNEY HEIGHT","GASTEMPERATURE","GAS FLOW",
                    "SEARCHKEY1","SEARCHKEY2","SEARCHKEY3",
                    "SEARCHKEY4","SEARCHKEY5","CHIMNEY OUT","CHIMNEY IN",
                    "HOUSE WIDTH","HOUSE HEIGHT","NOSEGMENTS","BUILD_WIDTHS",
                    "BUILD_HEIGHTS","BUILD_LENGTHS","BUILD_DISTFARWALL",
                    "BUILD_CENTER","GEOCODE","FORMULAMACRO","ALOB"]:
            val=sdbDefaults.get(par,None)
            if par in sdbCols:
                colId=sdbCols[par]
                tableVal=row[table.colIndex[colId]]
                if tableVal is not None:
                    val = tableVal
            if val is not None:
                #validate code
                if par=="GEOCODE" and val is not None:
                    gcList=val.split()
                    for codeIndex,code in enumerate(gcList):
                        if not gcCodeTables[codeIndex].hasCode(code):
                            log.error("Invalid geo code %s on row %i" %(code,rowInd))
                            invalid=True
                #Too long names are truncated
                if par=="NAME" and len(val)>45:
                    val=val[:45]

                #Store in src object and convert to correct type
                src._fieldvalues[par] = lazy_parse(
                    src, par, val)

        gc1=geocodeRasters[0].getVal(src.get_coord()[0],src.get_coord()[1])
        gc2=geocodeRasters[1].getVal(src.get_coord()[0],src.get_coord()[1])
        src.GEOCODE = [str(int(gc1)) + "." + str(int(gc2))]

        for emisNr,emis in substEmisCols.items():
            substEmis={"unit":None,"ac":None,"substance":None,"emis":None}

            for par in substEmis.keys():
                if par in emis:
                    substEmis[par]=row[table.colIndex[emis[par]]]
                else:
                    try:
                        substEmis[par]=substEmisDefaults[emisNr][par]
                    except KeyError:
                        log.error(
                            "Need to specify column or default value for subgrp emis %i" %emisNr)

            
            substInd=subdb.substIndex(substEmis["substance"])
            if substInd is None:
                log.error("Invalid substance name %s on row %i" %(
                        substEmis["substance"],rowInd))
                sys.exit(1)

            try:
                unit=rsrc.sub[substEmis["unit"]]
            except KeyError:
                log.error("Invalid unit name %s on row %i" %(emis["unit"],rowInd))
                sys.exit(1)

            acList=substEmis["ac"].split('\\')[0].split()
            for codeIndex,code in enumerate(acList):
#                 if code == "2.A.4.2":
#                     import pdb; pdb.set_trace()
                refCode = acCodeTables[codeIndex].checkCode(code)
                if refCode == "-":
                    log.error("Invalid activity code %s on row %i" %(code,rowInd))
                    sys.exit(1)
                if refCode != code:
                    acList[codeIndex] = refCode
            substEmis["ac"] = acList
            
            if substEmis["emis"] is not None and substEmis["emis"]!="0":
                try:
                    emis = src.add_emission()
                    emis.UNIT = substEmis["unit"] 
                    emis.ACTCODE = substEmis["ac"]  # needs re-formatting
                    emis.EMISSION = float(substEmis["emis"])
                    emis.SUBSTANCE = substInd

                    emis.auto_adjust_unit(edb) 


                except:            
#                     print substEmis
#                     log.error("Invalid substance emission on row %i" %rowInd)
                    invalid=True
                    src.EMISSION=src.EMISSION[:-1]


        for emis in subgrpEmisCols.values():
            subgrpEmis={"unit":None,"ac":None,"name":None,"emis":None}
            for par in subgrpEmis.keys():
                if par in emis:
                    subgrpEmis[par]=row[table.colIndex[emis[par]]]
                else:
                    try:
                        subgrpEmis[par]=subgrpEmisDefaults[emisNr][par]
                    except KeyError:
                        log.error(
                            "Need to specify column or default value for subgrp emis %i" %emisNr)

            #validating subgrp name
            try:                        
                subgrp=subgrpdb.getByName(subgrpEmis["name"])
            except KeyError:
                log.error("Invalid subgrp name %s on row %i" %(subgrpEmis["name"],rowInd))
                invalid=True

            #validating subgrp emis unit
            try:                
                unitFactor=rsrc.subGrpEm[subgrpEmis["unit"]]
            except KeyError:
                log.error("Invalid unit %s for subgrp emission on row %i" %(
                        subgrpEmis["unit"],rowInd))
                invalid=True

            #validating subgrp activity code
            acList=subgrpEmis["ac"].split()
            for codeIndex,code in enumerate(acList):
                refCode = acCodeTables[codeIndex].checkCode(code)
                if refCode == "-":
                    log.error("Invalid activity code %s on row %i" %(code,rowInd))
                    invalid=True
                    break
                if refCode != code:
                    acList[codeIndex] = refCode
            substEmis["ac"] = acList

            try:
                src.addSubgrpEmis(subgrp.index,emis=subgrpEmis["emis"],unit=subgrpEmis["unit"],
                                  ac=subgrpEmis["ac"])
            except:            
                log.error("Invalid subgrp emission on row %i" %rowInd)
                invalid=True

        companydb.append(comp,force=True)
        facilitydb.append(fac,force=True)
        source_writer.write(src)
#         sourcedb.append(src)

    if invalid:
        log.info("No output written due to validation errors")
        sys.exit(0)
    if len(companydb.items)>0:
        if options.outfile is None:
            log.info("Writing companydb")
        else:
            log.info("Writing company db to file")
        companydb.write(filename=options.outfile+".companydb")

    if len(facilitydb.items)>0:
        if options.outfile is  None:
            log.info("Writing facilitydb")
        else:
            log.info("Writing facilitydb to file")
        facilitydb.write(filename=options.outfile+".facilitydb")

#     if len(sourcedb.sources)>0:
#         if options.outfile is None:
#             log.info("Writing sourcedb")
#         else:
#             log.info("Writing sourcedb to file")
#     sourcedb.write(filename=options.outfile+".sourcedb")

    if options.filterCol is not None:
        log.info("Filtered out %i out of %i" %(nFiltered,nRows))
def main():
    # setting up parser
    parser = OptionParser(usage=usage, version=version)
    parser.add_option("-r", "--rsrc", action="store", dest="rsrcPath", default=None, help="Path to edb.rsrc")

    parser.add_option(
        "--gcIndex", action="store", dest="gcIndex", default=1, help="Geocode tree index to use (first is 1)"
    )

    parser.add_option(
        "--acIndex", action="store", dest="acIndex", default=1, help="Activitycode tree index to use (first is 1)"
    )

    (options, args) = parser.parse_args()

    logger = logging.getLogger("trendReport")

    if len(args) != 2:
        parser.error("Incorrect number of arguments")

    infile = path.abspath(args[0])  # file with emission data per code
    outdir = path.abspath(args[1])  # path for output excel report

    if not path.exists(infile):
        logger.error("Input asciiReport does not exist")
        sys.exit(-1)

    rsrcPath = path.abspath(options.rsrcPath)
    if not path.exists(rsrcPath):
        logger.error("Specified edb.rsrc does not exist")
        sys.exit(-1)
    rsrc = Rsrc(rsrcPath)

    nac = rsrc.numberOfCodeTrees("ac")
    ngc = rsrc.numberOfCodeTrees("gc")

    acIndex = int(options.acIndex)
    gcIndex = int(options.gcIndex)

    if acIndex > nac:
        logger.error("Invalid ac index specified, should be <= %i" % nac)

    if gcIndex > ngc:
        logger.error("Invalid gc index specified, should be <= %i" % ngc)

    # reading emission data file
    f = codecs.open(infile, "r", "HP Roman8")
    dlines = f.readlines()
    f.close()

    # parsing report meta-data
    # dlines.pop(0) #removing html header
    # dlines.pop(0) #removing blank row
    name = dlines.pop(0)[5:].strip()
    nrmacros = int(dlines.pop(0).split()[1])
    nrsubstances = int(dlines.pop(0).split()[1])
    units = dlines.pop(0)[5:].strip()
    macros = []
    for mIndex in range(nrmacros):
        edbuser = dlines.pop(0).split()[1]
        edbname = dlines.pop(0).split()[1]
        label = dlines.pop(0).split()[1]
        macros.append({"edbuser": edbuser, "edbname": edbname, "label": label})

    # Creating codeTable object (for look-ups of activitycode names and geocodenames)
    gcTable = codetable.CodeTable(rsrcPath, "gc", gcIndex)
    acTable = codetable.CodeTable(rsrcPath, "ac", acIndex)

    # parsing emission data file
    macroIndex = 0
    emissions = OrderedDict()
    substances = []
    for line in dlines:
        data = line.split()
        if data[0] == """#MACRO""":  # if header, update substance and macro index
            vals = line.split('"')
            macroIndex = int(data[1])
            subst = vals[1]
            if subst not in substances:
                substances.append(subst)
        else:  # else get codes and corresponding name
            ac = data[1]
            ac = ac.split(".")
            if ac[0] == "<all>":  # add "all" to get the right depth in the dictionary
                ac = ["all", "all"]
            elif len(ac) == 1:
                ac.append("all")

            gc = data[3]
            gc = gc.split(".")
            if gc[0] == "<all>":
                gc = ["all", "all"]
            elif len(gc) == 1:
                gc.append("all")

            gcLev1Name = gcTable.codes[gc[0]]["name"]
            if gc[1] != "all":
                gcLev2Name = gcTable.codes[gc[0]][gc[1]]["name"]
            else:
                gcLev2Name = "Alla"

            acLev1Name = acTable.codes[ac[0]]["name"]
            if ac[1] != "all":
                acLev2Name = acTable.codes[ac[0]][ac[1]]["name"]
            else:
                acLev2Name = "Alla"
            val = data[4]

            # Quick and dirty construction of a tree-structure
            label = macros[macroIndex]["label"]
            if label not in emissions.keys():
                emissions[label] = OrderedDict()
            if gcLev1Name not in emissions[label].keys():
                emissions[label][gcLev1Name] = OrderedDict()
            if gcLev2Name not in emissions[label][gcLev1Name].keys():
                emissions[label][gcLev1Name][gcLev2Name] = OrderedDict()
            if acLev1Name not in emissions[label][gcLev1Name][gcLev2Name].keys():
                emissions[label][gcLev1Name][gcLev2Name][acLev1Name] = OrderedDict()
            if acLev2Name not in emissions[label][gcLev1Name][gcLev2Name][acLev1Name].keys():
                emissions[label][gcLev1Name][gcLev2Name][acLev1Name][acLev2Name] = OrderedDict()
            if float(val) == 0:
                emissions[label][gcLev1Name][gcLev2Name][acLev1Name][acLev2Name][subst] = -9999
            else:
                emissions[label][gcLev1Name][gcLev2Name][acLev1Name][acLev2Name][subst] = val

    # quick and dirty retrieval from emission tree structure
    # and writing to regional output tables (geocode level 1)
    regionReports = OrderedDict()
    regionIndex = 0
    regionIndexDict = OrderedDict()
    sectorIndexDict = OrderedDict()

    # Traverse the emissionTree to extract emission into a number of different reports
    for year, yearEmis in emissions.iteritems():
        for gcLev1Name, gcLev1Emis in yearEmis.iteritems():
            if gcLev1Name not in regionReports.keys():
                # Creating sub-dictionary for sector specific reports
                regionReports[gcLev1Name] = OrderedDict()

                # Creating fileobject for main-sector report and writing header
                mainSectorData = codecs.open(
                    path.join(outdir, "region_" + str(regionIndex) + "_sector_main.txt"), "w", "HP Roman8"
                )
                mainSectorData.write("Report:\t" + name + "\tUnits:\t" + units + "\tRegion:\t" + gcLev1Name + "\n")
                mainSectorData.write("Year\tHuvudsektor\tUndersektor\t")
                for s in substances:
                    mainSectorData.write(s + "\t")
                mainSectorData.write("\n")
                regionReports[gcLev1Name]["Main"] = mainSectorData
                regionIndexDict[gcLev1Name] = regionIndex
                regionIndex += 1

            # Loop over all first-level geo-codes (specifying second level geocodes to 'Alla')
            sectorIndex = 0
            for acLev1Name, acLev1Emis in gcLev1Emis["Alla"].iteritems():
                if acLev1Name not in regionReports[gcLev1Name]:
                    sectorIndexDict[acLev1Name] = sectorIndex
                    acData = codecs.open(
                        path.join(
                            outdir,
                            "region_" + str(regionIndexDict[gcLev1Name]) + "_sector_" + str(sectorIndex) + ".txt",
                        ),
                        "w",
                        "HP Roman8",
                    )
                    acData.write("Report:\t" + name + "\tUnits:\t" + units + "\tRegion:\t" + gcLev1Name + "\n")
                    acData.write("Year\tMainsector\tSubsector\t")
                    for s in substances:
                        acData.write(s + "\t")
                    acData.write("\n")
                    regionReports[gcLev1Name][acLev1Name] = acData
                    sectorIndex += 1

                # Writing main sector emissions to a separate file for each region
                regionReports[gcLev1Name]["Main"].write(year + "\t" + acLev1Name + "\tAlla\t")
                for s in substances:
                    if s not in acLev1Emis["Alla"].keys():
                        regionReports[gcLev1Name]["Main"].write("-9999\t")
                    elif float(acLev1Emis["Alla"][s]) == -9999:
                        regionReports[gcLev1Name]["Main"].write("-9999\t")
                    elif s in ekvFactors:
                        regionReports[gcLev1Name]["Main"].write(
                            str(float(acLev1Emis["Alla"][s]) * float(ekvFactors[s])) + "\t"
                        )
                    else:
                        regionReports[gcLev1Name]["Main"].write(str(acLev1Emis["Alla"][s]) + "\t")
                regionReports[gcLev1Name]["Main"].write("\n")

                # Writing sub sector emissions to separate files for each combination of region and main sector
                for acLev2Name, acLev2Emis in acLev1Emis.iteritems():
                    regionReports[gcLev1Name][acLev1Name].write(year + "\t" + acLev1Name + "\t" + acLev2Name + "\t")

                    for s in substances:
                        if s not in acLev2Emis.keys():
                            regionReports[gcLev1Name][acLev1Name].write("-9999\t")
                        elif float(acLev2Emis[s]) == -9999:
                            regionReports[gcLev1Name][acLev1Name].write("-9999\t")
                        elif s in ekvFactors:
                            regionReports[gcLev1Name][acLev1Name].write(
                                str(float(acLev2Emis[s]) * float(ekvFactors[s])) + "\t"
                            )
                        else:
                            regionReports[gcLev1Name][acLev1Name].write(str(acLev2Emis[s]) + "\t")
                    regionReports[gcLev1Name][acLev1Name].write("\n")

    # Creating fileobjects for look-up tables for sector and region name
    regionSpec = codecs.open(path.join(outdir, "regions.txt"), "w", "HP Roman8")
    for key, value in regionIndexDict.iteritems():
        regionSpec.write(str(value) + " " + '"' + key + '"\n')
    regionSpec.close()

    sectorSpec = codecs.open(path.join(outdir, "sectors.txt"), "w", "HP Roman8")
    for key, value in sectorIndexDict.iteritems():
        sectorSpec.write(str(value) + " " + '"' + key + '"\n')
    sectorSpec.close()

    for r in regionReports.values():
        if isinstance(r, dict):
            for s in r.values():
                s.close()
        else:
            r.close()
Example #3
0
def main():
    #-----------Setting up and unsing option parser-----------------------
    parser=OptionParser(usage= usage, version=version)
    
    parser.add_option('--src_user',
                      action="store",dest="src_user",
                      help="Name of target edb user")

    parser.add_option("--src_edb",
                      action="store",dest="src_edb",
                      help="Name of target edb")

    parser.add_option('--tgt_user',
                      action="store",dest="tgt_user",
                      help="Name of target edb user")

    parser.add_option("--tgt_edb",
                      action="store",dest="tgt_edb",
                      help="Name of target edb")

    parser.add_option("--codeIndex",
                      action="store",dest="codeIndex",
                      help="Activity code index to use for filter")

    parser.add_option("--codeValue",
                      action="store",dest="codeValue",
                      help="Activity code value to filter by")

    parser.add_option("-l", "--loglevel",
                      action="store",dest="loglevel",default=2,
                      help="Sets the loglevel (0-3 where 3=full logging)")

    parser.add_option("-o", "--outfile",
                      action="store",dest="outfile",
                      help="Output filenam, leave out to write directly to target edb")

    (options, args) = parser.parse_args()
    
    #--------------------Init logger-----------------------
    rootLogger = logger.RootLogger(level=options.loglevel)
    global log
    log = rootLogger.getLogger(sys.argv[0])

    #-----------------Validating options-------------------

    if options.src_user is None:
        log.error("Need to specify --src_user <user>")
        return 1
    if options.src_edb is None:
        log.error("Need to specify --src_edb <edb>")
        return 1

    if options.tgt_user is None:
        log.error("Need to specify --tgt_user <user>")
        return 1
    if options.tgt_edb is None:
        log.error("Need to specify --tgt_edb <edb>")
        return 1


    if len(args)!=0:
        log.error("No argument expected")
        return 1

    dmn=Domain()
    src_edb=Edb(dmn,options.src_user,options.src_edb)
    tgt_edb=Edb(dmn,options.tgt_user,options.tgt_edb)

    if not src_edb.exists():
        log.error("Source edb %s does not exist") %options.src_edb
        return 1

    if not tgt_edb.exists():
        log.error("Target edb %s does not exist") %options.tgt_edb
        return 1

    if options.codeIndex is None:
        log.error("No code index specified")
        return 1

    if options.codeValue is None:
        log.error("No code value specified")
        return 1

    log.info("Reading subdb...")
    src_subdb=Subdb(src_edb)
    src_subdb.read()

    log.info("Reading sourcedb...")
    src_sourcedb=Sourcedb(src_edb)
    src_sourcedb.read()
    

    tgt_sourcedb=Sourcedb(tgt_edb)
    tgt_sourcedb.read()

    #log.info("Reading subgrpdb")
    #src_subgrpdb=Subgrpdb(src_edb)
    #src_subgrpdb.read()

    log.info("Reading edb.rsrc")
    src_rsrc=Rsrc(src_edb.rsrcPath())
    tgt_rsrc=Rsrc(tgt_edb.rsrcPath())

    log.info("Reading target sourcedb...")


    acCodeTables=[]
    for i in range(tgt_rsrc.numberOfCodeTrees("ac")):
        acCodeTables.append(CodeTable(tgt_rsrc.path,codeType="ac",codeIndex=i+1))

    gcCodeTables=[]
    for i in range(tgt_rsrc.numberOfCodeTrees("gc")):
        gcCodeTables.append(CodeTable(tgt_rsrc.path,codeType="gc",codeIndex=i+1))



    copiedSources=0
    readSources=0
    for src in src_sourcedb.sources:
        hasEmis=False
        src["PX"]=0
        src["PY"]=0
                      
        toBeRemoved=[]
        for substInd,emis in src.subst_emis.iteritems():
            ac= emis["ACTCODE"][int(options.codeIndex)-1]
            ac=[c for c in ac if c!="-"]
            emis["ACTCODE"][int(options.codeIndex)-1]=ac #Remove '-'from code

            ac=".".join(ac)
            
            if ac!=options.codeValue:
                toBeRemoved.append(substInd)

        for key in toBeRemoved:
            src.removeSubstEmis(key)

        if src.subst_emis!={}:
            hasEmis=True
        
        toBeRemoved=[]
        for subgrpInd,emis in src.subgrp_emis.iteritems():
            ac= emis["ACTCODE"][int(options.codeIndex)-1]
            ac=[c for c in ac if c!="-"]
            emis["ACTCODE"][int(options.codeIndex)-1]=ac #Remove '-'from code
            ac=".".join(ac)

            
            if ac!=options.codeValue:
                toBeRemoved.append(subgrpInd)

        for key in toBeRemoved:
            src.removeSubgrpEmis(key)

        if src.subgrp_emis!={}:
            hasEmis=True


        toBeRemoved=[]
        for actInd,emis in src.activity_emis.iteritems():
            ac= emis["ACTCODE"][int(options.codeIndex)-1]
            ac=[c for c in ac if c!="-"]
            emis["ACTCODE"][int(options.codeIndex)-1]=ac #Remove '-'from code
            ac=".".join(ac)
            
            if ac!=options.codeValue:
                toBeRemoved.append(actInd)

        for key in toBeRemoved:
            src.removeActivityEmis(key)

        if src.activity_emis!={}:
            hasEmis=True

        readSources+=1
        if hasEmis:
            copiedSources+=1
            tgt_sourcedb.sources.append(src)

    if options.outfile is None:
        log.info("Writing sources to target edb")
    else:
        log.info("Writing sources to file")

    tgt_sourcedb.write(filename=options.outfile,force=True)

    log.info("Successfully Copied %i out of %i sources" %(copiedSources,readSources))
Example #4
0
    def read_rsrc(self):
        rsrc_path = path.join(path.dirname(self.get_db_file()), 'edb.rsrc')
        rsrc = Rsrc(rsrc_path)

        self.gc = rsrc.gc