示例#1
0
    def processLDS(self):
        '''Process with LDS as a source and the destination supplied as an argument'''

        #fname = dst.DRIVER_NAME.lower()+self.LP_SUFFIX
        
        self.dst.applyConfigOptions()

        self.dst.setSRS(self.epsg)
        #might as well initds here, its going to be needed eventually
        if not self.dst.getDS():
            ldslog.info('Initialising absent DST.DS. This is not recommended in GUI/threaded mode')
            self.dst.setDS(self.dst.initDS(self.dst.destinationURI(None)))#DataStore.LDS_CONFIG_TABLE))

        self.dst.versionCheck()
        (self.sixtyfourlayers,self.partitionlayers,self.partitionsize,self.prefetchsize) = self.dst.confwrap.readDSParameters('Misc',{'idp':self.src.idp})

        if not self.dst.getLayerConf():
            self.dst.setLayerConf(TransferProcessor.getNewLayerConf(self.dst))        

        #still used on command line
        if self.getInitConfig():
            TransferProcessor.initialiseLayerConfig(self.src,self.dst)
        if self.dst.getLayerConf() is None:
            raise LayerConfigurationException("Cannot initialise Layer-Configuration file/table, "+str(self.dst.getConfInternal()))

        #------------------------------------------------------------------------------------------
        #Valid layers are those that exist in LDS and are also configured in the LC
        self.readCapsDoc(self.src)
        lds_valid = [i[0] for i in self.assembleLayerList(intersect=True)]
        #if layer provided, check that layer is in valid list
        #else if group then intersect valid and group members
        lgid = self.idLayerOrGroup(self.lgval)
        if lgid == LORG.GROUP:
            self.lnl = set()
            group = set(self.lgval.split(','))
            for lid in lds_valid:
                cats = self.dst.getLayerConf().readLayerProperty(lid,'category')
                #if cats and set([f.encode('utf8').strip() for f in cats.split(',')]).intersection(group):
                if cats and set([LU.recode(f) for f in cats.split(',')]).intersection(group):
                    self.lnl.update((lid,))
                
            if not len(self.lnl):
                ldslog.warn('Possible mis-identified Group, {}'.format(group))
                lgid = LORG.LAYER
                
        if lgid == LORG.LAYER:
            layer = LU.checkLayerName(self.dst.getLayerConf(),self.lgval) 
            if layer in lds_valid:
                self.lnl = (layer,)
            else:
                raise InputMisconfigurationException('Layer '+str(layer)+' invalid')
            
        #override config file dates with command line dates if provided
        ldslog.debug("SelectedLayers={}".format(len(self.lnl)))
        #ldslog.debug("Layer List:"+str(self.lnl))
        
        #------------------------------------------------------------------------------------------  
        
        #Before we go any further, if this is a cleaning job, no point doing anymore setup. Start deleting
        if self.getCleanConfig():
            for cleanlayer in self.lnl:
                self.cleanLayer(cleanlayer,truncate=False)
            return

        #build a list of layers with corresponding lastmodified/incremental flags
        fd = LU.checkDateFormat(self.fromdate)#if date format wrong treated as None
        td = LU.checkDateFormat(self.todate)
        self.layer_total = len(self.lnl)
        self.layer_count = 0
        for each_layer in self.lnl:
            ldslog.debug('BENCHMARK '+each_layer)
            lm = LU.checkDateFormat(self.dst.getLastModified(each_layer))
            srs = self.dst.getEPSGConversion(each_layer)
            pk = self.hasPrimaryKey(each_layer)
            filt = self.dst.getLayerConf().readLayerProperty(each_layer,'cql')
            #Set (cql) filters in URI call using layer picking the one with highest precedence            
            self.src.setFilter(LU.precedence(self.cql,self.dst.getFilter(),filt))
        
            #SRS are set in the DST since the conversion takes place during the write process. Needed here to trigger bypass to featureCopy 
            #print 'tp.epsg=',self.epsg,'srs=',srs,'!getsrs=',self.dst.getSRS()
            self.dst.setSRS(LU.precedence(self.epsg,srs,None))

            #Destination URI won't change because of incremental so set it here
            self.dst.setURI(self.dst.destinationURI(each_layer))
            #RB dst (not implemented)
            #self.dst.setURI(self.dst.requestbuilder.destinationURI(each_layer))
                
            #if PK is none do paging since page index uses pk (can't lookup matching FIDs for updates/deletes?)
            if pk:
                gdal.SetConfigOption('OGR_WFS_PAGING_ALLOWED','ON')
            else:
                gdal.SetConfigOption('OGR_WFS_PAGING_ALLOWED','OFF')
                
            #check dates -> check incr read -> incr or non

            nonincr = False          
            if any(i for i in [lm, fd, td]) and pk:
                ldslog.debug('lm={}, fd={}, td={}'.format(lm,fd,td))
                final_fd = (DataStore.EARLIEST_INIT_DATE if lm is None else lm) if fd is None else fd
                final_td = self.dst.getCurrent() if td is None else td
          
                if (datetime.strptime(final_td,'%Y-%m-%dT%H:%M:%S')-datetime.strptime(final_fd,'%Y-%m-%dT%H:%M:%S')).days>0:
                    #self.src.setURI(self.src.sourceURIIncremental(each_layer,final_fd,final_td))
                    #RB srci
                    self.src.setURI(self.src.requestbuilder.sourceURIIncremental(each_layer,final_fd,final_td))
                    if self.readLayer():
                        self.dst.setIncremental()    
                        self.dst.setPrefetchSize(self.prefetchsize)
                        ldslog.info('Layer='+str(each_layer)+' lastmodified='+str(final_td))
                        ldslog.info('Layer='+str(each_layer)+' epsg='+str(self.dst.getSRS()))
                        self.dst.write(self.src, self.dst.getURI(), each_layer, self.getSixtyFour(each_layer))
                        #----------------------------------------------------------
                        #self.dst.getLayerConf().writeLayerProperty(each_layer,'lastmodified',final_td)
                        #self.dst.getLayerConf().writeLayerProperty(each_layer,'epsg',self.dst.getSRS())
                        self.dst.setLastModified(each_layer,final_td)
                        self.dst.saveEPSGConversion(each_layer,self.dst.getSRS())
                    else:
                        ldslog.warn('Incremental Read failed. Switching to Non-Incremental')
                        nonincr = True
                else:
                    ldslog.warning("No update required for layer "+each_layer+" since [start:"+final_fd+" >= finish:"+final_td+"] by at least 1 day")
                    continue
            else:
                nonincr = True
            #--------------------------------------------------    
            if nonincr:                
                #self.src.setURI(self.src.sourceURI(each_layer))
                #RB src
                self.src.setURI(self.src.requestbuilder.sourceURI(each_layer))
                if self.readLayer():
                    self.dst.clearIncremental()
                    self.cleanLayer(each_layer,truncate=True)
                    ldslog.info('Cleaning Layer={} epsg={}'.format(each_layer,self.dst.getSRS()))
                    self.dst.write(self.src, self.dst.getURI(), each_layer, self.getSixtyFour(each_layer))
                    #since no date provided defaults to current 
                    #self.dst.getLayerConf().writeLayerProperty(each_layer,'epsg',self.dst.getSRS())
                    self.dst.setLastModified(each_layer)
                    self.dst.saveEPSGConversion(each_layer,self.dst.getSRS())
                else:
                    ldslog.warn('Non-Incremental Read failed')
                    raise DatasourceInitialisationException('Unable to read from data source with URI '+self.src.getURI())
                
            self.layer_count += 1
            self.dst.src_feat_count = 0