def processLDS(self): '''Process with LDS as a source and the destination supplied as an argument''' #fname = dst.DRIVER_NAME.lower()+self.LP_SUFFIX self.dst.applyConfigOptions() self.dst.setSRS(self.epsg) #might as well initds here, its going to be needed eventually if not self.dst.getDS(): ldslog.info('Initialising absent DST.DS. This is not recommended in GUI/threaded mode') self.dst.setDS(self.dst.initDS(self.dst.destinationURI(None)))#DataStore.LDS_CONFIG_TABLE)) self.dst.versionCheck() (self.sixtyfourlayers,self.partitionlayers,self.partitionsize,self.prefetchsize) = self.dst.confwrap.readDSParameters('Misc',{'idp':self.src.idp}) if not self.dst.getLayerConf(): self.dst.setLayerConf(TransferProcessor.getNewLayerConf(self.dst)) #still used on command line if self.getInitConfig(): TransferProcessor.initialiseLayerConfig(self.src,self.dst) if self.dst.getLayerConf() is None: raise LayerConfigurationException("Cannot initialise Layer-Configuration file/table, "+str(self.dst.getConfInternal())) #------------------------------------------------------------------------------------------ #Valid layers are those that exist in LDS and are also configured in the LC self.readCapsDoc(self.src) lds_valid = [i[0] for i in self.assembleLayerList(intersect=True)] #if layer provided, check that layer is in valid list #else if group then intersect valid and group members lgid = self.idLayerOrGroup(self.lgval) if lgid == LORG.GROUP: self.lnl = set() group = set(self.lgval.split(',')) for lid in lds_valid: cats = self.dst.getLayerConf().readLayerProperty(lid,'category') #if cats and set([f.encode('utf8').strip() for f in cats.split(',')]).intersection(group): if cats and set([LU.recode(f) for f in cats.split(',')]).intersection(group): self.lnl.update((lid,)) if not len(self.lnl): ldslog.warn('Possible mis-identified Group, {}'.format(group)) lgid = LORG.LAYER if lgid == LORG.LAYER: layer = LU.checkLayerName(self.dst.getLayerConf(),self.lgval) if layer in lds_valid: self.lnl = (layer,) else: raise InputMisconfigurationException('Layer '+str(layer)+' invalid') #override config file dates with command line dates if provided ldslog.debug("SelectedLayers={}".format(len(self.lnl))) #ldslog.debug("Layer List:"+str(self.lnl)) #------------------------------------------------------------------------------------------ #Before we go any further, if this is a cleaning job, no point doing anymore setup. Start deleting if self.getCleanConfig(): for cleanlayer in self.lnl: self.cleanLayer(cleanlayer,truncate=False) return #build a list of layers with corresponding lastmodified/incremental flags fd = LU.checkDateFormat(self.fromdate)#if date format wrong treated as None td = LU.checkDateFormat(self.todate) self.layer_total = len(self.lnl) self.layer_count = 0 for each_layer in self.lnl: ldslog.debug('BENCHMARK '+each_layer) lm = LU.checkDateFormat(self.dst.getLastModified(each_layer)) srs = self.dst.getEPSGConversion(each_layer) pk = self.hasPrimaryKey(each_layer) filt = self.dst.getLayerConf().readLayerProperty(each_layer,'cql') #Set (cql) filters in URI call using layer picking the one with highest precedence self.src.setFilter(LU.precedence(self.cql,self.dst.getFilter(),filt)) #SRS are set in the DST since the conversion takes place during the write process. Needed here to trigger bypass to featureCopy #print 'tp.epsg=',self.epsg,'srs=',srs,'!getsrs=',self.dst.getSRS() self.dst.setSRS(LU.precedence(self.epsg,srs,None)) #Destination URI won't change because of incremental so set it here self.dst.setURI(self.dst.destinationURI(each_layer)) #RB dst (not implemented) #self.dst.setURI(self.dst.requestbuilder.destinationURI(each_layer)) #if PK is none do paging since page index uses pk (can't lookup matching FIDs for updates/deletes?) if pk: gdal.SetConfigOption('OGR_WFS_PAGING_ALLOWED','ON') else: gdal.SetConfigOption('OGR_WFS_PAGING_ALLOWED','OFF') #check dates -> check incr read -> incr or non nonincr = False if any(i for i in [lm, fd, td]) and pk: ldslog.debug('lm={}, fd={}, td={}'.format(lm,fd,td)) final_fd = (DataStore.EARLIEST_INIT_DATE if lm is None else lm) if fd is None else fd final_td = self.dst.getCurrent() if td is None else td if (datetime.strptime(final_td,'%Y-%m-%dT%H:%M:%S')-datetime.strptime(final_fd,'%Y-%m-%dT%H:%M:%S')).days>0: #self.src.setURI(self.src.sourceURIIncremental(each_layer,final_fd,final_td)) #RB srci self.src.setURI(self.src.requestbuilder.sourceURIIncremental(each_layer,final_fd,final_td)) if self.readLayer(): self.dst.setIncremental() self.dst.setPrefetchSize(self.prefetchsize) ldslog.info('Layer='+str(each_layer)+' lastmodified='+str(final_td)) ldslog.info('Layer='+str(each_layer)+' epsg='+str(self.dst.getSRS())) self.dst.write(self.src, self.dst.getURI(), each_layer, self.getSixtyFour(each_layer)) #---------------------------------------------------------- #self.dst.getLayerConf().writeLayerProperty(each_layer,'lastmodified',final_td) #self.dst.getLayerConf().writeLayerProperty(each_layer,'epsg',self.dst.getSRS()) self.dst.setLastModified(each_layer,final_td) self.dst.saveEPSGConversion(each_layer,self.dst.getSRS()) else: ldslog.warn('Incremental Read failed. Switching to Non-Incremental') nonincr = True else: ldslog.warning("No update required for layer "+each_layer+" since [start:"+final_fd+" >= finish:"+final_td+"] by at least 1 day") continue else: nonincr = True #-------------------------------------------------- if nonincr: #self.src.setURI(self.src.sourceURI(each_layer)) #RB src self.src.setURI(self.src.requestbuilder.sourceURI(each_layer)) if self.readLayer(): self.dst.clearIncremental() self.cleanLayer(each_layer,truncate=True) ldslog.info('Cleaning Layer={} epsg={}'.format(each_layer,self.dst.getSRS())) self.dst.write(self.src, self.dst.getURI(), each_layer, self.getSixtyFour(each_layer)) #since no date provided defaults to current #self.dst.getLayerConf().writeLayerProperty(each_layer,'epsg',self.dst.getSRS()) self.dst.setLastModified(each_layer) self.dst.saveEPSGConversion(each_layer,self.dst.getSRS()) else: ldslog.warn('Non-Incremental Read failed') raise DatasourceInitialisationException('Unable to read from data source with URI '+self.src.getURI()) self.layer_count += 1 self.dst.src_feat_count = 0