def buildIndex(self): '''Builds an index creation string for a new full replicate in PG format''' tableonly = self.dst_info.ascii_name.split('.')[-1] ALLOW_CONSTRAINT_CREATION=False #SpatiaLite doesnt have a unique constraint but since we're using a pk might a well declare it as such if ALLOW_CONSTRAINT_CREATION and LDSUtilities.assessNone(self.dst_info.pkey): #spatialite won't do post create constraint additions (could to a re-create?) cmd = 'ALTER TABLE {0} ADD PRIMARY KEY {1}_{2}_PK ({2})'.format(self.dst_info.ascii_name,tableonly,self.dst_info.pkey) try: self.executeSQL(cmd) ldslog.info("Index = {}({}). Execute = {}".format(tableonly,self.dst_info.pkey,cmd)) except RuntimeError as rte: if re.search('already exists', str(rte)): ldslog.warn(rte) else: raise #Unless we select SPATIAL_INDEX=no as a Layer option this should never be needed #because gcol is also used to determine whether a layer is spatial still do this check if LDSUtilities.assessNone(self.dst_info.geocolumn) and 'SPATIAL_INDEX=NO' in [opt.replace(' ','').upper() for opt in self.sl_local_opts]: cmd = "SELECT CreateSpatialIndex('{}','{}')".format(self.dst_info.ascii_name,self.DEFAULT_GCOL) try: self.executeSQL(cmd) ldslog.info("Index = {}({}). Execute = {}. NB Cannot override Geo-Column Name.".format(tableonly,self.DEFAULT_GCOL,cmd)) except RuntimeError as rte: if re.search('already exists', str(rte)): ldslog.warn(rte) else: raise
def getLayerNames(self,refresh=False): '''Returns configured layers for respective layer properties file''' #gdal.SetConfigOption('CPL_DEBUG','ON') #gdal.SetConfigOption('CPL_LOG_ERRORS','ON') if refresh or not self.namelist: self.namelist = [] layer = self.lcfname.ds.GetLayer(self.lcfname.LDS_CONFIG_TABLE)###fname -> lcfname if layer: layer.SetIgnoredFields(('OGR_GEOMETRY',)) layer.ResetReading() #HACK Win7 layer.GetFeatureCount() feat = layer.GetNextFeature() while feat: try: lcid = feat.GetField('id') lcname = feat.GetField('name') lccats = [LU.recode(f) if f else None for f in feat.GetField('category').split(',')] self.namelist += ((LU.recode(lcid) if lcid else None, LU.recode(lcname) if lcname else None,lccats),) #self.namelist += ((feat.GetField('id').encode('utf8'),feat.GetField('name').encode('utf8'),[f.encode('utf8').strip() for f in feat.GetField('category').split(',')]),) feat = layer.GetNextFeature() except UnicodeEncodeError as uee: raise except UnicodeDecodeError as ude: raise else: ldslog.error('REMINDER! TRIGGER CONF BUILD') #print '>>>>> NAMELIST',self.namelist return self.namelist
def closeEvent(self, event): '''Intercept close event to signal parent to update status''' self.parent.controls.setStatus(self.parent.controls.STATUS.IDLE, 'Done') #return last group selection #lastgroup = LU.recode(self.page.keywordcombo.lineEdit().text().toUtf8().data()) lastgroup = LU.recode( LQ.readWidgetText(self.page.keywordcombo.lineEdit().text())) #self.parent.controls.gpr.writeline('lgvalue',lastgroup) if LU.assessNone(lastgroup): dep = self.parent.confconn.reg.openEndPoint( self.parent.confconn.destname, self.parent.confconn.uconf) #sep = self.parent.confconn.reg.openEndPoint('WFS',self.parent.confconn.uconf) self.parent.confconn.reg.setupLayerConfig(self.parent.confconn.tp, None, dep) self.parent.confconn.setupCompleteLayerList(dep) self.parent.confconn.setupAssignedLayerList() self.parent.confconn.buildLayerGroupList() lgindex = self.parent.confconn.getLayerGroupIndex(lastgroup, col=1) self.parent.controls.refreshLGCombo() #current index wont be available in parent if this is the init run self.parent.controls.lgcombo.setCurrentIndex( lgindex if lgindex else 0) #self.parent.confconn.reg.closeEndPoint('WFS') self.parent.confconn.reg.closeEndPoint( self.parent.confconn.destname) sep, dep = None, None
def buildIndex(self): '''Builds an index creation string for a new full replicate in PG format''' tableonly = self.dst_info.ascii_name.split('.')[-1] ALLOW_TABLE_INDEX_CREATION = True #SpatiaLite doesnt have a unique constraint but since we're using a pk might a well declare it as such if ALLOW_TABLE_INDEX_CREATION and LDSUtilities.assessNone( self.dst_info.pkey): #spatialite won't do post create constraint additions (could to a re-create?) cmd = 'CREATE INDEX {0}_{1}_PK ON {0}({1})'.format( tableonly, self.dst_info.pkey) try: self.executeSQL(cmd) ldslog.info("Index = {}({}). Execute = {}".format( tableonly, self.dst_info.pkey, cmd)) except RuntimeError as rte: if re.search('already exists', str(rte)): ldslog.warn(rte) else: raise #Unless we select SPATIAL_INDEX=no as a Layer option this should never be needed #because gcol is also used to determine whether a layer is spatial still do this check if LDSUtilities.assessNone(self.dst_info.geocolumn): #untested and unlikely to work cmd = "CREATE INDEX {0}_{1}_SK ON {0}({1})".format( self.dst_info.ascii_name, self.dst_info.geocolumn) try: self.executeSQL(cmd) ldslog.info("Index = {}({}). Execute = {}.".format( tableonly, self.dst_info.geocolumn, cmd)) except RuntimeError as rte: if re.search('already exists', str(rte)): ldslog.warn(rte) else: raise
def buildIndex(self): '''Builds an index creation string for a new full replicate in PG format''' tableonly = self.dst_info.ascii_name.split('.')[-1] if LDSUtilities.assessNone(self.dst_info.pkey): cmd = 'ALTER TABLE {0} ADD CONSTRAINT {1}_{2}_PK UNIQUE({2})'.format(self.dst_info.ascii_name,tableonly,self.dst_info.pkey) try: self.executeSQL(cmd) ldslog.info("Index = {}({}). Execute = {}".format(tableonly,self.dst_info.pkey,cmd)) except RuntimeError as rte: if re.search('already exists', str(rte)): ldslog.warn(rte) else: raise #If a spatial index has already been created don't try to create another one if self.SPATIAL_INDEX == 'OFF' and LDSUtilities.assessNone(self.dst_info.geocolumn): cmd = 'CREATE INDEX {1}_{2}_GK ON {0} USING GIST({2})'.format(self.dst_info.ascii_name,tableonly,self.dst_info.geocolumn) try: self.executeSQL(cmd) ldslog.info("Index = {}({}). Execute = {}".format(tableonly,self.dst_info.geocolumn,cmd)) except RuntimeError as rte: if re.search('already exists', str(rte)): ldslog.warn(rte) else: raise
def getLayerNames(self, refresh=False): '''Returns configured layers for respective layer properties file''' #gdal.SetConfigOption('CPL_DEBUG','ON') #gdal.SetConfigOption('CPL_LOG_ERRORS','ON') if refresh or not self.namelist: self.namelist = [] layer = self.lcfname.ds.GetLayer( self.lcfname.LDS_CONFIG_TABLE) ###fname -> lcfname if layer: layer.SetIgnoredFields(('OGR_GEOMETRY', )) layer.ResetReading() #HACK Win7 layer.GetFeatureCount() feat = layer.GetNextFeature() while feat: try: lcid = feat.GetField('id') lcname = feat.GetField('name') lccats = [ LU.recode(f) if f else None for f in feat.GetField('category').split(',') ] self.namelist += ((LU.recode(lcid) if lcid else None, LU.recode(lcname) if lcname else None, lccats), ) #self.namelist += ((feat.GetField('id').encode('utf8'),feat.GetField('name').encode('utf8'),[f.encode('utf8').strip() for f in feat.GetField('category').split(',')]),) feat = layer.GetNextFeature() except UnicodeEncodeError as uee: raise except UnicodeDecodeError as ude: raise else: ldslog.error('REMINDER! TRIGGER CONF BUILD') #print '>>>>> NAMELIST',self.namelist return self.namelist
def buildIndex(self): '''Builds an index creation string for a new full replicate in PG format''' tableonly = self.dst_info.ascii_name.split('.')[-1] ALLOW_TABLE_INDEX_CREATION=True #SpatiaLite doesnt have a unique constraint but since we're using a pk might a well declare it as such if ALLOW_TABLE_INDEX_CREATION and LDSUtilities.assessNone(self.dst_info.pkey): #spatialite won't do post create constraint additions (could to a re-create?) cmd = 'CREATE INDEX {0}_{1}_PK ON {0}({1})'.format(tableonly,self.dst_info.pkey) try: self.executeSQL(cmd) ldslog.info("Index = {}({}). Execute = {}".format(tableonly,self.dst_info.pkey,cmd)) except RuntimeError as rte: if re.search('already exists', str(rte)): ldslog.warn(rte) else: raise #Unless we select SPATIAL_INDEX=no as a Layer option this should never be needed #because gcol is also used to determine whether a layer is spatial still do this check if LDSUtilities.assessNone(self.dst_info.geocolumn): #untested and unlikely to work cmd = "CREATE INDEX {0}_{1}_SK ON {0}({1})".format(self.dst_info.ascii_name,self.dst_info.geocolumn) try: self.executeSQL(cmd) ldslog.info("Index = {}({}). Execute = {}.".format(tableonly,self.dst_info.geocolumn,cmd)) except RuntimeError as rte: if re.search('already exists', str(rte)): ldslog.warn(rte) else: raise
def buildIndex(self): '''Builds an index creation string for a new full replicate''' tableonly = self.dst_info.ascii_name.split('.')[-1] if LU.assessNone(self.dst_info.pkey): cmd = 'ALTER TABLE {0} ADD CONSTRAINT {1}_{2}_PK UNIQUE({2})'.format(self.dst_info.ascii_name,tableonly,self.dst_info.pkey) try: self.executeSQL(cmd) ldslog.info("Index = {}({}). Execute = {}".format(tableonly,self.dst_info.pkey,cmd)) except RuntimeError as rte: if re.search('already exists', str(rte)): ldslog.warn(rte) else: raise if LU.assessNone(self.dst_info.geocolumn): cmd = 'CREATE SPATIAL INDEX {1}_{2}_GK ON {0}({2})'.format(self.dst_info.ascii_name,tableonly,self.dst_info.geocolumn) cmd += ' WITH (BOUNDING_BOX = (XMIN = {0},YMIN = {1},XMAX = {2},YMAX = {3}))'.format(self.BBOX['XMIN'],self.BBOX['YMIN'],self.BBOX['XMAX'],self.BBOX['YMAX']) #cmd = 'CREATE SPATIAL INDEX ON {}'.format(tableonly) try: self.executeSQL(cmd) ldslog.info("Index = {}({}). Execute = {}".format(tableonly,self.dst_info.geocolumn,cmd)) except RuntimeError as rte: if re.search('already exists', str(rte)): ldslog.warn(rte) else: raise
def buildLayerGroupList(self,groups=None,layers=None): '''Sets the array storing values displayed in the Layer/Group combo; format is ((lyr/grp, name, displaystring),)''' from lds.TransferProcessor import LORG #self.lgcombo.addItems(('',TransferProcessor.LG_PREFIX['g'])) self.lglist = [] # lorg,value,display for g in sorted(groups if groups else self.assigned): self.lglist += ((LORG.GROUP,LU.recode(g),u'{} (group)'.format(LU.recode(g))),) for l in sorted(layers if layers else self.vlayers): #if re.search('Electoral',l[0]): if not isinstance(l,(tuple,list)): raise LayerConfigInitialisationException('Layer init value error. {} is not a list/tuple'.format(l)) self.lglist += ((LORG.LAYER,l[0],u'{} ({})'.format(l[1],l[0])),)
def runReplicationScript(self,clean=False): '''Run the layer/group repliction script''' destination,lgval,uconf,epsg,fe,te,fd,td = self.readParameters() uconf_path = LU.standardiseUserConfigName(uconf) destination_path = LU.standardiseLayerConfigName(destination) destination_driver = LU.standardiseDriverNames(destination) if not os.path.exists(uconf_path): self.userConfMessage(uconf_path) return elif not MainFileReader(uconf_path).hasSection(destination_driver): self.userConfMessage(uconf_path,destination_driver) return #----------------------------------------------------- #'destname','layer','uconf','group','epsg','fd','td','int' self.parent.gpr.write((destination_driver,lgval,uconf,epsg,fd,td)) ldslog.info(u'dest={0}, lg={1}, conf={2}, epsg={3}'.format(destination_driver,lgval,uconf,epsg)) ldslog.info('fd={0}, td={1}, fe={2}, te={3}'.format(fd,td,fe,te)) lgindex = self.parent.confconn.getLayerGroupIndex(lgval,col=1) #lorg = self.parent.confconn.lglist[lgindex][0] #----------don't need lorg in TP anymore but it is useful for sorting/counting groups #self.parent.confconn.tp.setLayerOrGroup(lorg) self.parent.confconn.tp.setLayerGroupValue(lgval) if self.fromdateenable.isChecked(): self.parent.confconn.tp.setFromDate(fd) if self.todateenable.isChecked(): self.parent.confconn.tp.setToDate(td) self.parent.confconn.tp.setUserConf(uconf) if self.epsgenable: self.parent.confconn.tp.setEPSG(epsg) #because clean state persists in TP if clean: self.parent.confconn.tp.setCleanConfig() else: self.parent.confconn.tp.clearCleanConfig() #(re)initialise the data source since uconf may have changed #>>#self.parent.confconn.tp.src = self.parent.confconn.initSourceWrapper() #-------------------------- ###ep = self.parent.confconn.reg.openEndPoint(self.parent.confconn.destname,self.parent.confconn.uconf) ###self.parent.confconn.reg.closeEndPoint(self.parent.confconn.destname) ###ep = None #Open ProcessRunner and run with TP(proc)/self(gui) instances #HACK temp add of dest_drv to PR call try: #TODO. Test for valid LC first self.tpr = ProcessRunner(self,destination_driver) except Exception as e: ldslog.error('Cannot create ProcessRunner {}. NB Possible missing Layer Config {}'.format(str(e),destination_path)) self.layerConfMessage(destination_path) return #If PR has been successfully created we must vave a valid dst ldslog.debug('TRPstart') self.tpr.start()
def readLayerProperty(self, layer, key): try: if isinstance(layer, tuple) or isinstance(layer, list): value = () for l in layer: value += (LU.assessNone(self.cp.get(l, key)), ) else: value = LU.assessNone(self.cp.get(layer, key)) except: '''return a default value otherwise none which would also be a default for some keys''' #the logic here may be a bit suss, if the property is blank return none but if there is an error assume a default is needed? return {'pkey': 'ID', 'name': layer, 'geocolumn': 'SHAPE'}.get(key) return value
def readLayerProperty(self,layer,key): try: if isinstance(layer,tuple) or isinstance(layer,list): value = () for l in layer: value += (LU.assessNone(self.cp.get(l, key)),) else: value = LU.assessNone(self.cp.get(layer, key)) except: '''return a default value otherwise none which would also be a default for some keys''' #the logic here may be a bit suss, if the property is blank return none but if there is an error assume a default is needed? return {'pkey':'ID','name':layer,'geocolumn':'SHAPE'}.get(key) return value
def getLayerGroupIndex(self,dispval,col=2): '''Finds a matching group/layer entry from its displayed name''' #0=lorg,1=value,2=display compval = LU.recode(dispval) if not LU.assessNone(compval): ldslog.warn('No attempt made to find index for empty group/layer request, "{}"'.format(compval)) return None# or 0? try: #print 'lgl',[type(i[1]) for i in self.lglist],'\ncv',type(compval) index = [i[col] for i in self.lglist].index(compval) except ValueError as ve: ldslog.warn(u'Cannot find an index in column {} for the requested group/layer, "{}", from {} layers. Returning None index'.format(col,compval,len(self.lglist))) index = None return index
def buildLayerGroupList(self, groups=None, layers=None): '''Sets the array storing values displayed in the Layer/Group combo; format is ((lyr/grp, name, displaystring),)''' from lds.TransferProcessor import LORG #self.lgcombo.addItems(('',TransferProcessor.LG_PREFIX['g'])) self.lglist = [] # lorg,value,display for g in sorted(groups if groups else self.assigned): self.lglist += ((LORG.GROUP, LU.recode(g), u'{} (group)'.format(LU.recode(g))), ) for l in sorted(layers if layers else self.vlayers): #if re.search('Electoral',l[0]): if not isinstance(l, (tuple, list)): raise LayerConfigInitialisationException( 'Layer init value error. {} is not a list/tuple'.format(l)) self.lglist += ((LORG.LAYER, l[0], u'{} ({})'.format(l[1], l[0])), )
def fetchLayerInfo(cls,url,ver=None,proxy=None): '''Non-GDAL static method for fetching LDS layer ID's using etree parser.''' res = [] content = None wfs_ns = cls.NS['wfs20'] if re.match('^2',ver) else cls.NS['wfs11'] ftxp = "//{0}FeatureType".format(wfs_ns) nmxp = "./{0}Name".format(wfs_ns) ttxp = "./{0}Title".format(wfs_ns) kyxp = "./{0}Keywords/{0}Keyword".format(cls.NS['ows']) try: if not LDSUtilities.assessNone(proxy): install_opener(build_opener(ProxyHandler(proxy))) #content = urlopen(url)#bug in lxml doesnt close url/files using parse method with closing(urlopen(url)) as content: tree = etree.parse(content) for ft in tree.findall(ftxp): name = ft.find(nmxp).text#.encode('utf8') title = ft.find(ttxp).text#.encode('utf8') #keys = [x.text.encode('utf8') for x in ft.findall(kyxp)] keys = [x.text for x in ft.findall(kyxp)] res += ((name,title,keys),) except XMLSyntaxError as xe: ldslog.error('Error parsing URL;'+str(url)+' ERR;'+str(xe)) return res
def fetchLayerInfo(cls, url, ver=None, proxy=None): '''Non-GDAL static method for fetching LDS layer ID's using etree parser.''' res = [] content = None wfs_ns = cls.NS['wfs20'] if re.match('^2', ver) else cls.NS['wfs11'] ftxp = "//{0}FeatureType".format(wfs_ns) nmxp = "./{0}Name".format(wfs_ns) ttxp = "./{0}Title".format(wfs_ns) kyxp = "./{0}Keywords/{0}Keyword".format(cls.NS['ows']) try: if not LDSUtilities.assessNone(proxy): install_opener(build_opener(ProxyHandler(proxy))) #content = urlopen(url)#bug in lxml doesnt close url/files using parse method with closing(urlopen(url)) as content: tree = etree.parse(content) for ft in tree.findall(ftxp): name = ft.find(nmxp).text #.encode('utf8') title = ft.find(ttxp).text #.encode('utf8') #keys = [x.text.encode('utf8') for x in ft.findall(kyxp)] keys = [x.text for x in ft.findall(kyxp)] res += ((name, title, keys), ) except XMLSyntaxError as xe: ldslog.error('Error parsing URL;' + str(url) + ' ERR;' + str(xe)) return res
def doRejectClickAction(self): '''Takes available selected and moves to selection''' #ktext = LU.recode(self.keywordcombo.lineEdit().text().toUtf8().data()) ktext = LU.recode(LQ.readWidgetText(self.keywordcombo.lineEdit().text())) if not self.checkKeyword(ktext): return #------------------------------ select = self.selection.selectionModel() if select.hasSelection(): tlist = self.transferSelectedRows(select.selectedRows(),self.selection_sfpm,self.available_sfpm) #------------------------------ kindex = self.keywordcombo.findText(ktext) remainder = self.parent.deleteKeysFromLayerConfig([ll[1][0] for ll in tlist],ktext) if remainder > 0 and kindex == -1: #items+newkey -> add self.parent.writeKeysToLayerConfig(ktext) self.keywordcombo.addItem(ktext) elif remainder == 0 and kindex > -1: #empty+oldkey -> del self.keywordcombo.removeItem(kindex) self.keywordcombo.clearEditText() else: ldslog.warn('R2L < Transfer action without selection') #TRACE# #pdb.set_trace() self.selection.clearSelection()
def deleteKeysFromLayerConfig(self, layerlist, ckey): '''Remove custom keys from selected layers''' replacementlist = () dep = self.parent.confconn.reg.openEndPoint( self.parent.confconn.destname, self.parent.confconn.uconf) #print 'opening',dep#DEBUG self.parent.confconn.reg.setupLayerConfig(self.parent.confconn.tp, None, dep, initlc=False) categorylist = [ f.encode('utf8').strip() for f in dep.getLayerConf().readLayerProperty( layerlist, 'category') if f ] for cat in categorylist: #replacementlist += (re.sub(',+',',',''.join(cat.split(ckey)).strip(',')),) try: cat = cat.split(',') cat.remove(LU.recode(ckey, uflag='encode')) except ValueError: pass replacementlist += (','.join(cat), ) dep.getLayerConf().writeLayerProperty(layerlist, 'category', replacementlist) #----------------------------------- self.parent.confconn.setupCompleteLayerList(dep) self.parent.confconn.setupAssignedLayerList() self.parent.confconn.buildLayerGroupList() #self.refreshLayers(dep,customkey) self.parent.confconn.reg.closeEndPoint(self.parent.confconn.destname) #print 'closing', dep,self.parent.confconn.reg#DEBUG dep = None return self.selection_model.rowCount()
def testURL(self,url): '''Connect to a URL using the configured proxy (using urlopen method)''' return LDSUtilities.timedProcessRunner(LDSUtilities.readLDS, (url,self.pxy), None) #return LDSUtilities.readDocument(url, self.pxy)
def writeKeysToLayerConfig(self, ckey): '''Add custom key to the selection_model list of layers (assumes required av->sl transfer completed) not just the transferring entry''' layerlist = [ll[0] for ll in self.selection_model.mdata] replacementlist = () dep = self.parent.confconn.reg.openEndPoint( self.parent.confconn.destname, self.parent.confconn.uconf) #print 'opened dep=',dep,'reg=',self.parent.confconn.reg#DEBUG self.parent.confconn.reg.setupLayerConfig(self.parent.confconn.tp, None, dep, initlc=False) #categorylist = [f.encode('utf8').strip() for f in dep.getLayerConf().readLayerProperty(layerlist, 'category') if f] categorylist = [ LU.assessNone(f) for f in dep.getLayerConf().readLayerProperty( layerlist, 'category') ] for cat in categorylist: replacementlist += ((cat if ckey in cat.split(',') else cat + ',' + ckey) if cat else ckey, ) #print '>>> writing this replacementlist to LC',replacementlist dep.getLayerConf().writeLayerProperty(layerlist, 'category', replacementlist) #new keyword written so re-read complete (LC) and update assigned keys list self.parent.confconn.setupCompleteLayerList(dep, refresh=True) self.parent.confconn.setupAssignedLayerList() self.parent.confconn.buildLayerGroupList() #self.refreshLayers(dep,customkey) #print 'closing dep=',dep,'reg=',self.parent.confconn.reg#DEBUG self.parent.confconn.reg.closeEndPoint(self.parent.confconn.destname) dep = None
def doRejectClickAction(self): '''Takes available selected and moves to selection''' #ktext = LU.recode(self.keywordcombo.lineEdit().text().toUtf8().data()) ktext = LU.recode( LQ.readWidgetText(self.keywordcombo.lineEdit().text())) if not self.checkKeyword(ktext): return #------------------------------ select = self.selection.selectionModel() if select.hasSelection(): tlist = self.transferSelectedRows(select.selectedRows(), self.selection_sfpm, self.available_sfpm) #------------------------------ kindex = self.keywordcombo.findText(ktext) remainder = self.parent.deleteKeysFromLayerConfig( [ll[1][0] for ll in tlist], ktext) if remainder > 0 and kindex == -1: #items+newkey -> add self.parent.writeKeysToLayerConfig(ktext) self.keywordcombo.addItem(ktext) elif remainder == 0 and kindex > -1: #empty+oldkey -> del self.keywordcombo.removeItem(kindex) self.keywordcombo.clearEditText() else: ldslog.warn('R2L < Transfer action without selection') #TRACE# #pdb.set_trace() self.selection.clearSelection()
def deleteKeysFromLayerConfig(self,layerlist,ckey): '''Remove custom keys from selected layers''' replacementlist = () dep = self.parent.confconn.reg.openEndPoint(self.parent.confconn.destname,self.parent.confconn.uconf) #print 'opening',dep#DEBUG self.parent.confconn.reg.setupLayerConfig(self.parent.confconn.tp,None,dep,initlc=False) categorylist = [f.encode('utf8').strip() for f in dep.getLayerConf().readLayerProperty(layerlist, 'category') if f] for cat in categorylist: #replacementlist += (re.sub(',+',',',''.join(cat.split(ckey)).strip(',')),) try: cat = cat.split(',') cat.remove(LU.recode(ckey,uflag='encode')) except ValueError: pass replacementlist += (','.join(cat),) dep.getLayerConf().writeLayerProperty(layerlist, 'category', replacementlist) #----------------------------------- self.parent.confconn.setupCompleteLayerList(dep) self.parent.confconn.setupAssignedLayerList() self.parent.confconn.buildLayerGroupList() #self.refreshLayers(dep,customkey) self.parent.confconn.reg.closeEndPoint(self.parent.confconn.destname) #print 'closing', dep,self.parent.confconn.reg#DEBUG dep = None return self.selection_model.rowCount()
def _deregister(self,name): fn = LU.standardiseDriverNames(name) #sync/rel the DS #self.register[fn]['ep'].closeDS()#DS should already have closed self.register[fn]['ep'] = None self.register[fn]['type'] = None self.register[fn]['uri'] = None del self.register[fn]
def setupMainAndUserConfig(self, inituserconfig): '''Sets up a reader to the main configuration file or alternatively, a user specified config file. Userconfig is not mean't to replace mainconfig, just overwrite the parts the user has decided to customise''' #self.userconfig = None self.userconfig = MainFileReader( LDSUtilities.standardiseUserConfigName(inituserconfig), False) if inituserconfig else None self.mainconfig = MainFileReader()
def writesecline(self,section,field,value): try: self.cp.set(section,field,value if LU.assessNone(value) else '') with codecs.open(self.fn, 'w','utf-8') as configfile: self.cp.write(configfile) ldslog.debug(str(section)+':'+str(field)+'='+str(value)) except Exception as e: ldslog.warn('Problem writing GUI prefs. {} - sfv={}'.format(e,(section,field,value)))
def _deregister(self, name): fn = LU.standardiseDriverNames(name) #sync/rel the DS #self.register[fn]['ep'].closeDS()#DS should already have closed self.register[fn]['ep'] = None self.register[fn]['type'] = None self.register[fn]['uri'] = None del self.register[fn]
def writesecline(self, section, field, value): try: self.cp.set(section, field, value if LU.assessNone(value) else '') with codecs.open(self.fn, 'w', 'utf-8') as configfile: self.cp.write(configfile) ldslog.debug(str(section) + ':' + str(field) + '=' + str(value)) except Exception as e: ldslog.warn('Problem writing GUI prefs. {} - sfv={}'.format( e, (section, field, value)))
def checkKeyword(self,ktext): '''Checks keyword isn't null and isn't part of the LDS supplied keywords''' if LU.assessNone(ktext) is None: QMessageBox.about(self, "Keyword Required","Please enter a Keyword to assign Layer(s) to") return False if ktext in self.confconn_link.reserved: QMessageBox.about(self, "Reserved Keyword","'{}' is a reserved keyword, please select again".format(ktext)) return False return True
def readParameters(self): '''Read values out of dialogs''' destination = LU.assessNone(str(self.destlist[self.destcombo.currentIndex()])) #lgindex = self.parent.confconn.getLayerGroupIndex(self.lgcombo.currentText().toUtf8().data()) lgindex = self.parent.confconn.getLayerGroupIndex(LQ.readWidgetText(self.lgcombo.currentText())) #NB need to test for None explicitly since zero is a valid index lgval = self.parent.confconn.lglist[lgindex][1] if LU.assessNone(lgindex) else None #uconf = LU.standardiseUserConfigName(str(self.confcombo.lineEdit().text())) #uconf = str(self.confcombo.lineEdit().text()) uconf = str(self.cflist[self.confcombo.currentIndex()]) ee = self.epsgenable.isChecked() epsg = None if ee is False else re.match('^\s*(\d+).*',str(self.epsgcombo.lineEdit().text())).group(1) fe = self.fromdateenable.isChecked() te = self.todateenable.isChecked() fd = None if fe is False else str(self.fromdateedit.date().toString('yyyy-MM-dd')) td = None if te is False else str(self.todateedit.date().toString('yyyy-MM-dd')) return destination,lgval,uconf,epsg,fe,te,fd,td
def readMainProperty(self,driver,key): try: return LU.assessNone(self.cp.get(driver, key)) # if LU.assessNone(value) is None: # return None except: '''return a default value otherwise none which would also be a default for some keys''' ldslog.warn("Cannot find requested driver/key in {}; self.cp.get('{}','{}') combo".format(self.filename[self.filename.rfind('/'):],driver,key)) return None
def _initSection(self, section): checksec = LU.standardiseDriverNames(section) if checksec: self.cp.add_section(checksec) return True elif section == self.PREFS_SEC: self.cp.add_section(section) return True return False
def _initSection(self,section): checksec = LU.standardiseDriverNames(section) if checksec: self.cp.add_section(checksec) return True elif section == self.PREFS_SEC: self.cp.add_section(section) return True return False
def _reTag(self,layerlist,tagname,addtag): '''Add/Delete a keyword from all the layers in the provided list''' for layer in layerlist: keywords = set([LU.recode(f,uflag='encode') for f in self.readLayerProperty(layer, 'Category').split(',')]) if addtag: keywords.add(tagname) else: keywords.remove(tagname) self.writeLayerProperty(layer, 'Category', ','.join(keywords))
def __init__(self,lcfname): ''' Constructor ''' super(LayerFileReader,self).__init__(lcfname) self.cp = ConfigParser() self.lcfilename = LU.standardiseLayerConfigName(self.lcfname) self._readConfigFile(self.lcfilename)
def __init__(self, lcfname): ''' Constructor ''' super(LayerFileReader, self).__init__(lcfname) self.cp = ConfigParser() self.lcfilename = LU.standardiseLayerConfigName(self.lcfname) self._readConfigFile(self.lcfilename)
def readLayerProperty(self,pkey,field): '''Single property reader''' plist = () layer = self.lcfname.ds.GetLayer(self.lcfname.LDS_CONFIG_TABLE)###fname -> lcfname layer.ResetReading() #HACK Win7 layer.GetFeatureCount() if isinstance(pkey,tuple) or isinstance(pkey,list): for p in pkey: f = self.lcfname._findMatchingFeature(layer, 'id', p)###fname -> lcfname #plist += ('' if f is None else f.GetField(field).encode('utf8'),) plist += ('' if f is None else LU.recode(f.GetField(field)),) return plist else: feat = self.lcfname._findMatchingFeature(layer, 'id', pkey) if feat is None: return None prop = feat.GetField(field) return LU.recode(prop) if LU.assessNone(prop) else None#.encode('utf8')
def closeEndPoint(self,name): '''Closes the DS is a named EP or delete the EP completely if not needed''' fn = LU.standardiseDriverNames(name) #<HACK6>. Bypass DS closing for FileGDB connections if fn=='FileGDB': return #</HACK6> if self.register.has_key(fn): self._disconnect(fn) if self.register[fn]['rc'] == 0: self._deregister(fn)
def closeEndPoint(self, name): '''Closes the DS is a named EP or delete the EP completely if not needed''' fn = LU.standardiseDriverNames(name) #<HACK6>. Bypass DS closing for FileGDB connections if fn == 'FileGDB': return #</HACK6> if self.register.has_key(fn): self._disconnect(fn) if self.register[fn]['rc'] == 0: self._deregister(fn)
def getLayerGroupIndex(self, dispval, col=2): '''Finds a matching group/layer entry from its displayed name''' #0=lorg,1=value,2=display compval = LU.recode(dispval) if not LU.assessNone(compval): ldslog.warn( 'No attempt made to find index for empty group/layer request, "{}"' .format(compval)) return None # or 0? try: #print 'lgl',[type(i[1]) for i in self.lglist],'\ncv',type(compval) index = [i[col] for i in self.lglist].index(compval) except ValueError as ve: ldslog.warn( u'Cannot find an index in column {} for the requested group/layer, "{}", from {} layers. Returning None index' .format(col, compval, len(self.lglist))) index = None return index
def _commonURI(self,layer): '''Refers to common connection instance for example in a DB where it doesn't matter whether your reading or writing''' if hasattr(self,'conn_str') and self.conn_str: return self.validateConnStr(self.conn_str) #return "MSSQL:server={};database={};trusted_connection={};".format(self.server, self.dbname, self.trust) if LU.assessNone(self.pwd): if self.pwd.startswith(Encrypt.ENC_PREFIX): pwd = ";PWD={}".format(Encrypt.unSecure(self.pwd)) else: pwd = ";PWD={}".format(self.pwd) else: pwd = "" sstr = ";Schema={}".format(self.schema) if LU.assessNone(self.schema) else "" usr = "******".format(self.usr) if LU.assessNone(self.usr) else "" drv = ";Driver='{}'".format(self.odbc) if LU.assessNone(self.odbc) else "" tcn = ";trusted_connection={}".format(self.trust) if LU.assessNone(self.trust) else "" uri = "MSSQL:server={};database={}".format(self.server, self.dbname, self.odbc)+usr+pwd+drv+sstr+tcn ldslog.debug(uri) return uri
def _commonURI(self,layer): '''Refers to common connection instance for reading or writing''' if hasattr(self,'conn_str') and self.conn_str: return self.validateConnStr(self.conn_str) #can't put schema in quotes, causes error but without quotes tables get created in public anyway, still need schema.table syntax if LDSUtilities.assessNone(self.pwd): if self.pwd.startswith(Encrypt.ENC_PREFIX): pwd = " password='******'".format(Encrypt.unSecure(self.pwd)) else: pwd = " password='******'".format(self.pwd) else: pwd = "" sch = " active_schema={}".format(self.schema) if LDSUtilities.assessNone(self.schema) else "" usr = "******".format(self.usr) if LDSUtilities.assessNone(self.usr) else "" hst = " host='{}'".format(self.host) if LDSUtilities.assessNone(self.host) else "" prt = " port='{}'".format(self.port) if LDSUtilities.assessNone(self.port) else "" uri = "PG:dbname='{}'".format(self.dbname)+hst+prt+usr+pwd+sch ldslog.debug(uri) return uri
def getLayerOptions(self,layer_id): '''Get MS options for GEO_NAME''' #GEOM_TYPE, OVERWRITE, LAUNDER, PRECISION, DIM={2,3}, GEOM_NAME, SCHEMA, SRID local_opts = ['MARS Connection=TRUE'] gc = self.layerconf.readLayerProperty(layer_id,'geocolumn') if gc: local_opts += ['GEOM_NAME='+gc] schema = self.confwrap.readDSProperty(self.DRIVER_NAME,'schema') if schema is None: schema = self.schema if LU.assessNone(schema) and LU.containsOnlyAlphaNumeric(schema): local_opts += ['SCHEMA='+schema] srid = self.layerconf.readLayerProperty(layer_id,'epsg') if srid: local_opts += ['SRID='+srid] return super(MSSQLSpatialDataStore,self).getLayerOptions(layer_id) + local_opts
def readLayerParameters(self,id): '''Full Feature config reader''' from DataStore import InaccessibleFeatureException layer = self.lcfname.ds.GetLayer(self.lcfname.LDS_CONFIG_TABLE)###fname -> lcfname layer.ResetReading() #HACK Win7 layer.GetFeatureCount() feat = self.lcfname._findMatchingFeature(layer, 'id', id)###fname -> lcfname if not feat: InaccessibleFeatureException('Cannot access feature with id='+str(id)+' in layer '+str(layer.GetName())) return LU.extractFields(feat)
def readMainProperty(self, driver, key): try: return LU.assessNone(self.cp.get(driver, key)) # if LU.assessNone(value) is None: # return None except: '''return a default value otherwise none which would also be a default for some keys''' ldslog.warn( "Cannot find requested driver/key in {}; self.cp.get('{}','{}') combo" .format(self.filename[self.filename.rfind('/'):], driver, key)) return None
def __init__(self,cfpath=None,use_defaults=True): ''' Constructor ''' from lds.DataStore import DataStore self.driverconfig = {i:() for i in DataStore.DRIVER_NAMES} self.use_defaults = use_defaults #if we dont give the constructor a file path is uses the template file which will fill in any default values missed in the user conf #but if cpath is requested and doesn't exist we initialise it, otherwise you end up trying to overwrite the template if cfpath is None: self.filename = LU.standardiseUserConfigName(self.DEFAULT_MF) else: #Standardise since there is no guarantee cfpath is going to be in the reqd format to start with (since its a user entry) self.filename = LU.standardiseUserConfigName(cfpath) self.cp = None self.initMainFile() self.fn = re.search('(.+)\.conf',os.path.basename(self.filename)).group(1)
def closeEvent(self,event): '''Intercept close event to signal parent to update status''' self.parent.controls.setStatus(self.parent.controls.STATUS.IDLE,'Done') #return last group selection #lastgroup = LU.recode(self.page.keywordcombo.lineEdit().text().toUtf8().data()) lastgroup = LU.recode(LQ.readWidgetText(self.page.keywordcombo.lineEdit().text())) #self.parent.controls.gpr.writeline('lgvalue',lastgroup) if LU.assessNone(lastgroup): dep = self.parent.confconn.reg.openEndPoint(self.parent.confconn.destname,self.parent.confconn.uconf) #sep = self.parent.confconn.reg.openEndPoint('WFS',self.parent.confconn.uconf) self.parent.confconn.reg.setupLayerConfig(self.parent.confconn.tp,None,dep) self.parent.confconn.setupCompleteLayerList(dep) self.parent.confconn.setupAssignedLayerList() self.parent.confconn.buildLayerGroupList() lgindex = self.parent.confconn.getLayerGroupIndex(lastgroup,col=1) self.parent.controls.refreshLGCombo() #current index wont be available in parent if this is the init run self.parent.controls.lgcombo.setCurrentIndex(lgindex if lgindex else 0) #self.parent.confconn.reg.closeEndPoint('WFS') self.parent.confconn.reg.closeEndPoint(self.parent.confconn.destname) sep,dep = None,None
def findLayerIdByName(self,lname): '''Reverse lookup of section by associated name, finds first occurance only''' layer = self.lcfname.ds.GetLayer(self.lcfname.LDS_CONFIG_TABLE)###fname -> lcfname layer.ResetReading() #HACK Win7 layer.GetFeatureCount() feat = layer.GetNextFeature() while feat: if LU.unicodeCompare(lname,feat.GetField('name')):#.encode('utf8'): return feat.GetField('id')#.encode('utf8') feat = layer.GetNextFeature() return None
def __init__(self, cfpath=None, use_defaults=True): ''' Constructor ''' from lds.DataStore import DataStore self.driverconfig = {i: () for i in DataStore.DRIVER_NAMES} self.use_defaults = use_defaults #if we dont give the constructor a file path is uses the template file which will fill in any default values missed in the user conf #but if cpath is requested and doesn't exist we initialise it, otherwise you end up trying to overwrite the template if cfpath is None: self.filename = LU.standardiseUserConfigName(self.DEFAULT_MF) else: #Standardise since there is no guarantee cfpath is going to be in the reqd format to start with (since its a user entry) self.filename = LU.standardiseUserConfigName(cfpath) self.cp = None self.initMainFile() self.fn = re.search('(.+)\.conf', os.path.basename(self.filename)).group(1)
def checkKeyword(self, ktext): '''Checks keyword isn't null and isn't part of the LDS supplied keywords''' if LU.assessNone(ktext) is None: QMessageBox.about(self, "Keyword Required", "Please enter a Keyword to assign Layer(s) to") return False if ktext in self.confconn_link.reserved: QMessageBox.about( self, "Reserved Keyword", "'{}' is a reserved keyword, please select again".format( ktext)) return False return True
def readLayerProperty(self, pkey, field): '''Single property reader''' plist = () layer = self.lcfname.ds.GetLayer( self.lcfname.LDS_CONFIG_TABLE) ###fname -> lcfname layer.ResetReading() #HACK Win7 layer.GetFeatureCount() if isinstance(pkey, tuple) or isinstance(pkey, list): for p in pkey: f = self.lcfname._findMatchingFeature(layer, 'id', p) ###fname -> lcfname #plist += ('' if f is None else f.GetField(field).encode('utf8'),) plist += ('' if f is None else LU.recode(f.GetField(field)), ) return plist else: feat = self.lcfname._findMatchingFeature(layer, 'id', pkey) if feat is None: return None prop = feat.GetField(field) return LU.recode(prop) if LU.assessNone( prop) else None #.encode('utf8')
def _reTag(self, layerlist, tagname, addtag): '''Add/Delete a keyword from all the layers in the provided list''' for layer in layerlist: keywords = set([ LU.recode(f, uflag='encode') for f in self.readLayerProperty(layer, 'Category').split(',') ]) if addtag: keywords.add(tagname) else: keywords.remove(tagname) self.writeLayerProperty(layer, 'Category', ','.join(keywords))
def buildIndex(self): '''Builds an index creation string for a new full replicate in PG format''' tableonly = self.dst_info.ascii_name.split('.')[-1] ALLOW_CONSTRAINT_CREATION = False #SpatiaLite doesnt have a unique constraint but since we're using a pk might a well declare it as such if ALLOW_CONSTRAINT_CREATION and LDSUtilities.assessNone( self.dst_info.pkey): #spatialite won't do post create constraint additions (could to a re-create?) cmd = 'ALTER TABLE {0} ADD PRIMARY KEY {1}_{2}_PK ({2})'.format( self.dst_info.ascii_name, tableonly, self.dst_info.pkey) try: self.executeSQL(cmd) ldslog.info("Index = {}({}). Execute = {}".format( tableonly, self.dst_info.pkey, cmd)) except RuntimeError as rte: if re.search('already exists', str(rte)): ldslog.warn(rte) else: raise #Unless we select SPATIAL_INDEX=no as a Layer option this should never be needed #because gcol is also used to determine whether a layer is spatial still do this check if LDSUtilities.assessNone( self.dst_info.geocolumn) and 'SPATIAL_INDEX=NO' in [ opt.replace(' ', '').upper() for opt in self.sl_local_opts ]: cmd = "SELECT CreateSpatialIndex('{}','{}')".format( self.dst_info.ascii_name, self.DEFAULT_GCOL) try: self.executeSQL(cmd) ldslog.info( "Index = {}({}). Execute = {}. NB Cannot override Geo-Column Name." .format(tableonly, self.DEFAULT_GCOL, cmd)) except RuntimeError as rte: if re.search('already exists', str(rte)): ldslog.warn(rte) else: raise
def findLayerIdByName(self, lname): '''Reverse lookup of section by associated name, finds first occurance only''' layer = self.lcfname.ds.GetLayer( self.lcfname.LDS_CONFIG_TABLE) ###fname -> lcfname layer.ResetReading() #HACK Win7 layer.GetFeatureCount() feat = layer.GetNextFeature() while feat: if LU.unicodeCompare(lname, feat.GetField('name')): #.encode('utf8'): return feat.GetField('id') #.encode('utf8') feat = layer.GetNextFeature() return None