def fetch(self): if self.isLoaded: tool.VERBOSE('use query {}', self.name) return True tool.VERBOSE('fetching query {}', self.name) with tool.INDENT(): for i in self.inputs: if isinstance(i, TableBase): if not i.fetch(): raise 'failed to fetch ' + i.name connection = self.db.engine.connect() connection.execute('create table {} as {}'.format( self.name, self.sql)) self._count = connection.execute( 'select count(*) as count from {}'.format(self.name)).scalar() connection.close() tool.VERBOSE('wrote {} row(s) to table {}', self.count, self.name) self._loaded = True return self._loaded
def _make_request( self, method, url, **kwargs ): tool.VERBOSE( method.upper() + ' ' + url ) # tool.VERBOSE( json.dumps( kwargs['headers'], sort_keys=True, indent=2, separators=(',', ': ') ) ) if 'data' in kwargs: tool.VERBOSE( json.dumps( json.loads(kwargs['data']), sort_keys=True, indent=2, separators=(',', ': ') ) ) if args.dryrun: return ( 'dryrun', '', {} ) r = getattr( requests, method.lower() )( url, **kwargs ) res = True if r.ok else ( None if r.status_code >= 500 else False ) ver = eval( r.headers.get( 'ETag', '""') ) tool.VERBOSE( 'status {0}: {1}', r.status_code, res ) try: blob = json.loads( r.text ) if 'links' in blob: del blob['links'] tool.VERBOSE( json.dumps( blob, sort_keys=True, indent=2, separators=(',', ': ') ) ) return ( res, ver, blob ) except: tool.VERBOSE( r.text ) return ( res, ver, r.text )
def fetch(self): if self.isLoaded: tool.VERBOSE('use csv {}', self.name) return True tool.VERBOSE('fetching csv {}', self.name) with tool.INDENT(): csv_table = table.Table.from_csv( open(self.inputs[0], 'rb'), name=self.name, # sniff_limit = 0, blanks_as_nulls=True, infer_types=True, no_header_row=False) tool.VERBOSE('read {} row(s) from {}', csv_table.count_rows(), self.inputs[0]) connection = self.db.engine.connect() transaction = connection.begin() sql_table = sql.make_table(csv_table, self.name, False, None, self.db.metadata) sql_table.create() if csv_table.count_rows() > 0: insert = sql_table.insert() headers = csv_table.headers() connection.execute( insert, [dict(zip(headers, row)) for row in csv_table.to_rows()]) self._count = csv_table.count_rows() tool.VERBOSE('wrote {} row(s) to table {}', self.count, self.name) self._loaded = True transaction.commit() connection.close() return self._loaded
def getRows(self): if not self.isLoaded: if not self.fetch(): raise 'failed to fetch ' + self.name connection = self.db.engine.connect() rows = connection.execute('select * from {}'.format(self.name)) connection.close() tool.VERBOSE('read {} row(s) from {}', self.count, self.name) return rows
def make_request( method, url, **kwargs ): tool.VERBOSE( method.upper() + ' ' + url ) # tool.VERBOSE( json.dumps( kwargs['headers'], sort_keys=True, indent=2, separators=(',', ': ') ) ) if 'data' in kwargs: tool.VERBOSE( kwargs['data'] ) if not 'auth' in kwargs: kwargs['auth'] = ( env.gsUser(), env.gsPassword() ) r = getattr( requests, method.lower() )( url, **kwargs ) res = True if r.ok else ( None if r.status_code >= 500 else False ) tool.VERBOSE( 'status {0}: {1}', r.status_code, res ) if r.headers.get('content-type','').endswith('png'): tool.VERBOSE( '<binary data>' ) else: tool.VERBOSE( r.text ) return res, r
def expand_object( parents, obj, index ): obj_parents = obj.get( '@parents', [] ) obj_parents.extend( parents ) parents = obj_parents context = '{0} [{1}]'.format( '.'.join( [o['name'] for o in parents[::-1]] ), index ) obj_type = obj.get( '@type', False ) if not obj_type: tool.INFO( 'object at {0} is missing @type', context ) return ( None, None ) if obj_type.rfind('/') >= 0: obj_type = obj_type[ obj_type.rfind('/')+1 : ] context = '[{0}: {1}]'.format( obj_type, context ) template_param = { k:obj[ k ] for k in obj if k[ 0 ] != '@' and k != 'id' } template_param[ 'parents' ] = parents try: cwm_cls = getattr( cwm, obj_type ) argspec = inspect.getargspec( cwm_cls.__init__ ) required_args = argspec.args[ 1:( len(argspec.args) - len(argspec.defaults or []) ) ] cwm_obj = cwm_cls( **template_param ) # print cwm_obj.data return ( parents, cwm_obj ) except TypeError as e: # print 'object at ' + str(context) + ' is missing property.\n Provided: ' + str(template_param.keys()) + '\n Needed: ' + str(required_args) tool.INFO( 'object {0} has wrong properties\n Provided: {1}\n Needed: {2}', context, sorted(template_param.keys()), sorted(required_args) ) tool.VERBOSE( traceback.format_exc() ) return ( None, None ) except Exception as e: tool.INFO( 'object {0} failed: {1}', context, e ) tool.VERBOSE( traceback.format_exc() ) return ( None, None )
def get_headers( self, **extra ): if not self.headers: query = urllib.urlencode( env.oauthParams( self.scope ) ) requestUrl = env.oauthBase() + 'oauth/token?' + query tool.VERBOSE( 'GET ' + requestUrl ) r = requests.get( requestUrl, auth=( self.user, self.password ) ) if not r.ok: tool.VERBOSE( 'response: ' + str(r.status_code) ) tool.VERBOSE( r.text ) tool.INFO( 'Failed to get token for {0}', self.user ) exit(1) self.headers = { 'Authorization': 'Bearer ' + json.loads( r.text )[ 'access_token' ], 'Content-Type': 'application/json' } if extra: h = self.headers.copy() h.update( extra ) return h return self.headers
def create_style( layer ): url = '{0}/rest/styles'.format( env.gsHost() ) template = ''' <style> <name>{0}</name> <filename>{1}.sld</filename> </style> ''' data = template.format( layer['name'], layer['styleName'] ) r,_ = make_request( 'post', url, headers={ 'Content-type': 'text/xml' }, data=data ) if not r: return False url = '{0}/rest/styles/{1}'.format( env.gsHost(), layer['styleName'] ) fh = None try: fn = '{0}/{1}.SLD'.format( args.styleDir, layer['styleName'] ) fh = open( fn, 'rb' ) tool.VERBOSE( 'using SLD {0}', fn ) except: tool.INFO( '{0} missing', fn ) return False template = fh.read() fh.close() data = template.format( layer['name'], layer['title'] ) r,_ = make_request( 'put', url, headers={ 'Content-type': 'application/vnd.ogc.sld+xml' }, data=data ) if not r: return False url = '{0}/rest/layers/{1}'.format( env.gsHost(), layer['name'] ) template = ''' <layer> <defaultStyle> <name>{0}</name> </defaultStyle> </layer> ''' data = template.format( layer['styleName'] ) r,_ = make_request( 'put', url, headers={ 'Content-type': 'text/xml' }, data=data ) return r
def writeCsv(self, filename=None): rows = self.getRows() if filename == None: filename = self.name + '.csv' # Output result of last query as CSV row_count = 0 with open(filename, 'wb') as out: output = CSVKitWriter(out) output.writerow(rows._metadata.keys) for row in rows: output.writerow(row) row_count += 1 tool.VERBOSE('wrote {} row(s) to csv {}', row_count, filename)
def process_object( obj_parents, obj_param, status, index ): parents, cwm_obj = expand_object( obj_parents, obj_param, index ) if not cwm_obj: return ( 0, False ) obj = OrderedDict() detail = OrderedDict() preDelete = obj_param.get( '@preDelete', [] ) obj['mode'] = args.mode or obj_param.get( '@mode', args.defaultMode ) mode = object_mode.get( obj['mode'], 0 ) data = cwm_obj.data detail['type'] = cwm_obj.type_name detail['name'] = data['name'] detail['parents'] = [ o['name'] for o in parents[::-1] ] obj['obj'] = "{0}: {1}".format( detail['type'], '.'.join( detail['parents'] + [detail['name']] ) ) if not mode: tool.INFO( "mode {0} not valid for {1}", obj['mode'], obj['obj'] ) return ( 0, False ) if not obj_param.get( '@enabled', True ): tool.INFO( "{0} not enabled, skipping", obj['obj'] ) return ( 0, False ) status.append( obj ) svc = service[ cwm_obj.service ] detail['exists'], detail['ver'], detail['id'] = svc.element_exists( cwm_obj.access ) if mode == TEST: if detail['exists']: tool.VERBOSE( obj['obj'] + ' exists' ) elif detail['exists'] == False: tool.VERBOSE( obj['obj'] + ' missing' ) else: tool.VERBOSE( obj['obj'] + ' unknown' ) obj['successful'] = detail['exists'] == True if preDelete and ( mode == DELETE or mode == RECREATE ): detail['preDeleted'] = {} for pd in preDelete: s, u = pd pd_exists, pd_ver, pd_id = service[s].element_exists( u ) if pd_exists != False or args.force: detail['preDeleted'][u],_,_ = service[s].delete_element( u, pd_ver ) # if detail['preDeleted'][pd]: # tool.VERBOSE( obj['obj'] +' pre-deleted '+pd ) status.append( OrderedDict([ ( 'mode', 'predelete' ), ( 'obj', u ), ( 'successful', detail['preDeleted'][u] ) ])) if mode == DELETE: if args.force or detail['exists'] != False: detail['deleted'],_,_ = svc.delete_element( cwm_obj.access, detail['ver'] ) # if detail['deleted']: # tool.VERBOSE( obj['obj'] + ' deleted' ) obj['successful'] = detail['deleted'] else: obj['successful'] = True if mode == RECREATE: if args.force or detail['exists'] != False: detail['deleted'],_,_ = svc.delete_element( cwm_obj.access, detail['ver'] ) # if detail['deleted']: # tool.VERBOSE( obj['obj'] + ' deleted' ) if detail['exists'] == False or detail['deleted'] or args.force: detail['created'], detail['newVer'], detail['id'] = svc.load_element( cwm_obj.create, data ) # if detail['created']: # tool.VERBOSE( obj['obj'] + ' created' ) obj['successful'] = detail['created'] else: obj['successful'] = False if mode == CREATE: if detail['exists'] == False: detail['created'], detail['newVer'], detail['id'] = svc.load_element( cwm_obj.create, data ) # if detail['created']: # tool.VERBOSE( obj['obj'] + ' created' ) obj['successful'] = detail['created'] else: obj['successful'] = detail['exists'] == True if mode == UPDATE: if detail['exists']: detail['updated'], detail['newVer'], detail['id'] = svc.update_element( cwm_obj.access, detail['ver'], detail['id'], data ) # if detail['updated']: # tool.VERBOSE( obj['obj'] + ' updated' ) obj['successful'] = detail['updated'] elif detail['exists'] == False: detail['created'], detail['newVer'], detail['id'] = svc.load_element( cwm_obj.create, data ) # if detail['created']: # tool.VERBOSE( obj['obj'] + ' created' ) obj['successful'] = detail['created'] else: obj['successful'] = False tool.STATUS( '({1}) {0}', obj['obj'], 'OK' if obj['successful'] else 'FAIL' ) tool.VERBOSE() if args.stop and not obj['successful']: return ( 1, False ) count = 0 if obj_param.get( '@children' ): parent = { 'name': detail[ 'name' ], 'id': detail[ 'id' ] } c, ok = process_child_objects( [ parent ] + parents, obj_param['@children'], status ) count += c if args.stop and not ok: return ( 1 + count, False ) return ( 1 + count, True )
status = [] for name in tool.get_filenames(): file_status = OrderedDict() file_status[ 'filename' ] = name file_status[ 'objects' ] = [] status.append( file_status ) objects = [] try: tool.INFO( 'reading {0}... \\', name ) with open( name, 'rb') as file: objects = json.loads( file.read() ) except Exception as e: tool.INFO( 'failed' ) tool.VERBOSE( traceback.format_exc() ) tool.INFO() file_status[ 'error' ] = str(e) continue tool.VERBOSE() file_status[ 'count' ], ok = process_child_objects( [], objects, file_status[ 'objects' ] ) tool.STATUS( '' ) tool.INFO( 'processed {0} object(s)\n', file_status[ 'count' ] ) if args.stop and not ok: break if not status:
if not ok: return ( False, 'Unable get layer with WMS GetMap' ) return ( True, None ) # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - status = [] total_layers = 0 total_failed = 0 for name in tool.get_filenames(): layer_status = OrderedDict([('filename',name)]) status.append( layer_status ) layers = [] try: tool.VERBOSE( 'reading {0}...', name ) with open( name, 'rb') as file: layers = json.loads( file.read() ) except Exception as e: tool.VERBOSE( traceback.format_exc() ) layer_status['error']=str(e) layer_status['layers'] = layers failed = 0 for layer in layers: try: total_layers += 1 if args.test: tool.INFO( 'Testing {0}: \\', layer['name'] )