def fetch(self):
if self.isLoaded:
tool.VERBOSE('use query {}', self.name)
return True
tool.VERBOSE('fetching query {}', self.name)
with tool.INDENT():
for i in self.inputs:
if isinstance(i, TableBase):
if not i.fetch():
raise 'failed to fetch ' + i.name
connection = self.db.engine.connect()
connection.execute('create table {} as {}'.format(
self.name, self.sql))
self._count = connection.execute(
'select count(*) as count from {}'.format(self.name)).scalar()
connection.close()
tool.VERBOSE('wrote {} row(s) to table {}', self.count, self.name)
self._loaded = True
return self._loaded
def _make_request( self, method, url, **kwargs ):
tool.VERBOSE( method.upper() + ' ' + url )
# tool.VERBOSE( json.dumps( kwargs['headers'], sort_keys=True, indent=2, separators=(',', ': ') ) )
if 'data' in kwargs:
tool.VERBOSE( json.dumps( json.loads(kwargs['data']), sort_keys=True, indent=2, separators=(',', ': ') ) )
if args.dryrun:
return ( 'dryrun', '', {} )
r = getattr( requests, method.lower() )( url, **kwargs )
res = True if r.ok else ( None if r.status_code >= 500 else False )
ver = eval( r.headers.get( 'ETag', '""') )
tool.VERBOSE( 'status {0}: {1}', r.status_code, res )
try:
blob = json.loads( r.text )
if 'links' in blob:
del blob['links']
tool.VERBOSE( json.dumps( blob, sort_keys=True, indent=2, separators=(',', ': ') ) )
return ( res, ver, blob )
except:
tool.VERBOSE( r.text )
return ( res, ver, r.text )
def fetch(self):
if self.isLoaded:
tool.VERBOSE('use csv {}', self.name)
return True
tool.VERBOSE('fetching csv {}', self.name)
with tool.INDENT():
csv_table = table.Table.from_csv(
open(self.inputs[0], 'rb'),
name=self.name,
# sniff_limit = 0,
blanks_as_nulls=True,
infer_types=True,
no_header_row=False)
tool.VERBOSE('read {} row(s) from {}', csv_table.count_rows(),
self.inputs[0])
connection = self.db.engine.connect()
transaction = connection.begin()
sql_table = sql.make_table(csv_table, self.name, False, None,
self.db.metadata)
sql_table.create()
if csv_table.count_rows() > 0:
insert = sql_table.insert()
headers = csv_table.headers()
connection.execute(
insert,
[dict(zip(headers, row)) for row in csv_table.to_rows()])
self._count = csv_table.count_rows()
tool.VERBOSE('wrote {} row(s) to table {}', self.count,
self.name)
self._loaded = True
transaction.commit()
connection.close()
return self._loaded
def getRows(self):
if not self.isLoaded:
if not self.fetch():
raise 'failed to fetch ' + self.name
connection = self.db.engine.connect()
rows = connection.execute('select * from {}'.format(self.name))
connection.close()
tool.VERBOSE('read {} row(s) from {}', self.count, self.name)
return rows
def make_request( method, url, **kwargs ):
tool.VERBOSE( method.upper() + ' ' + url )
# tool.VERBOSE( json.dumps( kwargs['headers'], sort_keys=True, indent=2, separators=(',', ': ') ) )
if 'data' in kwargs:
tool.VERBOSE( kwargs['data'] )
if not 'auth' in kwargs:
kwargs['auth'] = ( env.gsUser(), env.gsPassword() )
r = getattr( requests, method.lower() )( url, **kwargs )
res = True if r.ok else ( None if r.status_code >= 500 else False )
tool.VERBOSE( 'status {0}: {1}', r.status_code, res )
if r.headers.get('content-type','').endswith('png'):
tool.VERBOSE( '<binary data>' )
else:
tool.VERBOSE( r.text )
return res, r
def expand_object( parents, obj, index ):
obj_parents = obj.get( '@parents', [] )
obj_parents.extend( parents )
parents = obj_parents
context = '{0} [{1}]'.format( '.'.join( [o['name'] for o in parents[::-1]] ), index )
obj_type = obj.get( '@type', False )
if not obj_type:
tool.INFO( 'object at {0} is missing @type', context )
return ( None, None )
if obj_type.rfind('/') >= 0:
obj_type = obj_type[ obj_type.rfind('/')+1 : ]
context = '[{0}: {1}]'.format( obj_type, context )
template_param = { k:obj[ k ] for k in obj if k[ 0 ] != '@' and k != 'id' }
template_param[ 'parents' ] = parents
try:
cwm_cls = getattr( cwm, obj_type )
argspec = inspect.getargspec( cwm_cls.__init__ )
required_args = argspec.args[ 1:( len(argspec.args) - len(argspec.defaults or []) ) ]
cwm_obj = cwm_cls( **template_param )
# print cwm_obj.data
return ( parents, cwm_obj )
except TypeError as e:
# print 'object at ' + str(context) + ' is missing property.\n Provided: ' + str(template_param.keys()) + '\n Needed: ' + str(required_args)
tool.INFO( 'object {0} has wrong properties\n Provided: {1}\n Needed: {2}', context, sorted(template_param.keys()), sorted(required_args) )
tool.VERBOSE( traceback.format_exc() )
return ( None, None )
except Exception as e:
tool.INFO( 'object {0} failed: {1}', context, e )
tool.VERBOSE( traceback.format_exc() )
return ( None, None )
def get_headers( self, **extra ):
if not self.headers:
query = urllib.urlencode( env.oauthParams( self.scope ) )
requestUrl = env.oauthBase() + 'oauth/token?' + query
tool.VERBOSE( 'GET ' + requestUrl )
r = requests.get( requestUrl, auth=( self.user, self.password ) )
if not r.ok:
tool.VERBOSE( 'response: ' + str(r.status_code) )
tool.VERBOSE( r.text )
tool.INFO( 'Failed to get token for {0}', self.user )
exit(1)
self.headers = {
'Authorization': 'Bearer ' + json.loads( r.text )[ 'access_token' ],
'Content-Type': 'application/json'
}
if extra:
h = self.headers.copy()
h.update( extra )
return h
return self.headers
def create_style( layer ):
url = '{0}/rest/styles'.format( env.gsHost() )
template = '''
<style>
<name>{0}</name>
<filename>{1}.sld</filename>
</style>
'''
data = template.format( layer['name'], layer['styleName'] )
r,_ = make_request( 'post', url, headers={ 'Content-type': 'text/xml' }, data=data )
if not r:
return False
url = '{0}/rest/styles/{1}'.format( env.gsHost(), layer['styleName'] )
fh = None
try:
fn = '{0}/{1}.SLD'.format( args.styleDir, layer['styleName'] )
fh = open( fn, 'rb' )
tool.VERBOSE( 'using SLD {0}', fn )
except:
tool.INFO( '{0} missing', fn )
return False
template = fh.read()
fh.close()
data = template.format( layer['name'], layer['title'] )
r,_ = make_request( 'put', url, headers={ 'Content-type': 'application/vnd.ogc.sld+xml' }, data=data )
if not r:
return False
url = '{0}/rest/layers/{1}'.format( env.gsHost(), layer['name'] )
template = '''
<layer>
<defaultStyle>
<name>{0}</name>
</defaultStyle>
</layer>
'''
data = template.format( layer['styleName'] )
r,_ = make_request( 'put', url, headers={ 'Content-type': 'text/xml' }, data=data )
return r
def writeCsv(self, filename=None):
rows = self.getRows()
if filename == None:
filename = self.name + '.csv'
# Output result of last query as CSV
row_count = 0
with open(filename, 'wb') as out:
output = CSVKitWriter(out)
output.writerow(rows._metadata.keys)
for row in rows:
output.writerow(row)
row_count += 1
tool.VERBOSE('wrote {} row(s) to csv {}', row_count, filename)
def process_object( obj_parents, obj_param, status, index ):
parents, cwm_obj = expand_object( obj_parents, obj_param, index )
if not cwm_obj:
return ( 0, False )
obj = OrderedDict()
detail = OrderedDict()
preDelete = obj_param.get( '@preDelete', [] )
obj['mode'] = args.mode or obj_param.get( '@mode', args.defaultMode )
mode = object_mode.get( obj['mode'], 0 )
data = cwm_obj.data
detail['type'] = cwm_obj.type_name
detail['name'] = data['name']
detail['parents'] = [ o['name'] for o in parents[::-1] ]
obj['obj'] = "{0}: {1}".format( detail['type'], '.'.join( detail['parents'] + [detail['name']] ) )
if not mode:
tool.INFO( "mode {0} not valid for {1}", obj['mode'], obj['obj'] )
return ( 0, False )
if not obj_param.get( '@enabled', True ):
tool.INFO( "{0} not enabled, skipping", obj['obj'] )
return ( 0, False )
status.append( obj )
svc = service[ cwm_obj.service ]
detail['exists'], detail['ver'], detail['id'] = svc.element_exists( cwm_obj.access )
if mode == TEST:
if detail['exists']:
tool.VERBOSE( obj['obj'] + ' exists' )
elif detail['exists'] == False:
tool.VERBOSE( obj['obj'] + ' missing' )
else:
tool.VERBOSE( obj['obj'] + ' unknown' )
obj['successful'] = detail['exists'] == True
if preDelete and ( mode == DELETE or mode == RECREATE ):
detail['preDeleted'] = {}
for pd in preDelete:
s, u = pd
pd_exists, pd_ver, pd_id = service[s].element_exists( u )
if pd_exists != False or args.force:
detail['preDeleted'][u],_,_ = service[s].delete_element( u, pd_ver )
# if detail['preDeleted'][pd]:
# tool.VERBOSE( obj['obj'] +' pre-deleted '+pd )
status.append( OrderedDict([
( 'mode', 'predelete' ),
( 'obj', u ),
( 'successful', detail['preDeleted'][u] )
]))
if mode == DELETE:
if args.force or detail['exists'] != False:
detail['deleted'],_,_ = svc.delete_element( cwm_obj.access, detail['ver'] )
# if detail['deleted']:
# tool.VERBOSE( obj['obj'] + ' deleted' )
obj['successful'] = detail['deleted']
else:
obj['successful'] = True
if mode == RECREATE:
if args.force or detail['exists'] != False:
detail['deleted'],_,_ = svc.delete_element( cwm_obj.access, detail['ver'] )
# if detail['deleted']:
# tool.VERBOSE( obj['obj'] + ' deleted' )
if detail['exists'] == False or detail['deleted'] or args.force:
detail['created'], detail['newVer'], detail['id'] = svc.load_element( cwm_obj.create, data )
# if detail['created']:
# tool.VERBOSE( obj['obj'] + ' created' )
obj['successful'] = detail['created']
else:
obj['successful'] = False
if mode == CREATE:
if detail['exists'] == False:
detail['created'], detail['newVer'], detail['id'] = svc.load_element( cwm_obj.create, data )
# if detail['created']:
# tool.VERBOSE( obj['obj'] + ' created' )
obj['successful'] = detail['created']
else:
obj['successful'] = detail['exists'] == True
if mode == UPDATE:
if detail['exists']:
detail['updated'], detail['newVer'], detail['id'] = svc.update_element( cwm_obj.access, detail['ver'], detail['id'], data )
# if detail['updated']:
# tool.VERBOSE( obj['obj'] + ' updated' )
obj['successful'] = detail['updated']
elif detail['exists'] == False:
detail['created'], detail['newVer'], detail['id'] = svc.load_element( cwm_obj.create, data )
# if detail['created']:
# tool.VERBOSE( obj['obj'] + ' created' )
obj['successful'] = detail['created']
else:
obj['successful'] = False
tool.STATUS( '({1}) {0}', obj['obj'], 'OK' if obj['successful'] else 'FAIL' )
tool.VERBOSE()
if args.stop and not obj['successful']:
return ( 1, False )
count = 0
if obj_param.get( '@children' ):
parent = {
'name': detail[ 'name' ],
'id': detail[ 'id' ]
}
c, ok = process_child_objects( [ parent ] + parents, obj_param['@children'], status )
count += c
if args.stop and not ok:
return ( 1 + count, False )
return ( 1 + count, True )
status = []
for name in tool.get_filenames():
file_status = OrderedDict()
file_status[ 'filename' ] = name
file_status[ 'objects' ] = []
status.append( file_status )
objects = []
try:
tool.INFO( 'reading {0}... \\', name )
with open( name, 'rb') as file:
objects = json.loads( file.read() )
except Exception as e:
tool.INFO( 'failed' )
tool.VERBOSE( traceback.format_exc() )
tool.INFO()
file_status[ 'error' ] = str(e)
continue
tool.VERBOSE()
file_status[ 'count' ], ok = process_child_objects( [], objects, file_status[ 'objects' ] )
tool.STATUS( '' )
tool.INFO( 'processed {0} object(s)\n', file_status[ 'count' ] )
if args.stop and not ok:
break
if not status:
if not ok:
return ( False, 'Unable get layer with WMS GetMap' )
return ( True, None )
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
status = []
total_layers = 0
total_failed = 0
for name in tool.get_filenames():
layer_status = OrderedDict([('filename',name)])
status.append( layer_status )
layers = []
try:
tool.VERBOSE( 'reading {0}...', name )
with open( name, 'rb') as file:
layers = json.loads( file.read() )
except Exception as e:
tool.VERBOSE( traceback.format_exc() )
layer_status['error']=str(e)
layer_status['layers'] = layers
failed = 0
for layer in layers:
try:
total_layers += 1
if args.test:
tool.INFO( 'Testing {0}: \\', layer['name'] )
