def GET(self, url, **params):
headers = getExtraHeaders()
if 'minerva_credentials' in params:
creds = params['minerva_credentials']
del params['minerva_credentials']
auth = 'Basic ' + b64encode(decryptCredentials(bytes(creds)))
headers['Authorization'] = auth
r = requests.get(url, params=params, headers=headers)
cherrypy.response.headers['Content-Type'] = r.headers['content-type']
return r.content
def GET(self, url, **params):
headers = getExtraHeaders()
if 'minerva_credentials' in params:
creds = params['minerva_credentials']
del params['minerva_credentials']
auth = 'Basic ' + b64encode(decryptCredentials(bytes(creds)))
headers['Authorization'] = auth
r = requests.get(url, params=params, headers=headers)
cherrypy.response.headers['Content-Type'] = r.headers['content-type']
return r.content
def createWmsDataset(self, wmsSource, params):
baseURL = wmsSource['wms_params']['base_url']
parsedUrl = getUrlParts(baseURL)
typeName = params['typeName']
try:
if params['metadata']:
layer_info = params['metadata']
else:
layer_info = WmsStyle(typeName, baseURL).get_layer_info()
except TypeError:
layer_info = ""
if 'credentials' in wmsSource['wms_params']:
credentials = (wmsSource['wms_params']['credentials'])
basic_auth = 'Basic ' + b64encode(decryptCredentials(credentials))
headers = {'Authorization': basic_auth}
else:
headers = {}
credentials = None
request_url = parsedUrl.scheme + '://' + parsedUrl.netloc + \
parsedUrl.path
r = requests.get(request_url,
params={
'service': 'WMS',
'request': 'GetLegendGraphic',
'format': 'image/png',
'width': 20,
'height': 20,
'layer': params['typeName']
},
headers=headers)
legend = b64encode(r.content)
self.requireParams(('name'), params)
name = params['name']
minerva_metadata = {
'dataset_type': 'wms',
'legend': legend,
'source': wmsSource,
'type_name': typeName,
'base_url': baseURL,
'layer_info': layer_info,
'abstract': params['abstract'],
'category': params['category']
}
if credentials:
minerva_metadata['credentials'] = credentials
dataset = self.constructDataset(name, minerva_metadata)
return dataset
def createWmsDataset(self, wmsSource, params):
baseURL = wmsSource['wms_params']['base_url']
parsedUrl = getUrlParts(baseURL)
typeName = params['typeName']
try:
if params['metadata']:
layer_info = params['metadata']
else:
layer_info = WmsStyle(typeName, baseURL).get_layer_info()
except TypeError:
layer_info = ""
if 'credentials' in wmsSource['wms_params']:
credentials = (
wmsSource['wms_params']['credentials']
)
basic_auth = 'Basic ' + b64encode(decryptCredentials(credentials))
headers = {'Authorization': basic_auth}
else:
headers = {}
credentials = None
request_url = parsedUrl.scheme + '://' + parsedUrl.netloc + \
parsedUrl.path
r = requests.get(request_url, params={
'service': 'WMS',
'request': 'GetLegendGraphic',
'format': 'image/png',
'width': 20,
'height': 20,
'layer': params['typeName']}, headers=headers)
legend = b64encode(r.content)
self.requireParams(('name'), params)
name = params['name']
minerva_metadata = {
'dataset_type': 'wms',
'legend': legend,
'source': wmsSource,
'type_name': typeName,
'base_url': baseURL,
'layer_info': layer_info,
'abstract': params['abstract'],
'category': params['category']
}
if credentials:
minerva_metadata['credentials'] = credentials
dataset = self.constructDataset(name, minerva_metadata)
return dataset
def testCreateWmsSourceWithAuthentication(self):
"""
Enter a username and password for a WMS source and ensure that they
are correctly encrypted/decrypted.
"""
with HTTMock(wms_mock):
response = self.request(path=self._path, method='POST',
params=self._params, user=self._user)
self.assertStatusOk(response)
wmsSource = response.json
from girder.plugins.minerva.utility.minerva_utility import decryptCredentials
credentials = [decryptCredentials(bytes(d['meta']['minerva']['credentials']))
for d in wmsSource]
self.assertEquals(set(credentials), {"{}:{}".format(self._params['username'],
self._params['password'])})
def save_geojson(self):
"""
Save GeoJSON from a Minerva item
TODO: Use caching like the girder_worker.girder_io plugin
TODO: Separate methods for saving geojson from different sources
TODO: Get geojson via WFS calls for local WMS vector layers
"""
minerva = self.meta['meta']['minerva']
if 'geojson_file' in minerva:
# Uploaded GeoJSON is stored as a file in Girder
self.client.downloadFile(minerva['geojson_file']['_id'], self.uri)
elif 'geojson' in minerva:
# Mongo collection is stored in item meta
geojson = json.loads(minerva['geojson']['data'])
# TODO: Don't use mongo metadata for filename
with open(self.uri, 'w') as outjson:
json.dump(geojson, outjson)
elif 'dataset_type' in minerva and minerva['dataset_type'] == 'wms':
servers = config.getConfig()['gaia_minerva_wms']['servers']
if minerva['base_url'] in servers:
params = 'srsName=EPSG:4326&typename={name}&outputFormat=json'\
+ '&version=1.0.0&service=WFS&request=GetFeature'
url = '{base}?{params}'.format(
base=minerva['base_url'].replace('/wms', '/wfs'),
params=params.format(name=minerva['type_name']))
headers = {}
if 'credentials' in minerva:
credentials = (minerva['credentials'])
basic_auth = 'Basic ' + b64encode(
decryptCredentials(credentials))
headers = {'Authorization': basic_auth}
with open(self.uri, 'w') as outjson:
r = requests.get(url, headers=headers)
r.raise_for_status()
json.dump(r.json(), outjson)
else:
raise GaiaException(
'This server {} is not supported. \n{}'.format(minerva))
else:
raise GaiaException(
'Unsupported data source. \n{}'.format(minerva))
def createWmsDataset(self, wmsSource, params):
baseURL = wmsSource['meta']['minerva']['wms_params']['base_url']
parsedUrl = getUrlParts(baseURL)
typeName = params['typeName']
if 'credentials' in wmsSource['meta']['minerva']['wms_params']:
credentials = (
wmsSource['meta']['minerva']['wms_params']['credentials']
)
basic_auth = 'Basic ' + b64encode(decryptCredentials(credentials))
headers = {'Authorization': basic_auth}
else:
headers = {}
credentials = None
conn = httplib.HTTPConnection(parsedUrl.netloc)
conn.request("GET",
parsedUrl.path +
"?service=WMS&request=" +
"GetLegendGraphic&format=image" +
"%2Fpng&width=20&height=20&layer=" +
typeName, headers=headers
)
response = conn.getresponse()
legend = binascii.b2a_base64(response.read())
self.requireParams(('name'), params)
name = params['name']
minerva_metadata = {
'dataset_type': 'wms',
'legend': legend,
'source_id': wmsSource['_id'],
'type_name': typeName,
'base_url': baseURL
}
if credentials:
minerva_metadata['credentials'] = credentials
dataset = self.constructDataset(name, minerva_metadata)
return dataset
def run(job):
job_model = ModelImporter.model('job', 'jobs')
job_model.updateJob(job, status=JobStatus.RUNNING)
try:
kwargs = job['kwargs']
# TODO better to create a job token rather than a user token?
token = kwargs['token']
datasetId = str(kwargs['dataset']['_id'])
# connect to girder and upload the file
# TODO will probably have to change this from local to girder worker
# so that can work on worker machine
# at least need host connection info
girderPort = config.getConfig()['server.socket_port']
client = girder_client.GirderClient(port=girderPort)
client.token = token['_id']
# Get datasource
source = client.getItem(kwargs['params']['sourceId'])
esUrl = 'https://%s@%s' % (decryptCredentials(
source['meta']['minerva']['elasticsearch_params']['credentials']),
source['meta']['minerva']['elasticsearch_params']['host_name'])
es = Elasticsearch([esUrl])
# TODO sleeping in async thread, probably starving other tasks
# would be better to split this into two or more parts, creating
# additional jobs as needed
searchResult = es.search(
index=source['meta']['minerva']['elasticsearch_params']['index'],
body=json.loads(kwargs['params']['searchParams']))
# write the output to a json file
tmpdir = tempfile.mkdtemp()
outFilepath = tempfile.mkstemp(suffix='.json', dir=tmpdir)[1]
writer = open(outFilepath, 'w')
writer.write(json.dumps(searchResult))
writer.close()
# rename the file so it will have the right name when uploaded
# could probably be done post upload
outFilename = 'search.json'
humanFilepath = os.path.join(tmpdir, outFilename)
shutil.move(outFilepath, humanFilepath)
client.uploadFileToItem(datasetId, humanFilepath)
# TODO some stuff here using models will only work on a local job
# will have to be rewritten using girder client to work in girder worker
# non-locally
user_model = ModelImporter.model('user')
user = user_model.load(job['userId'], force=True)
item_model = ModelImporter.model('item')
# TODO only works locally
dataset = item_model.load(datasetId, level=AccessType.WRITE, user=user)
metadata = dataset['meta']
minerva_metadata = metadata['minerva']
# TODO only works locally
file_model = ModelImporter.model('file')
existing = file_model.findOne({
'itemId': dataset['_id'],
'name': outFilename
})
if existing:
minerva_metadata['original_files'] = [{
'_id': existing['_id'],
'name': outFilename
}]
else:
raise (Exception('Cannot find file %s in dataset %s' %
(outFilename, datasetId)))
jsonRow = jsonArrayHead(humanFilepath, limit=1)[0]
minerva_metadata['json_row'] = jsonRow
shutil.rmtree(tmpdir)
metadata['minerva'] = minerva_metadata
# TODO only works locally
item_model.setMetadata(dataset, metadata)
# TODO only works locally
job_model.updateJob(job, status=JobStatus.SUCCESS)
except Exception:
t, val, tb = sys.exc_info()
log = '%s: %s\n%s' % (t.__name__, repr(val), traceback.extract_tb(tb))
# TODO only works locally
job_model.updateJob(job, status=JobStatus.ERROR, log=log)
raise