def _importAnalysis(self):
"""Setup and import analyses for bsve tests."""
if self._import_done:
return
path = "/minerva_analysis/folder"
response = self.request(path=path, method="POST", user=self._user)
self.assertStatusOk(response)
analyses_folder = response.json["folder"]
# import the bsve analysis
client = GirderClient("localhost", girder_port)
client.authenticate("minervauser", "password")
bsve_analysis_path = os.path.abspath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), "../analyses/bsve")
)
import_analyses.import_analyses(client, bsve_analysis_path)
path = "/item"
params = {"folderId": analyses_folder["_id"]}
response = self.request(path=path, method="GET", params=params, user=self._user)
self.assertStatusOk(response)
self.assertEquals(len(response.json), 2, "Expecting only one analysis")
for analysis in response.json:
if analysis["name"] == "bsve search":
search_analysis = analysis
elif analysis["name"] == "MMWR data import":
soda_analysis = analysis
else:
self.fail('Unexpected analysis found "%s".' % analysis["name"])
expected_meta = {
u"minerva": {
u"analysis_type": u"bsve_search",
u"analysis_name": u"bsve search",
u"analysis_id": search_analysis["_id"],
}
}
self.assertEquals(search_analysis["meta"], expected_meta, "Unexpected value for search meta data")
expected_meta = {
u"minerva": {
u"analysis_type": u"mmwr_import_data",
u"analysis_name": u"MMWR data import",
u"analysis_id": soda_analysis["_id"],
}
}
self.assertEquals(soda_analysis["meta"], expected_meta, "Unexpected value for soda meta data")
# create the dataset folder
path = "/minerva_dataset/folder"
params = {"userId": self._user["_id"]}
response = self.request(path=path, method="POST", params=params, user=self._user)
self.assertStatusOk(response)
self._importDone = True
def testBsveSearchAnalysis(self):
# create the analysis folder
path = '/minerva_analysis/folder'
response = self.request(path=path, method='POST', user=self._user)
self.assertStatusOk(response)
analyses_folder = response.json['folder']
# import the bsve analysis
client = GirderClient('localhost', girder_port)
client.authenticate('minervauser', 'password')
bsve_analysis_path = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../analyses/bsve'))
import_analyses.import_analyses(client, bsve_analysis_path)
path = '/item'
params = {
'folderId': analyses_folder['_id']
}
response = self.request(path=path, method='GET', params=params, user=self._user)
self.assertStatusOk(response)
self.assertEquals(len(response.json), 1, 'Expecting only one analysis')
analysis = response.json[0]
self.assertEquals(analysis['name'], 'bsve search', 'Expecting analysis name to be "bsve search"')
expected_meta = {
u'minerva': {
u'analysis_type': u'bsve_search',
u'analysis_name': u'bsve search',
u'analysis_id': analysis['_id']
}
}
self.assertEquals(analysis['meta'], expected_meta, 'Unexpected value for meta data')
# create the dataset folder
path = '/minerva_dataset/folder'
params = {
'userId': self._user['_id'],
}
response = self.request(path=path, method='POST', params=params, user=self._user)
self.assertStatusOk(response)
# mock the calls to bsve search
@urlmatch(netloc=r'(.*\.)?beta-search.bsvecosystem.net(.*)$')
def bsve_mock(url, request):
if url.path.split('/')[-1] == 'request':
return httmock.response(200, '12345')
else:
pluginTestDir = os.path.dirname(os.path.realpath(__file__))
filepath = os.path.join(pluginTestDir, 'data', 'bsve_search.json')
with open(filepath) as bsve_search_file:
content = {
'status': 1,
'results': json.load(bsve_search_file)
}
headers = {
'content-length': len(content),
'content-type': 'application/json'
}
return httmock.response(200, content, headers, request=request)
with HTTMock(bsve_mock):
response = self.request(
path='/minerva_analysis/bsve_search',
method='POST',
params={
'datasetName': 'test dataset',
'bsveSearchParams': '{}'
},
user=self._user
)
# wait for the async job to complete
searchResultsFinished = False
count = 0
while not searchResultsFinished and count < 5:
# get the dataset and check if it has been updated
path = '/minerva_dataset/%s/dataset' % str(response.json['dataset_id'])
response = self.request(
path=path,
method='GET',
user=self._user
)
dataset = response.json
if 'json_row' in dataset:
searchResultsFinished = True
else:
time.sleep(2)
count += 1
# ensure the first row of results was added to the dataset
self.assertTrue('json_row' in dataset, 'json_row expected in dataset')
self.assertTrue('data' in dataset['json_row'], 'data should be in json_row')
self.assertTrue('Longitude' in dataset['json_row']['data'], 'data.Longitude should be in json_row')
# ensure that we can map the Lat/Long to geojson, as this json has
# unicode values for Lat/Long
# update the minerva metadata with coordinate mapping
metadata = {'minerva': dataset}
metadata['minerva']['mapper'] = {
"latitudeKeypath": "data.Latitude",
"longitudeKeypath": "data.Longitude"
}
path = '/item/{}/metadata'.format(dataset['dataset_id'])
response = self.request(
path=path,
method='PUT',
user=self._user,
body=json.dumps(metadata),
type='application/json'
)
metadata = response.json
# create geojson in the dataset
path = '/minerva_dataset/{}/geojson'.format(dataset['dataset_id'])
response = self.request(
path=path,
method='POST',
user=self._user,
)
self.assertHasKeys(response.json, ['geojson_file'])
def _importAnalysis(self):
"""Setup and import analyses for bsve tests."""
if self._import_done:
return
path = '/minerva_analysis/folder'
response = self.request(path=path, method='POST', user=self._user)
self.assertStatusOk(response)
analyses_folder = response.json['folder']
# import the bsve analysis
client = GirderClient('localhost', girder_port)
client.authenticate('minervauser', 'password')
bsve_analysis_path = os.path.abspath(
os.path.join(os.path.dirname(os.path.realpath(__file__)),
'../analyses/bsve'))
import_analyses.import_analyses(client, bsve_analysis_path)
path = '/item'
params = {'folderId': analyses_folder['_id']}
response = self.request(path=path,
method='GET',
params=params,
user=self._user)
self.assertStatusOk(response)
self.assertEquals(len(response.json), 2, 'Expecting only one analysis')
for analysis in response.json:
if analysis['name'] == 'bsve search':
search_analysis = analysis
elif analysis['name'] == 'MMWR data import':
soda_analysis = analysis
else:
self.fail('Unexpected analysis found "%s".' % analysis['name'])
expected_meta = {
u'minerva': {
u'analysis_type': u'bsve_search',
u'analysis_name': u'bsve search',
u'analysis_id': search_analysis['_id']
}
}
self.assertEquals(search_analysis['meta'], expected_meta,
'Unexpected value for search meta data')
expected_meta = {
u'minerva': {
u'analysis_type': u'mmwr_import_data',
u'analysis_name': u'MMWR data import',
u'analysis_id': soda_analysis['_id']
}
}
self.assertEquals(soda_analysis['meta'], expected_meta,
'Unexpected value for soda meta data')
# create the dataset folder
path = '/minerva_dataset/folder'
params = {
'userId': self._user['_id'],
}
response = self.request(path=path,
method='POST',
params=params,
user=self._user)
self.assertStatusOk(response)
self._importDone = True
def testImportAnalyses(self):
"""
Test importing a romanesco analysis
"""
client = GirderClient('localhost', girder_port)
client.authenticate(self._username, self._password)
path = os.path.dirname(os.path.realpath(__file__))
analyses_path = os.path.join(path, 'analyses')
import_analyses.import_analyses(client, analyses_path)
# Get the analysis folder
path = '/minerva_analysis/folder'
response = self.request(path=path, method='GET', params={}, user=self._user)
self.assertStatusOk(response)
analyses_folder = response.json['folder']
path = '/item'
params = {
'folderId': analyses_folder['_id']
}
response = self.request(path=path, method='GET', params=params, user=self._user)
self.assertStatusOk(response)
self.assertEquals(len(response.json), 2, 'Expecting two analyses')
analysis = response.json[0]
self.assertEquals(analysis['name'], 'add', 'Expecting analysis one name to be "add"')
expected_meta = {
u'minerva': {
u'analysis_type': u'add',
u'analysis_name': u'add',
u'analysis_id': analysis['_id']
},
u'analysis': {
u'inputs': [{
u'default': {
u'data': u'0',
u'format': u'json'
},
u'type': u'number',
u'name': u'a',
u'format': u'number'
},
{
u'type': u'number',
u'name': u'b',
u'format':
u'number'
}],
u'script': u'c = a + b',
u'mode': u'python',
u'outputs': [{
u'type': u'number',
u'name': u'c',
u'format': u'number'
}],
u'name': u'add'
}
}
self.assertEquals(analysis['meta'], expected_meta, 'Unexpected value for meta data')
analysis = response.json[1]
self.assertEquals(analysis['name'], 'local', 'Expecting analysis two name to be "local"')
expected_meta = {
u'minerva': {
u'analysis_type': u'local type',
u'analysis_name': u'local',
u'analysis_id': analysis['_id']
}
}
self.assertEquals(analysis['meta'], expected_meta, 'Unexpected value for meta data')
def setUp(self):
"""
Set up the test case with a user
"""
super(ContourAnalysesTestCase, self).setUp()
self._username = '******'
self._password = '******'
self._user = self.model('user').createUser(
self._username, self._password, 'minerva', 'user',
'*****@*****.**')
# Import the analyses
self._client = GirderClient('localhost', girder_port)
self._client.authenticate(self._username, self._password)
path = os.path.dirname(os.path.realpath(__file__))
analyses_path = os.path.join(path, '../analyses/NEX/')
import_analyses.import_analyses(self._client, analyses_path)
# Get the analysis folder
path = '/minerva_analysis/folder'
response = self.request(path=path, method='GET', params={}, user=self._user)
self.assertStatusOk(response)
analyses_folder = response.json['folder']
path = '/item'
params = {
'folderId': analyses_folder['_id']
}
response = self.request(path=path, method='GET', params=params, user=self._user)
self.assertStatusOk(response)
# Find the contour analysis
for analysis in response.json:
if analysis['name'] == 'contour':
self._analysis = analysis
# Now import an S3 prefix
path = '/minerva_dataset/folder'
params = {
'userId': self._user['_id']
}
response = self.request(path=path, method='POST', params=params, user=self._user)
self.assertStatusOk(response)
folder = response.json['folder']
# create the item
params = {
'name': 'bobby',
'folderId': folder['_id']
}
response = self.request(path='/item', method='POST', params=params, user=self._user)
self.assertStatusOk(response)
itemId = response.json['_id']
# create a s3 dataset from the item
prefix = '/CMIP5/CommonGrid/hadcm3/rcp45/mon/r1i1p1/pr/'
bucket = 'nasanex'
params = {
'name': 'nasanex',
'bucket': bucket,
'prefix': prefix,
'accessKeyId': '',
'secret': '',
'service': '',
'readOnly': True
}
path = '/minerva_dataset_s3/%s/dataset' % str(itemId)
response = self.request(
path=path,
method='POST',
user=self._user,
params=params
)
self.assertStatusOk(response)
import_folder_id = response.json['folderId']
# Wait for import to occur
time.sleep(1)
path = '/item'
params = {
'folderId': import_folder_id
}
response = self.request(path=path, method='GET', user=self._user,
params=params)
self.assertStatusOk(response)
self.assertEqual(len(response.json), 1, 'Excepting only a single item')
item_id = str(response.json[0]['_id'])
# Now list the files
path = '/item/%s/files' % item_id
response = self.request(path=path, method='GET', user=self._user)
self.assertStatusOk(response)
self.assertEqual(len(response.json), 1, 'Excepting only a single file')
self._dataset_file_id = response.json[0]['_id']
