def _importAnalysis(self): """Setup and import analyses for bsve tests.""" if self._import_done: return path = "/minerva_analysis/folder" response = self.request(path=path, method="POST", user=self._user) self.assertStatusOk(response) analyses_folder = response.json["folder"] # import the bsve analysis client = GirderClient("localhost", girder_port) client.authenticate("minervauser", "password") bsve_analysis_path = os.path.abspath( os.path.join(os.path.dirname(os.path.realpath(__file__)), "../analyses/bsve") ) import_analyses.import_analyses(client, bsve_analysis_path) path = "/item" params = {"folderId": analyses_folder["_id"]} response = self.request(path=path, method="GET", params=params, user=self._user) self.assertStatusOk(response) self.assertEquals(len(response.json), 2, "Expecting only one analysis") for analysis in response.json: if analysis["name"] == "bsve search": search_analysis = analysis elif analysis["name"] == "MMWR data import": soda_analysis = analysis else: self.fail('Unexpected analysis found "%s".' % analysis["name"]) expected_meta = { u"minerva": { u"analysis_type": u"bsve_search", u"analysis_name": u"bsve search", u"analysis_id": search_analysis["_id"], } } self.assertEquals(search_analysis["meta"], expected_meta, "Unexpected value for search meta data") expected_meta = { u"minerva": { u"analysis_type": u"mmwr_import_data", u"analysis_name": u"MMWR data import", u"analysis_id": soda_analysis["_id"], } } self.assertEquals(soda_analysis["meta"], expected_meta, "Unexpected value for soda meta data") # create the dataset folder path = "/minerva_dataset/folder" params = {"userId": self._user["_id"]} response = self.request(path=path, method="POST", params=params, user=self._user) self.assertStatusOk(response) self._importDone = True
def testBsveSearchAnalysis(self): # create the analysis folder path = '/minerva_analysis/folder' response = self.request(path=path, method='POST', user=self._user) self.assertStatusOk(response) analyses_folder = response.json['folder'] # import the bsve analysis client = GirderClient('localhost', girder_port) client.authenticate('minervauser', 'password') bsve_analysis_path = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../analyses/bsve')) import_analyses.import_analyses(client, bsve_analysis_path) path = '/item' params = { 'folderId': analyses_folder['_id'] } response = self.request(path=path, method='GET', params=params, user=self._user) self.assertStatusOk(response) self.assertEquals(len(response.json), 1, 'Expecting only one analysis') analysis = response.json[0] self.assertEquals(analysis['name'], 'bsve search', 'Expecting analysis name to be "bsve search"') expected_meta = { u'minerva': { u'analysis_type': u'bsve_search', u'analysis_name': u'bsve search', u'analysis_id': analysis['_id'] } } self.assertEquals(analysis['meta'], expected_meta, 'Unexpected value for meta data') # create the dataset folder path = '/minerva_dataset/folder' params = { 'userId': self._user['_id'], } response = self.request(path=path, method='POST', params=params, user=self._user) self.assertStatusOk(response) # mock the calls to bsve search @urlmatch(netloc=r'(.*\.)?beta-search.bsvecosystem.net(.*)$') def bsve_mock(url, request): if url.path.split('/')[-1] == 'request': return httmock.response(200, '12345') else: pluginTestDir = os.path.dirname(os.path.realpath(__file__)) filepath = os.path.join(pluginTestDir, 'data', 'bsve_search.json') with open(filepath) as bsve_search_file: content = { 'status': 1, 'results': json.load(bsve_search_file) } headers = { 'content-length': len(content), 'content-type': 'application/json' } return httmock.response(200, content, headers, request=request) with HTTMock(bsve_mock): response = self.request( path='/minerva_analysis/bsve_search', method='POST', params={ 'datasetName': 'test dataset', 'bsveSearchParams': '{}' }, user=self._user ) # wait for the async job to complete searchResultsFinished = False count = 0 while not searchResultsFinished and count < 5: # get the dataset and check if it has been updated path = '/minerva_dataset/%s/dataset' % str(response.json['dataset_id']) response = self.request( path=path, method='GET', user=self._user ) dataset = response.json if 'json_row' in dataset: searchResultsFinished = True else: time.sleep(2) count += 1 # ensure the first row of results was added to the dataset self.assertTrue('json_row' in dataset, 'json_row expected in dataset') self.assertTrue('data' in dataset['json_row'], 'data should be in json_row') self.assertTrue('Longitude' in dataset['json_row']['data'], 'data.Longitude should be in json_row') # ensure that we can map the Lat/Long to geojson, as this json has # unicode values for Lat/Long # update the minerva metadata with coordinate mapping metadata = {'minerva': dataset} metadata['minerva']['mapper'] = { "latitudeKeypath": "data.Latitude", "longitudeKeypath": "data.Longitude" } path = '/item/{}/metadata'.format(dataset['dataset_id']) response = self.request( path=path, method='PUT', user=self._user, body=json.dumps(metadata), type='application/json' ) metadata = response.json # create geojson in the dataset path = '/minerva_dataset/{}/geojson'.format(dataset['dataset_id']) response = self.request( path=path, method='POST', user=self._user, ) self.assertHasKeys(response.json, ['geojson_file'])
def _importAnalysis(self): """Setup and import analyses for bsve tests.""" if self._import_done: return path = '/minerva_analysis/folder' response = self.request(path=path, method='POST', user=self._user) self.assertStatusOk(response) analyses_folder = response.json['folder'] # import the bsve analysis client = GirderClient('localhost', girder_port) client.authenticate('minervauser', 'password') bsve_analysis_path = os.path.abspath( os.path.join(os.path.dirname(os.path.realpath(__file__)), '../analyses/bsve')) import_analyses.import_analyses(client, bsve_analysis_path) path = '/item' params = {'folderId': analyses_folder['_id']} response = self.request(path=path, method='GET', params=params, user=self._user) self.assertStatusOk(response) self.assertEquals(len(response.json), 2, 'Expecting only one analysis') for analysis in response.json: if analysis['name'] == 'bsve search': search_analysis = analysis elif analysis['name'] == 'MMWR data import': soda_analysis = analysis else: self.fail('Unexpected analysis found "%s".' % analysis['name']) expected_meta = { u'minerva': { u'analysis_type': u'bsve_search', u'analysis_name': u'bsve search', u'analysis_id': search_analysis['_id'] } } self.assertEquals(search_analysis['meta'], expected_meta, 'Unexpected value for search meta data') expected_meta = { u'minerva': { u'analysis_type': u'mmwr_import_data', u'analysis_name': u'MMWR data import', u'analysis_id': soda_analysis['_id'] } } self.assertEquals(soda_analysis['meta'], expected_meta, 'Unexpected value for soda meta data') # create the dataset folder path = '/minerva_dataset/folder' params = { 'userId': self._user['_id'], } response = self.request(path=path, method='POST', params=params, user=self._user) self.assertStatusOk(response) self._importDone = True
def testImportAnalyses(self): """ Test importing a romanesco analysis """ client = GirderClient('localhost', girder_port) client.authenticate(self._username, self._password) path = os.path.dirname(os.path.realpath(__file__)) analyses_path = os.path.join(path, 'analyses') import_analyses.import_analyses(client, analyses_path) # Get the analysis folder path = '/minerva_analysis/folder' response = self.request(path=path, method='GET', params={}, user=self._user) self.assertStatusOk(response) analyses_folder = response.json['folder'] path = '/item' params = { 'folderId': analyses_folder['_id'] } response = self.request(path=path, method='GET', params=params, user=self._user) self.assertStatusOk(response) self.assertEquals(len(response.json), 2, 'Expecting two analyses') analysis = response.json[0] self.assertEquals(analysis['name'], 'add', 'Expecting analysis one name to be "add"') expected_meta = { u'minerva': { u'analysis_type': u'add', u'analysis_name': u'add', u'analysis_id': analysis['_id'] }, u'analysis': { u'inputs': [{ u'default': { u'data': u'0', u'format': u'json' }, u'type': u'number', u'name': u'a', u'format': u'number' }, { u'type': u'number', u'name': u'b', u'format': u'number' }], u'script': u'c = a + b', u'mode': u'python', u'outputs': [{ u'type': u'number', u'name': u'c', u'format': u'number' }], u'name': u'add' } } self.assertEquals(analysis['meta'], expected_meta, 'Unexpected value for meta data') analysis = response.json[1] self.assertEquals(analysis['name'], 'local', 'Expecting analysis two name to be "local"') expected_meta = { u'minerva': { u'analysis_type': u'local type', u'analysis_name': u'local', u'analysis_id': analysis['_id'] } } self.assertEquals(analysis['meta'], expected_meta, 'Unexpected value for meta data')
def setUp(self): """ Set up the test case with a user """ super(ContourAnalysesTestCase, self).setUp() self._username = '******' self._password = '******' self._user = self.model('user').createUser( self._username, self._password, 'minerva', 'user', '*****@*****.**') # Import the analyses self._client = GirderClient('localhost', girder_port) self._client.authenticate(self._username, self._password) path = os.path.dirname(os.path.realpath(__file__)) analyses_path = os.path.join(path, '../analyses/NEX/') import_analyses.import_analyses(self._client, analyses_path) # Get the analysis folder path = '/minerva_analysis/folder' response = self.request(path=path, method='GET', params={}, user=self._user) self.assertStatusOk(response) analyses_folder = response.json['folder'] path = '/item' params = { 'folderId': analyses_folder['_id'] } response = self.request(path=path, method='GET', params=params, user=self._user) self.assertStatusOk(response) # Find the contour analysis for analysis in response.json: if analysis['name'] == 'contour': self._analysis = analysis # Now import an S3 prefix path = '/minerva_dataset/folder' params = { 'userId': self._user['_id'] } response = self.request(path=path, method='POST', params=params, user=self._user) self.assertStatusOk(response) folder = response.json['folder'] # create the item params = { 'name': 'bobby', 'folderId': folder['_id'] } response = self.request(path='/item', method='POST', params=params, user=self._user) self.assertStatusOk(response) itemId = response.json['_id'] # create a s3 dataset from the item prefix = '/CMIP5/CommonGrid/hadcm3/rcp45/mon/r1i1p1/pr/' bucket = 'nasanex' params = { 'name': 'nasanex', 'bucket': bucket, 'prefix': prefix, 'accessKeyId': '', 'secret': '', 'service': '', 'readOnly': True } path = '/minerva_dataset_s3/%s/dataset' % str(itemId) response = self.request( path=path, method='POST', user=self._user, params=params ) self.assertStatusOk(response) import_folder_id = response.json['folderId'] # Wait for import to occur time.sleep(1) path = '/item' params = { 'folderId': import_folder_id } response = self.request(path=path, method='GET', user=self._user, params=params) self.assertStatusOk(response) self.assertEqual(len(response.json), 1, 'Excepting only a single item') item_id = str(response.json[0]['_id']) # Now list the files path = '/item/%s/files' % item_id response = self.request(path=path, method='GET', user=self._user) self.assertStatusOk(response) self.assertEqual(len(response.json), 1, 'Excepting only a single file') self._dataset_file_id = response.json[0]['_id']