class NewtIntegrationTest(BaseIntegrationTest):
def __init__(self, name, girder_url, girder_user, girder_password, machine, job_timeout=60 * 5):
super(NewtIntegrationTest, self).__init__(name, girder_url, girder_user, girder_password, job_timeout)
self._cluster_id = None
self._machine = machine
def setUp(self):
# First authenticate with NEWT
self._session = Session()
r = self._session.post(
"https://newt.nersc.gov/newt/auth", {"username": self._girder_user, "password": self._girder_password}
)
self.assertEqual(r.status_code, 200)
print r.json()
self._newt_session_id = r.json()["newt_sessionid"]
# Now authenticate with Girder using the session id
url = "%s/api/v1/newt/authenticate/%s" % (self._girder_url, self._newt_session_id)
r = self._session.put(url)
self.assertEqual(r.status_code, 200)
url = "%s/api/v1/newt/authenticate/%s" % (self._girder_url, self._newt_session_id)
r = self._session.put(url)
self.assertEqual(r.status_code, 200)
url = "%s/api/v1" % self._girder_url
self._client = GirderClient(apiUrl=url)
self._client.token = self._session.cookies["girderToken"]
user = self._client.get("user/me")
self._user_id = user["_id"]
r = self._client.listFolder(self._user_id, "user", name="Private")
r = list(r)
self.assertEqual(len(r), 1)
self._private_folder_id = r[0]["_id"]
def tearDown(self):
super(NewtIntegrationTest, self).tearDown()
if self._cluster_id:
try:
url = "clusters/%s" % self._cluster_id
self._client.delete(url)
except Exception:
traceback.print_exc()
def create_cluster(self):
body = {"config": {"host": self._machine}, "name": "NewtIntegrationTest", "type": "newt"}
r = self._client.post("clusters", data=json.dumps(body))
self._cluster_id = r["_id"]
# Now test the connection
r = self._client.put("clusters/%s/start" % self._cluster_id)
sleeps = 0
while True:
time.sleep(1)
r = self._client.get("clusters/%s/status" % self._cluster_id)
if r["status"] == "running":
break
elif r["status"] == "error":
r = self._client.get("clusters/%s/log" % self._cluster_id)
self.fail(str(r))
if sleeps > 9:
self.fail("Cluster never moved into running state")
sleeps += 1
def assert_output(self):
r = self._client.listItem(self._output_folder_id)
self.assertEqual(len(r), 4)
stdout_item = None
for i in r:
if i["name"].startswith("CumulusIntegrationTestJob-%s.o" % self._job_id):
stdout_item = i
break
self.assertIsNotNone(stdout_item)
r = self._client.get("item/%s/files" % i["_id"])
self.assertEqual(len(r), 1)
url = "%s/api/v1/file/%s/download" % (self._girder_url, r[0]["_id"])
r = self._session.get(url)
self.assertEqual(r.content, self._data)
def test(self):
try:
self.create_cluster()
self.create_script()
self.create_input()
self.create_output_folder()
self.create_job()
self.submit_job(timeout=self._job_timeout)
self.assert_output()
except HttpError as error:
self.fail(error.responseText)
def __init__(self, apiKey, objectStore):
"""initialization function to create a GirderCli instance, will attempt
to authenticate with the designated Girder instance.
"""
GirderClient.__init__(self, apiUrl="https://data.kitware.com/api/v1")
self.objectStore = objectStore
self.authenticate(apiKey=apiKey)
def __init__(self, girder_url, newt_sessionid):
'''
'''
self._client = None
self._cluster_id = None
self._girder_url = girder_url
self._input_folder_id = None
self._job_folder_id = None
self._job_id = None
self._output_folder_id = None
self._private_folder_id = None
self._script_id = None
self._session = requests.Session()
# Authenticate with Girder using the newt session id
url = '%s/api/v1/newt/authenticate/%s' % \
(self._girder_url, newt_sessionid)
r = self._session.put(url)
if r.status_code != 200:
raise HttpError(r.status_code, r.text, r.url, r.request.method)
# Instantiate Girder client
url = '%s/api/v1' % self._girder_url
self._client = GirderClient(apiUrl=url)
self._client.token = self._session.cookies['girderToken']
user = self._client.get('user/me')
#print 'user', user
user_id = user['_id']
r = self._client.listFolder(user_id, 'user', name='Private')
if len(r) != 1:
raise Exception('Wrong number of users; should be 1 got %s' %
len(r))
self._private_folder_id = r[0]['_id']
print 'private_folder_id', self._private_folder_id
def download_path(cluster_connection,
girder_token,
parent,
path,
assetstore_url,
assetstore_id,
upload=False,
include=None,
exclude=None):
"""
Download a given path on a cluster into an assetstore.
:params cluster_connection: The cluster connection to access the cluster.
:params girder_token: The Girder token to use to access Girder.
:params parent: The target folder to import the path into.
:params path: The path on the cluster to download.
:params assetstore_url: The url for the assetstore to use for the import.
:params assetstore_id: The id of the asseststore to import into.
:params upload: Indicate if the import should upload the file data or just
the metadata, the default is False.
:params include: List of include regexs
:params exclude: List of exclude regexs,
"""
girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl)
girder_client.token = girder_token
_import_path(cluster_connection,
girder_client,
parent,
path,
assetstore_url,
assetstore_id,
upload=upload,
include=include,
exclude=exclude)
class GirderClient(object):
def __init__(self):
token_obj = get_oc_token_obj()
url = token_obj.get('apiUrl')
api_key = token_obj.get('apiKey')
url = os.environ.get('OC_API_URL', url)
internal_url = os.environ.get('OC_INTERNAL_API_URL', url)
api_key = os.environ.get('OC_API_KEY', api_key)
token = os.environ.get('GIRDER_TOKEN')
self.client = None
self.url = url
self.internal_url = internal_url
if internal_url is not None:
self.client = GC(apiUrl=internal_url)
if api_key is not None:
self.client.authenticate(apiKey=api_key)
elif token is not None:
self.client.token = token
def __getattr__(self, name):
return getattr(self.client, name)
def __init__(self,
username,
password,
dryrun,
blacklist,
host=None,
port=None,
apiRoot=None,
scheme=None):
"""initialization function to create a GirderCli instance, will attempt
to authenticate with the designated Girder instance.
:param username: username to authenticate to Girder instance.
:param password: password to authenticate to Girder instance, leave
this blank to be prompted.
:param dryrun: boolean indicating whether to run the command or just
perform a dryrun showing which files and folders will be uploaded.
:param blacklist: list of filenames which will be ignored by upload.
:param host: host used to connect to Girder instance,
defaults to 'localhost'
:param port: port used to connect to Girder instance,
default is 80 for http: and 443 for https:
:param apiRoot: The path on the server corresponding to the root of the
Girder REST API. If None is passed, assumes '/api/v1'.
:param scheme: scheme used to connect to Girder instance,
defaults to 'http'
"""
GirderClient.__init__(self,
host=host,
port=port,
apiRoot=apiRoot,
scheme=scheme,
dryrun=dryrun,
blacklist=blacklist)
interactive = password is None
self.authenticate(username, password, interactive=interactive)
def wait_for_jobs(client: GirderClient, max_wait_timeout=30, expected_status=JobStatus.SUCCESS):
"""Wait for all worker jobs to complete"""
start_time = time.time()
incompleteJobs = []
while True and (time.time() - start_time < max_wait_timeout):
incompleteJobs = client.get(
'job',
parameters={
# https://github.com/girder/girder/blob/master/plugins/jobs/girder_jobs/constants.py
# https://github.com/girder/girder_worker/blob/master/girder_worker/girder_plugin/status.py
'statuses': json.dumps([0, 1, 2, 820, 821, 822, 823, 824]),
},
)
if len(incompleteJobs) == 0:
break
time.sleep(1)
if len(incompleteJobs) > 0:
raise Exception("Jobs were still running after timeout")
# Verify that all jobs succeeded
time.sleep(1)
lastJob = client.get(
'job',
parameters={
'limit': 1,
},
)
if len(lastJob) > 0 and lastJob[0]['status'] != expected_status:
raise Exception(f"Some jobs did not meet their expected status: {expected_status}")
def __init__(self, apiKey, objectStore):
"""initialization function to create a GirderCli instance, will attempt
to authenticate with the designated Girder instance.
"""
GirderClient.__init__(self, apiUrl="https://data.kitware.com/api/v1")
self.objectStore = objectStore
self.authenticate(apiKey=apiKey)
def main():
parser = argparse.ArgumentParser(
description='Import analyses into minerva')
parser.add_argument('--username', required=False, default=None)
parser.add_argument('--password', required=False, default=None)
parser.add_argument('--scheme', required=False, default='http')
parser.add_argument('--host', required=False, default='localhost')
parser.add_argument('--port', required=False, default='8080')
parser.add_argument('--api-root',
required=False,
default='/api/v1',
help='path to the Girder REST API')
parser.add_argument('--path',
required=True,
help='the path to import the analyses from')
config = parser.parse_args()
client = GirderClient(host=config.host,
port=config.port,
apiRoot=config.api_root,
scheme=config.scheme)
client.authenticate(config.username, config.password)
import_analyses(client, config.path)
def __init__(self,
username,
password,
dryrun,
blacklist,
host=None,
port=None,
apiRoot=None,
scheme=None,
apiUrl=None,
apiKey=None):
"""initialization function to create a GirderCli instance, will attempt
to authenticate with the designated Girder instance. Aside from username
and password, all other kwargs are passed directly through to the
:py:class:`girder_client.GirderClient` base class constructor.
:param username: username to authenticate to Girder instance.
:param password: password to authenticate to Girder instance, leave
this blank to be prompted.
"""
GirderClient.__init__(self,
host=host,
port=port,
apiRoot=apiRoot,
scheme=scheme,
dryrun=dryrun,
blacklist=blacklist,
apiUrl=apiUrl)
interactive = password is None
if apiKey:
self.authenticate(apiKey=apiKey)
elif username:
self.authenticate(username, password, interactive=interactive)
def _update_girder(taskflow, body):
taskflow = to_taskflow(taskflow)
taskflow_id = taskflow['id']
girder_token = taskflow['girder_token']
girder_api_url = taskflow['girder_api_url']
client = GirderClient(apiUrl=girder_api_url)
client.token = girder_token
client = _create_girder_client(girder_api_url, girder_token)
# If this is a retry then we have already create a task get it from
# the current tasks headers.
if body['retries'] > 0:
taskflow_task_id = \
current_task.request.headers[TASKFLOW_TASK_ID_HEADER]
# Celery always fires the postrun handler with a state of SUCCESS
# for retries. So we need to save the retries here so we can
# determine in the postrun handler if the task is really complete.
current_task.request.headers[TASKFLOW_RETRY_HEADER] \
= body['retries']
else:
# This is a new task so create a taskflow task instance
body = {
'celeryTaskId': body['id'],
'name': body['task']
}
url = 'taskflows/%s/tasks' % taskflow_id
r = client.post(url, data=json.dumps(body))
taskflow_task_id = r['_id']
return taskflow, taskflow_task_id
def setUp(self):
# First authenticate with NEWT
self._session = Session()
r = self._session.post('https://newt.nersc.gov/newt/auth', {
'username': self._girder_user,
'password': self._girder_password
})
self.assertEqual(r.status_code, 200)
print r.json()
self._newt_session_id = r.json()['newt_sessionid']
# Now authenticate with Girder using the session id
url = '%s/api/v1/newt/authenticate/%s' % (self._girder_url,
self._newt_session_id)
r = self._session.put(url)
self.assertEqual(r.status_code, 200)
url = '%s/api/v1/newt/authenticate/%s' % (self._girder_url,
self._newt_session_id)
r = self._session.put(url)
self.assertEqual(r.status_code, 200)
url = '%s/api/v1' % self._girder_url
self._client = GirderClient(apiUrl=url)
self._client.token = self._session.cookies['girderToken']
user = self._client.get('user/me')
self._user_id = user['_id']
r = self._client.listFolder(self._user_id, 'user', name='Private')
r = list(r)
self.assertEqual(len(r), 1)
self._private_folder_id = r[0]['_id']
def _update_girder(taskflow, body):
taskflow = to_taskflow(taskflow)
taskflow_id = taskflow['id']
girder_token = taskflow['girder_token']
girder_api_url = taskflow['girder_api_url']
client = GirderClient(apiUrl=girder_api_url)
client.token = girder_token
client = _create_girder_client(girder_api_url, girder_token)
# If this is a retry then we have already create a task get it from
# the current tasks headers.
if body['retries'] > 0:
taskflow_task_id = \
current_task.request.headers[TASKFLOW_TASK_ID_HEADER]
# Celery always fires the postrun handler with a state of SUCCESS
# for retries. So we need to save the retries here so we can
# determine in the postrun handler if the task is really complete.
current_task.request.headers[TASKFLOW_RETRY_HEADER] \
= body['retries']
else:
# This is a new task so create a taskflow task instance
body = {'celeryTaskId': body['id'], 'name': body['task']}
url = 'taskflows/%s/tasks' % taskflow_id
r = client.post(url, data=json.dumps(body))
taskflow_task_id = r['_id']
return taskflow, taskflow_task_id
def test_sanity_checks():
gc = GirderClient(apiUrl=source_api_root)
resp = gc.get('/', jsonResp=False)
assert resp.status_code == 200
assert resp.headers['Content-Type'] == 'text/html;charset=utf-8'
resp = gc.get('system/check', jsonResp=False)
assert resp.status_code == 200
def test_upgrade_pipelines(admin_client: GirderClient):
cnf = admin_client.get('dive_configuration/pipelines')
if 'detector' not in cnf:
admin_client.post(
'dive_configuration/upgrade_pipelines',
data=json.dumps(tasks.UPGRADE_JOB_DEFAULT_URLS),
)
wait_for_jobs(admin_client, 1000)
def _taskflow_task_finished(taskflow, taskflow_task_id):
girder_token = taskflow['girder_token']
girder_api_url = taskflow['girder_api_url']
client = GirderClient(apiUrl=girder_api_url)
client.token = girder_token
url = 'taskflows/%s/tasks/%s/finished' % (taskflow.id, taskflow_task_id)
return client.put(url)
def _taskflow_task_finished(taskflow, taskflow_task_id):
girder_token = taskflow['girder_token']
girder_api_url = taskflow['girder_api_url']
client = GirderClient(apiUrl=girder_api_url)
client.token = girder_token
url = 'taskflows/%s/tasks/%s/finished' % (taskflow.id, taskflow_task_id)
return client.put(url)
def _importAnalysis(self):
"""Setup and import analyses for bsve tests."""
if self._import_done:
return
path = "/minerva_analysis/folder"
response = self.request(path=path, method="POST", user=self._user)
self.assertStatusOk(response)
analyses_folder = response.json["folder"]
# import the bsve analysis
client = GirderClient("localhost", girder_port)
client.authenticate("minervauser", "password")
bsve_analysis_path = os.path.abspath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), "../analyses/bsve")
)
import_analyses.import_analyses(client, bsve_analysis_path)
path = "/item"
params = {"folderId": analyses_folder["_id"]}
response = self.request(path=path, method="GET", params=params, user=self._user)
self.assertStatusOk(response)
self.assertEquals(len(response.json), 2, "Expecting only one analysis")
for analysis in response.json:
if analysis["name"] == "bsve search":
search_analysis = analysis
elif analysis["name"] == "MMWR data import":
soda_analysis = analysis
else:
self.fail('Unexpected analysis found "%s".' % analysis["name"])
expected_meta = {
u"minerva": {
u"analysis_type": u"bsve_search",
u"analysis_name": u"bsve search",
u"analysis_id": search_analysis["_id"],
}
}
self.assertEquals(search_analysis["meta"], expected_meta, "Unexpected value for search meta data")
expected_meta = {
u"minerva": {
u"analysis_type": u"mmwr_import_data",
u"analysis_name": u"MMWR data import",
u"analysis_id": soda_analysis["_id"],
}
}
self.assertEquals(soda_analysis["meta"], expected_meta, "Unexpected value for soda meta data")
# create the dataset folder
path = "/minerva_dataset/folder"
params = {"userId": self._user["_id"]}
response = self.request(path=path, method="POST", params=params, user=self._user)
self.assertStatusOk(response)
self._importDone = True
def setUp(self):
url = '%s/api/v1' % self._girder_url
self._client = GirderClient(apiUrl=url)
self._client.authenticate(self._girder_user, self._girder_password)
user = self._client.get('user/me')
self._user_id = user['_id']
r = list(self._client.listFolder(self._user_id, 'user',
name='Private'))
self.assertEqual(len(r), 1)
self._private_folder_id = r[0]['_id']
def main(ctx, api_key, api_url):
"""Openchemistry Client
The client can be used to fetch molecules, add molecules, etc.
"""
gc = GirderClient(apiUrl=api_url)
if api_key is not None:
gc.authenticate(apiKey=api_key)
ctx.obj = gc
def import_calc(config):
try:
target_port = None
if config.port:
target_port = config.port
target_scheme = None
if config.scheme:
target_scheme = config.scheme
target_apiroot = None
if config.apiroot:
target_apiroot = config.apiroot
client = GirderClient(host=config.host, port=target_port,
scheme=target_scheme, apiRoot=target_apiroot)
client.authenticate(apiKey=config.apiKey)
me = client.get('/user/me')
if not me:
print('Error: Girder token invalid, please verify')
return
folderParams = {
'parentId': me['_id'],
'parentType': 'user',
'name': 'Private'
}
# Get the private folder id first
folder = next(client.listResource('folder', folderParams))
folder = next(client.listFolder(me['_id'], 'user', 'Private'))
for file_name in config.datafile:
print ('\nUploading ' + file_name)
file_id = {}
with open(file_name, 'r') as fp:
fileNameBase = os.path.basename(file_name)
size = os.path.getsize(file_name)
file_id = client.uploadFile(folder['_id'], fp, fileNameBase,
size, 'folder')
body = {
'fileId': file_id['_id']
}
if config.public:
body['public'] = True
mol = client.sendRestRequest('POST', 'molecules', data=json.dumps(body))
if mol and '_id' in mol:
config.moleculeId = mol['_id']
print('Molecule ID: ' + mol['_id'])
else:
print(mol)
except HttpError as error:
print(error.responseText, file=sys.stderr)
def test_user_creation(admin_client: GirderClient, user: dict):
try:
admin_client.createUser(
user['login'],
user['email'],
user['firstName'],
user['lastName'],
user['password'],
)
except HttpError as err:
if err.response.json()['message'] != 'That login is already registered.':
raise err
def gc_init(api_url='https://girder.hub.yt/api/v1', api_key=None):
from girder_client import GirderClient
if api_key is None:
try:
api_key = os.environ['GIRDER_API_KEY']
except KeyError as err:
msg = 'please define GIRDER_API_KEY environment variable\n'
msg += ' or pass api_key kwargs.'
raise KeyError(msg)
gc = GirderClient(apiUrl=api_url)
gc.authenticate(apiKey=api_key)
return gc
def _ingest(project, composite, dir, channel_map, api_url, api_key):
if dir[-1] != '/':
dir += '/'
gc = GirderClient(apiUrl=api_url)
gc.authenticate(apiKey=api_key)
channel_map = json.load(channel_map)
channel_map = {channel.upper():element.lower() for (channel,element) in channel_map.items()}
experiments = _ingest_runs(gc, project, composite, dir)
#(run_ids, runs) = _ingest_runs(gc, project, composite, dir)
_ingest_samples(gc, project, composite, dir, experiments, channel_map)
def upload_file(cluster, girder_token, file, path):
"""
Upload a file to a cluster
:param cluster: The cluster to upload to.
:param girder_tokebn: The Grider token for Girder access.
:param file: The Girder file object.
:param path: The path on the cluster to upload to.
"""
girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl)
girder_client.token = girder_token
with get_connection(girder_token, cluster) as conn:
conn.makedirs(os.path.dirname(path))
_upload_file(conn, girder_client, file, path)
def upload_file(cluster, girder_token, file, path):
"""
Upload a file to a cluster
:param cluster: The cluster to upload to.
:param girder_tokebn: The Grider token for Girder access.
:param file: The Girder file object.
:param path: The path on the cluster to upload to.
"""
girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl)
girder_client.token = girder_token
with get_connection(girder_token, cluster) as conn:
conn.makedirs(os.path.dirname(path))
_upload_file(conn, girder_client, file, path)
def setUp(self):
# First authenticate with NEWT
self._session = Session()
r = self._session.post('https://newt.nersc.gov/newt/auth',
{
'username': self._girder_user,
'password': self._girder_password})
self.assertEqual(r.status_code, 200)
print r.json()
self._newt_session_id = r.json()['newt_sessionid']
# Now authenticate with Girder using the session id
url = '%s/api/v1/newt/authenticate/%s' % (self._girder_url, self._newt_session_id)
r = self._session.put(url)
self.assertEqual(r.status_code, 200)
url = '%s/api/v1/newt/authenticate/%s' % (self._girder_url, self._newt_session_id)
r = self._session.put(url)
self.assertEqual(r.status_code, 200)
url = '%s/api/v1' % self._girder_url
self._client = GirderClient(apiUrl=url)
self._client.token = self._session.cookies['girderToken']
user = self._client.get('user/me')
self._user_id = user['_id']
r = self._client.listFolder(self._user_id, 'user', name='Private')
self.assertEqual(len(r), 1)
self._private_folder_id = r[0]['_id']
def testClientMetadataExtractor(testData, user):
item = Item().load(testData['item']['_id'], user=user)
assert item['name'] == testData['name']
del item['meta']
item = Item().save(item)
assert 'meta' not in item
client = GirderClient('localhost', int(os.environ['GIRDER_PORT']))
client.authenticate(user['login'], 'password')
extractor = ClientMetadataExtractor(client, testData['path'], testData['item']['_id'])
extractor.extractMetadata()
item = Item().load(testData['item']['_id'], user=user)
assert item['name'] == testData['name']
assert 'meta' in item
assert item['meta']['MIME type'] == testData['mimeType']
def __init__(self, username, password, dryrun, blacklist,
host=None, port=None, apiRoot=None, scheme=None, apiUrl=None):
"""initialization function to create a GirderCli instance, will attempt
to authenticate with the designated Girder instance. Aside from username
and password, all other kwargs are passed directly through to the
:py:class:`girder_client.GirderClient` base class constructor.
:param username: username to authenticate to Girder instance.
:param password: password to authenticate to Girder instance, leave
this blank to be prompted.
"""
GirderClient.__init__(self, host=host, port=port,
apiRoot=apiRoot, scheme=scheme, dryrun=dryrun,
blacklist=blacklist, apiUrl=apiUrl)
interactive = password is None
self.authenticate(username, password, interactive=interactive)
def setUp(self):
# First authenticate with NEWT
self._session = Session()
r = self._session.post(
"https://newt.nersc.gov/newt/auth", {"username": self._girder_user, "password": self._girder_password}
)
self.assertEqual(r.status_code, 200)
print r.json()
self._newt_session_id = r.json()["newt_sessionid"]
# Now authenticate with Girder using the session id
url = "%s/api/v1/newt/authenticate/%s" % (self._girder_url, self._newt_session_id)
r = self._session.put(url)
self.assertEqual(r.status_code, 200)
url = "%s/api/v1/newt/authenticate/%s" % (self._girder_url, self._newt_session_id)
r = self._session.put(url)
self.assertEqual(r.status_code, 200)
url = "%s/api/v1" % self._girder_url
self._client = GirderClient(apiUrl=url)
self._client.token = self._session.cookies["girderToken"]
user = self._client.get("user/me")
self._user_id = user["_id"]
r = self._client.listFolder(self._user_id, "user", name="Private")
r = list(r)
self.assertEqual(len(r), 1)
self._private_folder_id = r[0]["_id"]
def init_girder(api_key=None, api_url='https://girder.hub.yt/api/v1'):
"""Initialize girder client, rely on environment variable:
GIRDER_API_KEY, connect to yt Hub.
Args:
api_key (str, optional): use GIRDER_API_KEY env. var. by default
api_url (str, optional): use yt Hub v1 by default
Return:
GirderClient: initialized girder client
"""
from girder_client import GirderClient
if api_key is None:
import os
api_key = os.environ['GIRDER_API_KEY']
gc = GirderClient(apiUrl=api_url)
gc.authenticate(apiKey=api_key)
return gc
def main():
parser = argparse.ArgumentParser(description='Import analyses into minerva')
parser.add_argument('--username', required=False, default=None)
parser.add_argument('--password', required=False, default=None)
parser.add_argument('--scheme', required=False, default='http')
parser.add_argument('--host', required=False, default='localhost')
parser.add_argument('--port', required=False, default='8080')
parser.add_argument('--api-root', required=False, default='/api/v1',
help='path to the Girder REST API')
parser.add_argument('--path', required=True, help='the path to import the analyses from')
config = parser.parse_args()
client = GirderClient(host=config.host, port=config.port,
apiRoot=config.api_root, scheme=config.scheme)
client.authenticate(config.username, config.password)
import_analyses(client, config.path)
def testClientMetadataExtractor(self):
item = self.model('item').load(self.item['_id'], user=self.user)
self.assertEqual(item['name'], self.name)
self.assertNotHasKeys(item, ['meta'])
clientPath = os.path.join(ROOT_DIR, 'clients', 'python')
sys.path.insert(0, clientPath)
from girder_client import GirderClient
client = GirderClient('localhost', int(os.environ['GIRDER_PORT']))
client.authenticate(self.user['login'], self.password)
extractor = ClientMetadataExtractor(client, self.path, self.item['_id'])
extractor.extractMetadata()
sys.path.remove(clientPath)
item = self.model('item').load(self.item['_id'], user=self.user)
self.assertEqual(item['name'], self.name)
self.assertHasKeys(item, ['meta'])
self.assertEqual(item['meta']['MIME type'], self.mimeType)
def testClientMetadataExtractor(self):
item = Item().load(self.item['_id'], user=self.user)
self.assertEqual(item['name'], self.name)
self.assertNotHasKeys(item, ['meta'])
clientPath = os.path.join(ROOT_DIR, 'clients', 'python')
sys.path.insert(0, clientPath)
from girder_client import GirderClient
client = GirderClient('localhost', int(os.environ['GIRDER_PORT']))
client.authenticate(self.user['login'], self.password)
extractor = ClientMetadataExtractor(client, self.path,
self.item['_id'])
extractor.extractMetadata()
sys.path.remove(clientPath)
item = Item().load(self.item['_id'], user=self.user)
self.assertEqual(item['name'], self.name)
self.assertHasKeys(item, ['meta'])
self.assertEqual(item['meta']['MIME type'], self.mimeType)
def upload_zipped_flat_media_files(
gc: GirderClient,
manager: JobManager,
folderId: str,
working_directory: Path,
create_subfolder=False,
):
"""Takes a flat folder of media files and/or annotation and generates a dataset from it"""
listOfFileNames = os.listdir(working_directory)
validation = gc.sendRestRequest('POST',
'/dive_dataset/validate_files',
json=listOfFileNames)
root_folderId = folderId
default_fps = gc.getFolder(root_folderId).get(
f"meta.{constants.FPSMarker}", -1)
if validation.get('ok', False):
manager.write(f"Annotations: {validation['annotations']}\n")
manager.write(f"Media: {validation['media']}\n")
dataset_type = validation['type']
manager.write(f"Type: {dataset_type}\n")
if create_subfolder != '':
sub_folder = gc.createFolder(
folderId,
create_subfolder,
reuseExisting=True,
)
root_folderId = str(sub_folder["_id"])
# Upload all resulting items back into the root folder
manager.updateStatus(JobStatus.PUSHING_OUTPUT)
# create a source folder to place the zipFile inside of
gc.upload(f'{working_directory}/*', root_folderId)
if dataset_type == constants.ImageSequenceType and default_fps == -1:
default_fps = 1
gc.addMetadataToFolder(
str(root_folderId),
{
constants.TypeMarker: dataset_type,
constants.FPSMarker: default_fps
},
)
# After uploading the default files we do a the postprocess for video conversion now
gc.sendRestRequest("POST",
f"/dive_rpc/postprocess/{str(root_folderId)}")
else:
manager.write(f"Message: {validation['message']}\n")
raise Exception("Could not Validate media Files")
def setUp(self):
url = '%s/api/v1' % self._girder_url
self._client = GirderClient(apiUrl=url)
self._client.authenticate(self._girder_user,
self._girder_password)
user = self._client.get('user/me')
self._user_id = user['_id']
r = list(self._client.listFolder(self._user_id, 'user', name='Private'))
self.assertEqual(len(r), 1)
self._private_folder_id = r[0]['_id']
def main(config):
client = GirderClient(apiUrl=config.girder_api_url)
client.authenticate(config.girder_user,
config.girder_password)
# Load any parameters
params = {}
if config.taskflow_start_params is not None:
with open(config.taskflow_start_params) as fp:
params = json.load(fp)
print params
try:
print ('Running %s taskflow ...' % config.taskflow_start_params)
taskflow_id = create_taskflow(
client, config.taskflow_class)
# Start the task flow
url = 'taskflows/%s/start' % (taskflow_id)
client.put(url, data=json.dumps(params))
# Wait for it to complete
wait_for_complete(client, taskflow_id)
except HttpError as ex:
print( ex.responseText)
def get_girder_client() -> GirderClient:
client = GirderClient(apiUrl="https://viame.kitware.com/api/v1")
apikey = os.environ.get('GIRDER_API_KEY', None)
if apikey:
client.authenticate(apiKey=apikey)
else:
client.authenticate(interactive=True)
return client
def download_path(cluster_connection, girder_token, parent, path,
assetstore_url, assetstore_id, upload=False, include=None,
exclude=None):
"""
Download a given path on a cluster into an assetstore.
:params cluster_connection: The cluster connection to access the cluster.
:params girder_token: The Girder token to use to access Girder.
:params parent: The target folder to import the path into.
:params path: The path on the cluster to download.
:params assetstore_url: The url for the assetstore to use for the import.
:params assetstore_id: The id of the asseststore to import into.
:params upload: Indicate if the import should upload the file data or just
the metadata, the default is False.
:params include: List of include regexs
:params exclude: List of exclude regexs,
"""
girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl)
girder_client.token = girder_token
_import_path(cluster_connection, girder_client, parent, path,
assetstore_url, assetstore_id, upload=upload, include=include,
exclude=exclude)
def __init__(self, username, password, dryrun, blacklist, host="localhost", port=8080, apiRoot=None, scheme="http"):
"""initialization function to create a GirderCli instance, will attempt
to authenticate with the designated Girder instance.
:param username: username to authenticate to Girder instance.
:param password: password to authenticate to Girder instance, leave
this blank to be prompted.
:param dryrun: boolean indicating whether to run the command or just
perform a dryrun showing which files and folders will be uploaded.
:param blacklist: list of filenames which will be ignored by upload.
:param host: host used to connect to Girder instance,
defaults to 'localhost'
:param port: port used to connect to Girder instance,
defaults to 8080
:param apiRoot: The path on the server corresponding to the root of the
Girder REST API. If None is passed, assumes '/api/v1'.
:param scheme: scheme used to connect to Girder instance,
defaults to 'http'; if passing 'https' port should likely be 443.
"""
GirderClient.__init__(
self, host=host, port=port, apiRoot=apiRoot, scheme=scheme, dryrun=dryrun, blacklist=blacklist
)
interactive = password is None
self.authenticate(username, password, interactive=interactive)
def test_extract_download(data):
fileId, filename = data
filepath = localDataRoot / str(filename)
if not filepath.exists():
gc = GirderClient(apiUrl=source_api_root)
gc.authenticate(apiKey=os.environ.get('GIRDER_API_KEY'))
gc.downloadFile(fileId, str(filepath))
with zipfile.ZipFile(filepath, 'r') as zipref:
zipref.extractall(localDataRoot)
def main(args=None):
parser = argparse.ArgumentParser(
description='Mount Girder filesystem assetstore.')
parser.add_argument('--api-url',
required=True,
default=None,
help='full URL to the RESTful API of Girder server')
parser.add_argument('--username', required=False, default=None)
parser.add_argument('--password', required=False, default=None)
parser.add_argument('--api-key', required=False, default=None)
parser.add_argument('--token', required=False, default=None)
parser.add_argument('-c',
default='remote',
choices=['remote', 'direct'],
help='command to run')
parser.add_argument('--foreground', dest='foreground', action='store_true')
parser.add_argument('--hostns', dest='hostns', action='store_true')
parser.add_argument('local_folder', help='path to local target folder')
parser.add_argument('remote_folder', help='Girder\'s folder id')
args = parser.parse_args()
gc = GirderClient(apiUrl=args.api_url)
if args.token:
gc.token = args.token
elif args.api_key:
gc.authenticate(apiKey=args.api_key)
elif args.username and args.password:
gc.authenticate(username=args.username, password=args.password)
else:
raise RuntimeError("You need to specify apiKey or user/pass")
if args.hostns:
targetns = os.path.join(os.environ.get('HOSTDIR', '/'),
'proc/1/ns/mnt')
with open(targetns) as fd:
setns(fd, CLONE_NEWNS)
if args.c == 'remote':
FUSE(RESTGirderFS(args.remote_folder, gc),
args.local_folder,
foreground=args.foreground,
ro=True,
allow_other=True)
elif args.c == 'direct':
FUSE(LocalGirderFS(args.remote_folder, gc),
args.local_folder,
foreground=args.foreground,
ro=True,
allow_other=True)
else:
print('No implementation for command %s' % args.c)
def main():
"""Create the folder hierarchy with metadata in a Girder instance."""
args = parser.parse_args()
g = GirderClient(host=args.host, port=args.port, scheme=args.scheme)
g.authenticate(args.username, args.password)
def create_folder_on_demand(parent_folder_id, folder_name):
existing_folders = list(
g.listFolder(parent_folder_id, name=folder_name))
if not len(existing_folders):
sought_folder = g.createFolder(parent_folder_id, name=folder_name)
else:
sought_folder = existing_folders[0]
return sought_folder
metadata_file = 'metadata.json'
with open(metadata_file) as json_file:
metadata = json.load(json_file)
parent_folder_id = args.parent_folder_id
for subject_id, subject_metadata in metadata.items():
subject_folder = create_folder_on_demand(parent_folder_id, subject_id)
for (scan_time, scan_date, scan_weight) in subject_metadata['scans']:
create_folder_on_demand(subject_folder['_id'], scan_time)
def upload_benchmark_results(benchmark_bin, api_key=None):
hostname = socket.gethostname().lower()
results_dir = os.path.join(benchmark_bin, 'BenchmarkResults', hostname)
if not os.path.exists(results_dir):
sys.stderr.write('Expected results directory does not exist: ' +
results_dir)
sys.exit(1)
from girder_client import GirderClient
gc = GirderClient(apiUrl='https://data.kitware.com/api/v1')
gc.authenticate(apiKey=api_key)
# ITK/PerformanceBenchmarkingResults
folder_id = '5af50c818d777f06857985e3'
hostname_folder = gc.loadOrCreateFolder(hostname, folder_id, 'folder')
gc.upload(os.path.join(results_dir, '*.json'),
hostname_folder['_id'],
leafFoldersAsItems=False,
reuseExisting=True)
def upload_benchmark_results(benchmark_bin, api_key=None):
hostname = socket.gethostname().lower()
results_dir = os.path.join(benchmark_bin, 'BenchmarkResults',
hostname)
if not os.path.exists(results_dir):
sys.stderr.write('Expected results directory does not exist: ' + results_dir)
sys.exit(1)
from girder_client import GirderClient
gc = GirderClient(apiUrl='https://data.kitware.com/api/v1')
gc.authenticate(apiKey=api_key)
# ITK/PerformanceBenchmarkingResults
folder_id = '5af50c818d777f06857985e3'
hostname_folder = gc.loadOrCreateFolder(hostname, folder_id, 'folder')
gc.upload(os.path.join(results_dir, '*.json'), hostname_folder['_id'],
leafFoldersAsItems=False, reuseExisting=True)
def gw_task_prerun(task=None,
sender=None,
task_id=None,
args=None,
kwargs=None,
**rest):
"""Deserialize the jobInfoSpec passed in through the headers.
This provides the a JobManager class as an attribute of the
task before task execution. decorated functions may bind to
their task and have access to the job_manager for logging and
updating their status in girder.
"""
if is_builtin_celery_task(sender.name):
return
try:
task.job_manager = _job_manager(task.request, task.request.headers)
_update_status(task, JobStatus.RUNNING)
except JobSpecNotFound:
task.job_manager = None
logger.warn('No jobInfoSpec. Setting job_manager to None.')
except StateTransitionException:
# Fetch the current status of the job
status = task.job_manager.refreshStatus()
# If we are canceling we want to stay in that state
if status != JobStatus.CANCELING:
raise
try:
task.girder_client = GirderClient(apiUrl=task.request.girder_api_url)
task.girder_client.token = task.request.girder_client_token
except AttributeError:
task.girder_client = None
# Deserialize girder_result_hooks if they exist
if hasattr(task.request, 'girder_result_hooks'):
u = jsonpickle.unpickler.Unpickler()
task.request.girder_result_hooks = \
[u.restore(grh) for grh in task.request.girder_result_hooks]
def main(args=None):
parser = argparse.ArgumentParser(
description='Mount Girder filesystem assetstore.')
parser.add_argument('--api-url', required=True, default=None,
help='full URL to the RESTful API of Girder server')
parser.add_argument('--username', required=False, default=None)
parser.add_argument('--password', required=False, default=None)
parser.add_argument('--api-key', required=False, default=None)
parser.add_argument('--token', required=False, default=None)
parser.add_argument('-c', default='remote', choices=['remote', 'direct'],
help='command to run')
parser.add_argument('--foreground', dest='foreground',
action='store_true')
parser.add_argument('--hostns', dest='hostns', action='store_true')
parser.add_argument('local_folder', help='path to local target folder')
parser.add_argument('remote_folder', help='Girder\'s folder id')
args = parser.parse_args()
gc = GirderClient(apiUrl=args.api_url)
if args.token:
gc.token = args.token
elif args.api_key:
gc.authenticate(apiKey=args.api_key)
elif args.username and args.password:
gc.authenticate(username=args.username, password=args.password)
else:
raise RuntimeError("You need to specify apiKey or user/pass")
if args.hostns:
targetns = os.path.join(os.environ.get('HOSTDIR', '/'),
'proc/1/ns/mnt')
with open(targetns) as fd:
setns(fd, CLONE_NEWNS)
if args.c == 'remote':
FUSE(RESTGirderFS(args.remote_folder, gc), args.local_folder,
foreground=args.foreground, ro=True, allow_other=True)
elif args.c == 'direct':
FUSE(LocalGirderFS(args.remote_folder, gc), args.local_folder,
foreground=args.foreground, ro=True, allow_other=True)
else:
print('No implementation for command %s' % args.c)
def testImportAnalyses(self):
"""
Test importing a romanesco analysis
"""
client = GirderClient('localhost', girder_port)
client.authenticate(self._username, self._password)
path = os.path.dirname(os.path.realpath(__file__))
analyses_path = os.path.join(path, 'analyses')
import_analyses.import_analyses(client, analyses_path)
# Get the analysis folder
path = '/minerva_analysis/folder'
response = self.request(path=path, method='GET', params={}, user=self._user)
self.assertStatusOk(response)
analyses_folder = response.json['folder']
path = '/item'
params = {
'folderId': analyses_folder['_id']
}
response = self.request(path=path, method='GET', params=params, user=self._user)
self.assertStatusOk(response)
self.assertEquals(len(response.json), 2, 'Expecting two analyses')
analysis = response.json[0]
self.assertEquals(analysis['name'], 'add', 'Expecting analysis one name to be "add"')
expected_meta = {
u'minerva': {
u'analysis_type': u'add',
u'analysis_name': u'add',
u'analysis_id': analysis['_id']
},
u'analysis': {
u'inputs': [{
u'default': {
u'data': u'0',
u'format': u'json'
},
u'type': u'number',
u'name': u'a',
u'format': u'number'
},
{
u'type': u'number',
u'name': u'b',
u'format':
u'number'
}],
u'script': u'c = a + b',
u'mode': u'python',
u'outputs': [{
u'type': u'number',
u'name': u'c',
u'format': u'number'
}],
u'name': u'add'
}
}
self.assertEquals(analysis['meta'], expected_meta, 'Unexpected value for meta data')
analysis = response.json[1]
self.assertEquals(analysis['name'], 'local', 'Expecting analysis two name to be "local"')
expected_meta = {
u'minerva': {
u'analysis_type': u'local type',
u'analysis_name': u'local',
u'analysis_id': analysis['_id']
}
}
self.assertEquals(analysis['meta'], expected_meta, 'Unexpected value for meta data')
def _create_girder_client(girder_api_url, girder_token):
client = GirderClient(apiUrl=girder_api_url)
client.token = girder_token
return client
from girder_client import GirderClient
import json
import os
import pymongo
import sys
if len(sys.argv) < 2:
print "%s /path/to/arborCollections" % sys.argv[0]
print "This should be the root of a directory tree containing .json analysis files"
sys.exit(1)
arbor_collections_path = sys.argv[1]
# Authenticate with Girder.
c = GirderClient(host='localhost', port=8080)
c.authenticate('girder', 'girder')
# Recursively search the specified directory for .json files.
for root, dirs, files in os.walk(arbor_collections_path):
for file in files:
if not file.endswith(".json"):
continue
# Get the name of this file and the directory that it's in.
# We use the directory name as the collection name in Girder.
fullpath = os.path.join(root, file)
analysis_filename = os.path.basename(fullpath)
analysis_name = os.path.splitext(analysis_filename)[0]
analysis_dir = os.path.dirname(fullpath)
collection_name = os.path.basename(analysis_dir)
# Create this collection if it doesn't already exist.
parser = ArgumentParser(description='Initialize the girder environment')
parser.add_argument('--user', help='name of the user to create')
parser.add_argument('--password', help='password of the user to create')
parser.add_argument('--host', help='host to connect to')
parser.add_argument('--port', type=int, help='port to connect to')
parser.add_argument('--data-root', help=("root directory for the girder "
"server's local filesystem data"))
parser.add_argument('--broker', help='romanesco broker URI')
parser.add_argument('--hdfs-user', help='Hadoop HDFS user')
parser.add_argument('--hdfs-namenode', help='Hadoop HDFS namenode')
parser.add_argument('--hdfs-port', type=int, help='Hadoop HDFS port')
parser.add_argument('--web-hdfs-port', type=int, help='Hadoop WebHDFS port')
args = parser.parse_args()
client = GirderClient(host=args.host, port=args.port)
register_and_authenticate(client,
login=args.user,
password=args.password,
email='{}@localhost.com'.format(args.user),
firstName='Girder',
lastName='Admin')
local_assetstore_name = 'local'
if find_assetstore(local_assetstore_name) is None:
client.post('assetstore',
parameters=dict(name=local_assetstore_name,
type=str(AssetstoreType.FILESYSTEM),
root=os.path.join(args.data_root,
'assetstores',
class BaseIntegrationTest(unittest.TestCase):
def __init__(self, name, girder_url, girder_user, girder_password, job_timeout=60, cleanup=True):
super(BaseIntegrationTest, self).__init__(name)
self._job_id = None
self._script_id = None
self._output_folder_id = None
self._input_folder_id = None
self._girder_url = girder_url
self._girder_user = girder_user
self._girder_password = girder_password
self._job_timeout = job_timeout
self._data = 'Need more input!'
self._cleanup = cleanup
def setUp(self):
url = '%s/api/v1' % self._girder_url
self._client = GirderClient(apiUrl=url)
self._client.authenticate(self._girder_user,
self._girder_password)
user = self._client.get('user/me')
self._user_id = user['_id']
r = list(self._client.listFolder(self._user_id, 'user', name='Private'))
self.assertEqual(len(r), 1)
self._private_folder_id = r[0]['_id']
def tearDown(self):
if not self._cleanup:
return
if self._job_id:
try:
url = 'jobs/%s' % self._job_id
self._client.delete(url)
except Exception as e:
traceback.print_exc()
if self._script_id:
try:
url = 'scripts/%s' % self._script_id
self._client.delete(url)
except Exception:
traceback.print_exc()
if self._output_folder_id:
try:
url = 'folder/%s' % self._output_folder_id
self._client.delete(url)
except Exception:
traceback.print_exc()
if self._input_folder_id:
try:
url = 'folder/%s' % self._input_folder_id
self._client.delete(url)
except Exception:
traceback.print_exc()
def create_script(self, commands=[
'sleep 10', 'cat CumulusIntegrationTestInput'
]):
body = {
'commands': commands,
'name': 'CumulusIntegrationTestLob'
}
r = self._client.post('scripts', data=json.dumps(body))
self._script_id = r['_id']
def create_input(self, folder_name='CumulusInput'):
r = self._client.createFolder(self._private_folder_id, folder_name)
self._input_folder_id = r['_id']
size = len(self._data)
item = self._client.uploadFile(self._input_folder_id,
StringIO(self._data), 'CumulusIntegrationTestInput', size,
parentType='folder')
self._item_id = item['itemId']
def create_output_folder(self, folder_name='CumulusOutput'):
r = self._client.createFolder(self._private_folder_id, folder_name)
self._output_folder_id = r['_id']
def create_job(self, job_name='CumulusIntegrationTestJob', tail=None):
body = {
'name': job_name,
'scriptId': self._script_id,
'output': [{
'folderId': self._output_folder_id,
'path': '.'
}],
'input': [
{
'folderId': self._input_folder_id,
'path': '.'
}
]
}
if tail:
body['output'].append({
"path": tail,
"tail": True
})
job = self._client.post('jobs', data=json.dumps(body))
self._job_id = job['_id']
def submit_job(self, job_params={}, timeout=None):
url = 'clusters/%s/job/%s/submit' % (self._cluster_id, self._job_id)
self._client.put(url, data=json.dumps(job_params))
start = time.time()
while True:
time.sleep(1)
r = self._client.get('jobs/%s' % self._job_id)
if r['status'] in ['error', 'unexpectederror']:
r = self._client.get('jobs/%s/log' % self._job_id)
self.fail(str(r))
elif r['status'] == 'complete':
break
if time.time() - start > timeout:
self.fail('Job didn\'t complete in timeout')
def assert_output(self):
r = self._client.listItem(self._output_folder_id)
self.assertEqual(len(r), 4)
stdout_item = None
for i in r:
if i['name'].startswith('CumulusIntegrationTestJob-%s.o' % self._job_id):
stdout_item = i
break
self.assertIsNotNone(stdout_item)
r = self._client.get('item/%s/files' % i['_id'])
self.assertEqual(len(r), 1)
path = os.path.join(tempfile.gettempdir(), self._job_id)
try:
self._client.downloadFile(r[0]['_id'], path)
with open(path, 'rb') as fp:
self.assertEqual(fp.read(), self._data)
finally:
if os.path.exists(path):
os.remove(path)
@click.group(context_settings=_CONTEXT_SETTINGS, cls=_Group)
@click.option('--api-url', default=None,
help='RESTful API URL '
'(e.g https://girder.example.com:443/%s)' % GirderClient.DEFAULT_API_ROOT)
@click.option('--api-key', envvar='GIRDER_API_KEY', default=None,
help='[default: GIRDER_API_KEY env. variable]')
@click.option('--username', default=None)
@click.option('--password', default=None)
# Advanced options
@click.option('--host', default=None,
cls=_AdvancedOption,
help="[default: %s]" % GirderClient.DEFAULT_HOST)
@click.option('--scheme', default=None,
cls=_AdvancedOption,
help="[default: %s if %s else %s]" % (
GirderClient.getDefaultScheme(GirderClient.DEFAULT_HOST),
GirderClient.DEFAULT_HOST,
GirderClient.getDefaultScheme("girder.example.com")))
@click.option('--port', default=None,
cls=_AdvancedOption,
help="[default: %s if %s; %s if %s else %s]" % (
GirderClient.DEFAULT_HTTPS_PORT, "https",
GirderClient.DEFAULT_LOCALHOST_PORT, "localhost",
GirderClient.DEFAULT_HTTP_PORT,
))
@click.option('--api-root', default=None,
help='relative path to the Girder REST API '
'[default: %s]' % GirderClient.DEFAULT_API_ROOT,
show_default=True,
cls=_AdvancedOption)
@click.version_option(version=__version__, prog_name='Girder command line interface')
offset += limit
return result
parser = ArgumentParser(description='Initialize the girder environment')
parser.add_argument('--admin', help='name:pass for the admin user')
parser.add_argument('--host', help='host to connect to')
parser.add_argument('--port', type=int, help='port to connect to')
parser.add_argument('--broker', help='girder worker broker URI')
parser.add_argument('--s3', help='name of S3 bucket')
parser.add_argument('--aws-key-id', help='aws key id')
parser.add_argument('--aws-secret-key', help='aws secret key')
args = parser.parse_args()
client = GirderClient(host=args.host, port=args.port)
user, password = args.admin.split(":", 1)
if find_user('girder'):
client.authenticate('girder', 'girder')
ensure_user(client,
login=user,
password=password,
email='*****@*****.**',
firstName='Girder',
lastName='Admin')
client.authenticate(user, password)
def testBsveSearchAnalysis(self):
# create the analysis folder
path = '/minerva_analysis/folder'
response = self.request(path=path, method='POST', user=self._user)
self.assertStatusOk(response)
analyses_folder = response.json['folder']
# import the bsve analysis
client = GirderClient('localhost', girder_port)
client.authenticate('minervauser', 'password')
bsve_analysis_path = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../analyses/bsve'))
import_analyses.import_analyses(client, bsve_analysis_path)
path = '/item'
params = {
'folderId': analyses_folder['_id']
}
response = self.request(path=path, method='GET', params=params, user=self._user)
self.assertStatusOk(response)
self.assertEquals(len(response.json), 1, 'Expecting only one analysis')
analysis = response.json[0]
self.assertEquals(analysis['name'], 'bsve search', 'Expecting analysis name to be "bsve search"')
expected_meta = {
u'minerva': {
u'analysis_type': u'bsve_search',
u'analysis_name': u'bsve search',
u'analysis_id': analysis['_id']
}
}
self.assertEquals(analysis['meta'], expected_meta, 'Unexpected value for meta data')
# create the dataset folder
path = '/minerva_dataset/folder'
params = {
'userId': self._user['_id'],
}
response = self.request(path=path, method='POST', params=params, user=self._user)
self.assertStatusOk(response)
# mock the calls to bsve search
@urlmatch(netloc=r'(.*\.)?beta-search.bsvecosystem.net(.*)$')
def bsve_mock(url, request):
if url.path.split('/')[-1] == 'request':
return httmock.response(200, '12345')
else:
pluginTestDir = os.path.dirname(os.path.realpath(__file__))
filepath = os.path.join(pluginTestDir, 'data', 'bsve_search.json')
with open(filepath) as bsve_search_file:
content = {
'status': 1,
'results': json.load(bsve_search_file)
}
headers = {
'content-length': len(content),
'content-type': 'application/json'
}
return httmock.response(200, content, headers, request=request)
with HTTMock(bsve_mock):
response = self.request(
path='/minerva_analysis/bsve_search',
method='POST',
params={
'datasetName': 'test dataset',
'bsveSearchParams': '{}'
},
user=self._user
)
# wait for the async job to complete
searchResultsFinished = False
count = 0
while not searchResultsFinished and count < 5:
# get the dataset and check if it has been updated
path = '/minerva_dataset/%s/dataset' % str(response.json['dataset_id'])
response = self.request(
path=path,
method='GET',
user=self._user
)
dataset = response.json
if 'json_row' in dataset:
searchResultsFinished = True
else:
time.sleep(2)
count += 1
# ensure the first row of results was added to the dataset
self.assertTrue('json_row' in dataset, 'json_row expected in dataset')
self.assertTrue('data' in dataset['json_row'], 'data should be in json_row')
self.assertTrue('Longitude' in dataset['json_row']['data'], 'data.Longitude should be in json_row')
# ensure that we can map the Lat/Long to geojson, as this json has
# unicode values for Lat/Long
# update the minerva metadata with coordinate mapping
metadata = {'minerva': dataset}
metadata['minerva']['mapper'] = {
"latitudeKeypath": "data.Latitude",
"longitudeKeypath": "data.Longitude"
}
path = '/item/{}/metadata'.format(dataset['dataset_id'])
response = self.request(
path=path,
method='PUT',
user=self._user,
body=json.dumps(metadata),
type='application/json'
)
metadata = response.json
# create geojson in the dataset
path = '/minerva_dataset/{}/geojson'.format(dataset['dataset_id'])
response = self.request(
path=path,
method='POST',
user=self._user,
)
self.assertHasKeys(response.json, ['geojson_file'])
def main(config):
client = GirderClient(apiUrl=config.girder_api_url)
client.authenticate(config.girder_user,
config.girder_password)
try:
# First run the simple flow
print ('Running simple taskflow ...')
taskflow_id = create_taskflow(
client, 'cumulus.taskflow.core.test.mytaskflows.SimpleTaskFlow')
# Start the task flow
url = 'taskflows/%s/start' % (taskflow_id)
client.put(url)
# Wait for it to complete
wait_for_complete(client, taskflow_id)
# First run the simple flow
print ('Running linked taskflow ...')
taskflow_id = create_taskflow(
client, 'cumulus.taskflow.core.test.mytaskflows.LinkTaskFlow')
# Start the task flow
url = 'taskflows/%s/start' % (taskflow_id)
client.put(url)
# Wait for it to complete
wait_for_complete(client, taskflow_id)
# Test terminating a simple flow
print ('Running simple taskflow ...')
taskflow_id = create_taskflow(
client, 'cumulus.taskflow.core.test.mytaskflows.SimpleTaskFlow')
# Start the task flow
url = 'taskflows/%s/start' % (taskflow_id)
client.put(url)
time.sleep(4)
print ('Terminate the taskflow')
url = 'taskflows/%s/terminate' % (taskflow_id)
client.put(url)
# Wait for it to terminate
wait_for_terminated(client, taskflow_id)
# Now delete it
print ('Delete the taskflow')
url = 'taskflows/%s' % (taskflow_id)
try:
client.delete(url)
except HttpError as ex:
if ex.status != 202:
raise
# Wait for it to terminate
wait_for_deletion(client, taskflow_id)
# Now try something with a chord
print ('Running taskflow containing a chord ...')
taskflow_id = create_taskflow(
client, 'cumulus.taskflow.core.test.mytaskflows.ChordTaskFlow')
# Start the task flow
url = 'taskflows/%s/start' % (taskflow_id)
client.put(url)
# Wait for it to complete
wait_for_complete(client, taskflow_id)
# Now try a workflow that is the two connected together
print ('Running taskflow that connects to parts together ...')
taskflow_id = create_taskflow(
client, 'cumulus.taskflow.core.test.mytaskflows.ConnectTwoTaskFlow')
# Start the task flow
url = 'taskflows/%s/start' % (taskflow_id)
client.put(url)
# Wait for it to complete
wait_for_complete(client, taskflow_id)
# # Now try a composite workflow approach ...
# print ('Running taskflow that is a composite ...')
# taskflow_id = create_taskflow(
# client, 'cumulus.taskflow.core.test.mytaskflows.MyCompositeTaskFlow')
#
# # Start the task flow
# url = 'taskflows/%s/start' % (taskflow_id)
# client.put(url)
#
# # Wait for it to complete
# wait_for_complete(client, taskflow_id)
except HttpError as ex:
print( ex.responseText)
import sys
parser = argparse.ArgumentParser()
parser.add_argument("path", type=str,
help="path to Arbor web apps")
parser.add_argument("-g", "--girder-host", type=str, default='localhost',
help="host to Girder instance")
parser.add_argument("-p", "--girder-port", type=int, default=9000,
help="port to Girder instance")
args = parser.parse_args()
# Get the ID for our Analyses folder.
c = GirderClient(host=args.girder_host, port=args.girder_port)
c.authenticate('girder', 'girder')
folderSearch = c.get('resource/search', parameters={
'q': 'Analyses',
'types': '["folder"]'
})
folderId = folderSearch['folder'][0]['_id']
# Disable authorization requirements for running romanesco tasks
c.put('system/setting', parameters={
'key': 'flow.require_auth',
'value': 'false'
})
# Check if these analyses already exist. If so, we won't re-upload them.
uploadACR = False
def upload_path(cluster_connection, girder_token, folder_id, path):
girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl)
girder_client.token = girder_token
cluster_connection.makedirs(path)
_upload_path(cluster_connection, girder_client, folder_id, path)
