Exemplo n.º 1
1
class NewtIntegrationTest(BaseIntegrationTest):
    def __init__(self, name, girder_url, girder_user, girder_password, machine, job_timeout=60 * 5):
        super(NewtIntegrationTest, self).__init__(name, girder_url, girder_user, girder_password, job_timeout)
        self._cluster_id = None
        self._machine = machine

    def setUp(self):

        # First authenticate with NEWT
        self._session = Session()
        r = self._session.post(
            "https://newt.nersc.gov/newt/auth", {"username": self._girder_user, "password": self._girder_password}
        )

        self.assertEqual(r.status_code, 200)
        print r.json()
        self._newt_session_id = r.json()["newt_sessionid"]

        # Now authenticate with Girder using the session id
        url = "%s/api/v1/newt/authenticate/%s" % (self._girder_url, self._newt_session_id)
        r = self._session.put(url)
        self.assertEqual(r.status_code, 200)

        url = "%s/api/v1/newt/authenticate/%s" % (self._girder_url, self._newt_session_id)
        r = self._session.put(url)
        self.assertEqual(r.status_code, 200)

        url = "%s/api/v1" % self._girder_url
        self._client = GirderClient(apiUrl=url)
        self._client.token = self._session.cookies["girderToken"]

        user = self._client.get("user/me")
        self._user_id = user["_id"]
        r = self._client.listFolder(self._user_id, "user", name="Private")
        r = list(r)
        self.assertEqual(len(r), 1)
        self._private_folder_id = r[0]["_id"]

    def tearDown(self):
        super(NewtIntegrationTest, self).tearDown()
        if self._cluster_id:
            try:
                url = "clusters/%s" % self._cluster_id
                self._client.delete(url)
            except Exception:
                traceback.print_exc()

    def create_cluster(self):
        body = {"config": {"host": self._machine}, "name": "NewtIntegrationTest", "type": "newt"}

        r = self._client.post("clusters", data=json.dumps(body))
        self._cluster_id = r["_id"]

        # Now test the connection
        r = self._client.put("clusters/%s/start" % self._cluster_id)
        sleeps = 0
        while True:
            time.sleep(1)
            r = self._client.get("clusters/%s/status" % self._cluster_id)

            if r["status"] == "running":
                break
            elif r["status"] == "error":
                r = self._client.get("clusters/%s/log" % self._cluster_id)
                self.fail(str(r))

            if sleeps > 9:
                self.fail("Cluster never moved into running state")
            sleeps += 1

    def assert_output(self):
        r = self._client.listItem(self._output_folder_id)
        self.assertEqual(len(r), 4)

        stdout_item = None
        for i in r:
            if i["name"].startswith("CumulusIntegrationTestJob-%s.o" % self._job_id):
                stdout_item = i
                break

        self.assertIsNotNone(stdout_item)
        r = self._client.get("item/%s/files" % i["_id"])
        self.assertEqual(len(r), 1)

        url = "%s/api/v1/file/%s/download" % (self._girder_url, r[0]["_id"])
        r = self._session.get(url)
        self.assertEqual(r.content, self._data)

    def test(self):
        try:
            self.create_cluster()
            self.create_script()
            self.create_input()
            self.create_output_folder()
            self.create_job()
            self.submit_job(timeout=self._job_timeout)
            self.assert_output()
        except HttpError as error:
            self.fail(error.responseText)
 def __init__(self, apiKey, objectStore):
     """initialization function to create a GirderCli instance, will attempt
     to authenticate with the designated Girder instance.
     """
     GirderClient.__init__(self, apiUrl="https://data.kitware.com/api/v1")
     self.objectStore = objectStore
     self.authenticate(apiKey=apiKey)
Exemplo n.º 3
0
    def __init__(self, girder_url, newt_sessionid):
        '''
    '''
        self._client = None
        self._cluster_id = None
        self._girder_url = girder_url
        self._input_folder_id = None
        self._job_folder_id = None
        self._job_id = None
        self._output_folder_id = None
        self._private_folder_id = None
        self._script_id = None
        self._session = requests.Session()

        # Authenticate with Girder using the newt session id
        url = '%s/api/v1/newt/authenticate/%s' % \
          (self._girder_url, newt_sessionid)
        r = self._session.put(url)
        if r.status_code != 200:
            raise HttpError(r.status_code, r.text, r.url, r.request.method)

        # Instantiate Girder client
        url = '%s/api/v1' % self._girder_url
        self._client = GirderClient(apiUrl=url)
        self._client.token = self._session.cookies['girderToken']

        user = self._client.get('user/me')
        #print 'user', user
        user_id = user['_id']
        r = self._client.listFolder(user_id, 'user', name='Private')
        if len(r) != 1:
            raise Exception('Wrong number of users; should be 1 got %s' %
                            len(r))
        self._private_folder_id = r[0]['_id']
        print 'private_folder_id', self._private_folder_id
Exemplo n.º 4
0
def download_path(cluster_connection,
                  girder_token,
                  parent,
                  path,
                  assetstore_url,
                  assetstore_id,
                  upload=False,
                  include=None,
                  exclude=None):
    """
    Download a given path on a cluster into an assetstore.

    :params cluster_connection: The cluster connection to access the cluster.
    :params girder_token: The Girder token to use to access Girder.
    :params parent: The target folder to import the path into.
    :params path: The path on the cluster to download.
    :params assetstore_url: The url for the assetstore to use for the import.
    :params assetstore_id: The id of the asseststore to import into.
    :params upload: Indicate if the import should upload the file data or just
                    the metadata, the default is False.
    :params include: List of include regexs
    :params exclude: List of exclude regexs,
    """
    girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl)
    girder_client.token = girder_token

    _import_path(cluster_connection,
                 girder_client,
                 parent,
                 path,
                 assetstore_url,
                 assetstore_id,
                 upload=upload,
                 include=include,
                 exclude=exclude)
Exemplo n.º 5
0
class GirderClient(object):
    def __init__(self):
        token_obj = get_oc_token_obj()
        url = token_obj.get('apiUrl')
        api_key = token_obj.get('apiKey')

        url = os.environ.get('OC_API_URL', url)
        internal_url = os.environ.get('OC_INTERNAL_API_URL', url)
        api_key = os.environ.get('OC_API_KEY', api_key)
        token = os.environ.get('GIRDER_TOKEN')

        self.client = None
        self.url = url
        self.internal_url = internal_url

        if internal_url is not None:
            self.client = GC(apiUrl=internal_url)

            if api_key is not None:
                self.client.authenticate(apiKey=api_key)
            elif token is not None:
                self.client.token = token

    def __getattr__(self, name):
        return getattr(self.client, name)
Exemplo n.º 6
0
 def __init__(self,
              username,
              password,
              dryrun,
              blacklist,
              host=None,
              port=None,
              apiRoot=None,
              scheme=None):
     """initialization function to create a GirderCli instance, will attempt
     to authenticate with the designated Girder instance.
     :param username: username to authenticate to Girder instance.
     :param password: password to authenticate to Girder instance, leave
         this blank to be prompted.
     :param dryrun: boolean indicating whether to run the command or just
         perform a dryrun showing which files and folders will be uploaded.
     :param blacklist: list of filenames which will be ignored by upload.
     :param host: host used to connect to Girder instance,
         defaults to 'localhost'
     :param port: port used to connect to Girder instance,
         default is 80 for http: and 443 for https:
     :param apiRoot: The path on the server corresponding to the root of the
         Girder REST API. If None is passed, assumes '/api/v1'.
     :param scheme: scheme used to connect to Girder instance,
         defaults to 'http'
     """
     GirderClient.__init__(self,
                           host=host,
                           port=port,
                           apiRoot=apiRoot,
                           scheme=scheme,
                           dryrun=dryrun,
                           blacklist=blacklist)
     interactive = password is None
     self.authenticate(username, password, interactive=interactive)
Exemplo n.º 7
0
def wait_for_jobs(client: GirderClient, max_wait_timeout=30, expected_status=JobStatus.SUCCESS):
    """Wait for all worker jobs to complete"""
    start_time = time.time()
    incompleteJobs = []
    while True and (time.time() - start_time < max_wait_timeout):
        incompleteJobs = client.get(
            'job',
            parameters={
                # https://github.com/girder/girder/blob/master/plugins/jobs/girder_jobs/constants.py
                # https://github.com/girder/girder_worker/blob/master/girder_worker/girder_plugin/status.py
                'statuses': json.dumps([0, 1, 2, 820, 821, 822, 823, 824]),
            },
        )
        if len(incompleteJobs) == 0:
            break
        time.sleep(1)
    if len(incompleteJobs) > 0:
        raise Exception("Jobs were still running after timeout")
    # Verify that all jobs succeeded
    time.sleep(1)
    lastJob = client.get(
        'job',
        parameters={
            'limit': 1,
        },
    )
    if len(lastJob) > 0 and lastJob[0]['status'] != expected_status:
        raise Exception(f"Some jobs did not meet their expected status: {expected_status}")
Exemplo n.º 8
0
 def __init__(self, apiKey, objectStore):
     """initialization function to create a GirderCli instance, will attempt
     to authenticate with the designated Girder instance.
     """
     GirderClient.__init__(self, apiUrl="https://data.kitware.com/api/v1")
     self.objectStore = objectStore
     self.authenticate(apiKey=apiKey)
Exemplo n.º 9
0
def main():
    parser = argparse.ArgumentParser(
        description='Import analyses into minerva')
    parser.add_argument('--username', required=False, default=None)
    parser.add_argument('--password', required=False, default=None)
    parser.add_argument('--scheme', required=False, default='http')
    parser.add_argument('--host', required=False, default='localhost')
    parser.add_argument('--port', required=False, default='8080')
    parser.add_argument('--api-root',
                        required=False,
                        default='/api/v1',
                        help='path to the Girder REST API')
    parser.add_argument('--path',
                        required=True,
                        help='the path to import the analyses from')

    config = parser.parse_args()

    client = GirderClient(host=config.host,
                          port=config.port,
                          apiRoot=config.api_root,
                          scheme=config.scheme)
    client.authenticate(config.username, config.password)

    import_analyses(client, config.path)
Exemplo n.º 10
0
Arquivo: cli.py Projeto: salamb/girder
    def __init__(self,
                 username,
                 password,
                 dryrun,
                 blacklist,
                 host=None,
                 port=None,
                 apiRoot=None,
                 scheme=None,
                 apiUrl=None,
                 apiKey=None):
        """initialization function to create a GirderCli instance, will attempt
        to authenticate with the designated Girder instance. Aside from username
        and password, all other kwargs are passed directly through to the
        :py:class:`girder_client.GirderClient` base class constructor.

        :param username: username to authenticate to Girder instance.
        :param password: password to authenticate to Girder instance, leave
            this blank to be prompted.
        """
        GirderClient.__init__(self,
                              host=host,
                              port=port,
                              apiRoot=apiRoot,
                              scheme=scheme,
                              dryrun=dryrun,
                              blacklist=blacklist,
                              apiUrl=apiUrl)
        interactive = password is None

        if apiKey:
            self.authenticate(apiKey=apiKey)
        elif username:
            self.authenticate(username, password, interactive=interactive)
Exemplo n.º 11
0
    def _update_girder(taskflow, body):
        taskflow = to_taskflow(taskflow)
        taskflow_id = taskflow['id']
        girder_token = taskflow['girder_token']
        girder_api_url = taskflow['girder_api_url']

        client = GirderClient(apiUrl=girder_api_url)
        client.token = girder_token

        client = _create_girder_client(girder_api_url, girder_token)

        # If this is a retry then we have already create a task get it from
        # the current tasks headers.
        if body['retries'] > 0:
            taskflow_task_id = \
                current_task.request.headers[TASKFLOW_TASK_ID_HEADER]

            # Celery always fires the postrun handler with a state of SUCCESS
            # for retries. So we need to save the retries here so we can
            # determine in the postrun handler if the task is really complete.
            current_task.request.headers[TASKFLOW_RETRY_HEADER] \
                = body['retries']
        else:
                # This is a new task so create a taskflow task instance
            body = {
                'celeryTaskId': body['id'],
                'name': body['task']
            }
            url = 'taskflows/%s/tasks' % taskflow_id
            r = client.post(url, data=json.dumps(body))
            taskflow_task_id = r['_id']
        return taskflow, taskflow_task_id
    def setUp(self):

        # First authenticate with NEWT
        self._session = Session()
        r = self._session.post('https://newt.nersc.gov/newt/auth', {
            'username': self._girder_user,
            'password': self._girder_password
        })

        self.assertEqual(r.status_code, 200)
        print r.json()
        self._newt_session_id = r.json()['newt_sessionid']

        # Now authenticate with Girder using the session id
        url = '%s/api/v1/newt/authenticate/%s' % (self._girder_url,
                                                  self._newt_session_id)
        r = self._session.put(url)
        self.assertEqual(r.status_code, 200)

        url = '%s/api/v1/newt/authenticate/%s' % (self._girder_url,
                                                  self._newt_session_id)
        r = self._session.put(url)
        self.assertEqual(r.status_code, 200)

        url = '%s/api/v1' % self._girder_url
        self._client = GirderClient(apiUrl=url)
        self._client.token = self._session.cookies['girderToken']

        user = self._client.get('user/me')
        self._user_id = user['_id']
        r = self._client.listFolder(self._user_id, 'user', name='Private')
        r = list(r)
        self.assertEqual(len(r), 1)
        self._private_folder_id = r[0]['_id']
Exemplo n.º 13
0
    def _update_girder(taskflow, body):
        taskflow = to_taskflow(taskflow)
        taskflow_id = taskflow['id']
        girder_token = taskflow['girder_token']
        girder_api_url = taskflow['girder_api_url']

        client = GirderClient(apiUrl=girder_api_url)
        client.token = girder_token

        client = _create_girder_client(girder_api_url, girder_token)

        # If this is a retry then we have already create a task get it from
        # the current tasks headers.
        if body['retries'] > 0:
            taskflow_task_id = \
                current_task.request.headers[TASKFLOW_TASK_ID_HEADER]

            # Celery always fires the postrun handler with a state of SUCCESS
            # for retries. So we need to save the retries here so we can
            # determine in the postrun handler if the task is really complete.
            current_task.request.headers[TASKFLOW_RETRY_HEADER] \
                = body['retries']
        else:
            # This is a new task so create a taskflow task instance
            body = {'celeryTaskId': body['id'], 'name': body['task']}
            url = 'taskflows/%s/tasks' % taskflow_id
            r = client.post(url, data=json.dumps(body))
            taskflow_task_id = r['_id']
        return taskflow, taskflow_task_id
Exemplo n.º 14
0
def test_sanity_checks():
    gc = GirderClient(apiUrl=source_api_root)
    resp = gc.get('/', jsonResp=False)
    assert resp.status_code == 200
    assert resp.headers['Content-Type'] == 'text/html;charset=utf-8'
    resp = gc.get('system/check', jsonResp=False)
    assert resp.status_code == 200
Exemplo n.º 15
0
def test_upgrade_pipelines(admin_client: GirderClient):
    cnf = admin_client.get('dive_configuration/pipelines')
    if 'detector' not in cnf:
        admin_client.post(
            'dive_configuration/upgrade_pipelines',
            data=json.dumps(tasks.UPGRADE_JOB_DEFAULT_URLS),
        )
    wait_for_jobs(admin_client, 1000)
Exemplo n.º 16
0
def _taskflow_task_finished(taskflow, taskflow_task_id):
    girder_token = taskflow['girder_token']
    girder_api_url = taskflow['girder_api_url']

    client = GirderClient(apiUrl=girder_api_url)
    client.token = girder_token
    url = 'taskflows/%s/tasks/%s/finished' % (taskflow.id, taskflow_task_id)

    return client.put(url)
Exemplo n.º 17
0
def _taskflow_task_finished(taskflow, taskflow_task_id):
    girder_token = taskflow['girder_token']
    girder_api_url = taskflow['girder_api_url']

    client = GirderClient(apiUrl=girder_api_url)
    client.token = girder_token
    url = 'taskflows/%s/tasks/%s/finished' % (taskflow.id, taskflow_task_id)

    return client.put(url)
Exemplo n.º 18
0
    def _importAnalysis(self):
        """Setup and import analyses for bsve tests."""
        if self._import_done:
            return

        path = "/minerva_analysis/folder"
        response = self.request(path=path, method="POST", user=self._user)
        self.assertStatusOk(response)
        analyses_folder = response.json["folder"]

        # import the bsve analysis
        client = GirderClient("localhost", girder_port)
        client.authenticate("minervauser", "password")

        bsve_analysis_path = os.path.abspath(
            os.path.join(os.path.dirname(os.path.realpath(__file__)), "../analyses/bsve")
        )
        import_analyses.import_analyses(client, bsve_analysis_path)

        path = "/item"
        params = {"folderId": analyses_folder["_id"]}
        response = self.request(path=path, method="GET", params=params, user=self._user)
        self.assertStatusOk(response)
        self.assertEquals(len(response.json), 2, "Expecting only one analysis")
        for analysis in response.json:
            if analysis["name"] == "bsve search":
                search_analysis = analysis
            elif analysis["name"] == "MMWR data import":
                soda_analysis = analysis
            else:
                self.fail('Unexpected analysis found "%s".' % analysis["name"])
        expected_meta = {
            u"minerva": {
                u"analysis_type": u"bsve_search",
                u"analysis_name": u"bsve search",
                u"analysis_id": search_analysis["_id"],
            }
        }
        self.assertEquals(search_analysis["meta"], expected_meta, "Unexpected value for search meta data")
        expected_meta = {
            u"minerva": {
                u"analysis_type": u"mmwr_import_data",
                u"analysis_name": u"MMWR data import",
                u"analysis_id": soda_analysis["_id"],
            }
        }
        self.assertEquals(soda_analysis["meta"], expected_meta, "Unexpected value for soda meta data")

        # create the dataset folder
        path = "/minerva_dataset/folder"
        params = {"userId": self._user["_id"]}
        response = self.request(path=path, method="POST", params=params, user=self._user)
        self.assertStatusOk(response)
        self._importDone = True
Exemplo n.º 19
0
    def setUp(self):
        url = '%s/api/v1' % self._girder_url
        self._client = GirderClient(apiUrl=url)
        self._client.authenticate(self._girder_user, self._girder_password)

        user = self._client.get('user/me')
        self._user_id = user['_id']
        r = list(self._client.listFolder(self._user_id, 'user',
                                         name='Private'))
        self.assertEqual(len(r), 1)
        self._private_folder_id = r[0]['_id']
Exemplo n.º 20
0
def main(ctx, api_key, api_url):
    """Openchemistry Client

    The client can be used to fetch molecules, add molecules, etc.
    """
    gc = GirderClient(apiUrl=api_url)

    if api_key is not None:
        gc.authenticate(apiKey=api_key)

    ctx.obj = gc
Exemplo n.º 21
0
def import_calc(config):
    try:
        target_port = None
        if config.port:
            target_port = config.port
        target_scheme = None
        if config.scheme:
            target_scheme = config.scheme
        target_apiroot = None
        if config.apiroot:
            target_apiroot = config.apiroot

        client = GirderClient(host=config.host, port=target_port,
                              scheme=target_scheme, apiRoot=target_apiroot)
        client.authenticate(apiKey=config.apiKey)

        me = client.get('/user/me')
        if not me:
            print('Error: Girder token invalid, please verify')
            return

        folderParams = {
            'parentId': me['_id'],
            'parentType': 'user',
            'name': 'Private'
        }

        # Get the private folder id first
        folder = next(client.listResource('folder', folderParams))
        folder = next(client.listFolder(me['_id'], 'user', 'Private'))

        for file_name in config.datafile:
            print ('\nUploading ' + file_name)
            file_id = {}
            with open(file_name, 'r') as fp:
                fileNameBase = os.path.basename(file_name)
                size = os.path.getsize(file_name)
                file_id = client.uploadFile(folder['_id'], fp, fileNameBase,
                                            size, 'folder')

            body = {
                'fileId': file_id['_id']
            }

            if config.public:
                body['public'] = True


            mol = client.sendRestRequest('POST', 'molecules', data=json.dumps(body))

            if mol and '_id' in mol:
                config.moleculeId = mol['_id']
                print('Molecule ID: ' + mol['_id'])
            else:
                print(mol)

    except HttpError as error:
        print(error.responseText, file=sys.stderr)
Exemplo n.º 22
0
def test_user_creation(admin_client: GirderClient, user: dict):
    try:
        admin_client.createUser(
            user['login'],
            user['email'],
            user['firstName'],
            user['lastName'],
            user['password'],
        )
    except HttpError as err:
        if err.response.json()['message'] != 'That login is already registered.':
            raise err
Exemplo n.º 23
0
def gc_init(api_url='https://girder.hub.yt/api/v1', api_key=None):
    from girder_client import GirderClient
    if api_key is None:
        try:
            api_key = os.environ['GIRDER_API_KEY']
        except KeyError as err:
            msg = 'please define GIRDER_API_KEY environment variable\n'
            msg += ' or pass api_key kwargs.'
            raise KeyError(msg)
    gc = GirderClient(apiUrl=api_url)
    gc.authenticate(apiKey=api_key)
    return gc
Exemplo n.º 24
0
def _ingest(project, composite, dir, channel_map, api_url, api_key):
    if dir[-1] != '/':
        dir += '/'
    gc = GirderClient(apiUrl=api_url)
    gc.authenticate(apiKey=api_key)

    channel_map = json.load(channel_map)
    channel_map = {channel.upper():element.lower() for (channel,element) in channel_map.items()}

    experiments = _ingest_runs(gc, project, composite, dir)
    #(run_ids, runs) = _ingest_runs(gc, project, composite, dir)
    _ingest_samples(gc, project, composite, dir, experiments, channel_map)
Exemplo n.º 25
0
def upload_file(cluster, girder_token, file, path):
    """
    Upload a file to a cluster

    :param cluster: The cluster to upload to.
    :param girder_tokebn: The Grider token for Girder access.
    :param file: The Girder file object.
    :param path: The path on the cluster to upload to.
    """
    girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl)
    girder_client.token = girder_token
    with get_connection(girder_token, cluster) as conn:
        conn.makedirs(os.path.dirname(path))
        _upload_file(conn, girder_client, file, path)
Exemplo n.º 26
0
def upload_file(cluster, girder_token, file, path):
    """
    Upload a file to a cluster

    :param cluster: The cluster to upload to.
    :param girder_tokebn: The Grider token for Girder access.
    :param file: The Girder file object.
    :param path: The path on the cluster to upload to.
    """
    girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl)
    girder_client.token = girder_token
    with get_connection(girder_token, cluster) as conn:
        conn.makedirs(os.path.dirname(path))
        _upload_file(conn, girder_client, file, path)
Exemplo n.º 27
0
    def setUp(self):

        # First authenticate with NEWT
        self._session = Session()
        r = self._session.post('https://newt.nersc.gov/newt/auth',
                               {
                                    'username': self._girder_user,
                                    'password': self._girder_password})

        self.assertEqual(r.status_code, 200)
        print r.json()
        self._newt_session_id = r.json()['newt_sessionid']

        # Now authenticate with Girder using the session id
        url = '%s/api/v1/newt/authenticate/%s' % (self._girder_url, self._newt_session_id)
        r = self._session.put(url)
        self.assertEqual(r.status_code, 200)

        url = '%s/api/v1/newt/authenticate/%s' % (self._girder_url, self._newt_session_id)
        r = self._session.put(url)
        self.assertEqual(r.status_code, 200)

        url = '%s/api/v1' % self._girder_url
        self._client = GirderClient(apiUrl=url)
        self._client.token = self._session.cookies['girderToken']

        user = self._client.get('user/me')
        self._user_id = user['_id']
        r = self._client.listFolder(self._user_id, 'user', name='Private')
        self.assertEqual(len(r), 1)
        self._private_folder_id = r[0]['_id']
def testClientMetadataExtractor(testData, user):
    item = Item().load(testData['item']['_id'], user=user)
    assert item['name'] == testData['name']

    del item['meta']
    item = Item().save(item)
    assert 'meta' not in item

    client = GirderClient('localhost', int(os.environ['GIRDER_PORT']))
    client.authenticate(user['login'], 'password')
    extractor = ClientMetadataExtractor(client, testData['path'], testData['item']['_id'])
    extractor.extractMetadata()
    item = Item().load(testData['item']['_id'], user=user)
    assert item['name'] == testData['name']
    assert 'meta' in item
    assert item['meta']['MIME type'] == testData['mimeType']
Exemplo n.º 29
0
    def __init__(self, username, password, dryrun, blacklist,
                 host=None, port=None, apiRoot=None, scheme=None, apiUrl=None):
        """initialization function to create a GirderCli instance, will attempt
        to authenticate with the designated Girder instance. Aside from username
        and password, all other kwargs are passed directly through to the
        :py:class:`girder_client.GirderClient` base class constructor.

        :param username: username to authenticate to Girder instance.
        :param password: password to authenticate to Girder instance, leave
            this blank to be prompted.
        """
        GirderClient.__init__(self, host=host, port=port,
                              apiRoot=apiRoot, scheme=scheme, dryrun=dryrun,
                              blacklist=blacklist, apiUrl=apiUrl)
        interactive = password is None
        self.authenticate(username, password, interactive=interactive)
Exemplo n.º 30
0
    def setUp(self):

        # First authenticate with NEWT
        self._session = Session()
        r = self._session.post(
            "https://newt.nersc.gov/newt/auth", {"username": self._girder_user, "password": self._girder_password}
        )

        self.assertEqual(r.status_code, 200)
        print r.json()
        self._newt_session_id = r.json()["newt_sessionid"]

        # Now authenticate with Girder using the session id
        url = "%s/api/v1/newt/authenticate/%s" % (self._girder_url, self._newt_session_id)
        r = self._session.put(url)
        self.assertEqual(r.status_code, 200)

        url = "%s/api/v1/newt/authenticate/%s" % (self._girder_url, self._newt_session_id)
        r = self._session.put(url)
        self.assertEqual(r.status_code, 200)

        url = "%s/api/v1" % self._girder_url
        self._client = GirderClient(apiUrl=url)
        self._client.token = self._session.cookies["girderToken"]

        user = self._client.get("user/me")
        self._user_id = user["_id"]
        r = self._client.listFolder(self._user_id, "user", name="Private")
        r = list(r)
        self.assertEqual(len(r), 1)
        self._private_folder_id = r[0]["_id"]
Exemplo n.º 31
0
def init_girder(api_key=None, api_url='https://girder.hub.yt/api/v1'):
    """Initialize girder client, rely on environment variable:
    GIRDER_API_KEY, connect to yt Hub.

  Args:
    api_key (str, optional): use GIRDER_API_KEY env. var. by default
    api_url (str, optional): use yt Hub v1 by default
  Return:
    GirderClient: initialized girder client
  """
    from girder_client import GirderClient
    if api_key is None:
        import os
        api_key = os.environ['GIRDER_API_KEY']
    gc = GirderClient(apiUrl=api_url)
    gc.authenticate(apiKey=api_key)
    return gc
Exemplo n.º 32
0
def main():
    parser = argparse.ArgumentParser(description='Import analyses into minerva')
    parser.add_argument('--username', required=False, default=None)
    parser.add_argument('--password', required=False, default=None)
    parser.add_argument('--scheme', required=False, default='http')
    parser.add_argument('--host', required=False, default='localhost')
    parser.add_argument('--port', required=False, default='8080')
    parser.add_argument('--api-root', required=False, default='/api/v1',
                        help='path to the Girder REST API')
    parser.add_argument('--path', required=True, help='the path to import the analyses from')

    config = parser.parse_args()

    client = GirderClient(host=config.host, port=config.port,
                          apiRoot=config.api_root, scheme=config.scheme)
    client.authenticate(config.username, config.password)

    import_analyses(client, config.path)
    def testClientMetadataExtractor(self):
        item = self.model('item').load(self.item['_id'], user=self.user)
        self.assertEqual(item['name'], self.name)
        self.assertNotHasKeys(item, ['meta'])
        clientPath = os.path.join(ROOT_DIR, 'clients', 'python')
        sys.path.insert(0, clientPath)

        from girder_client import GirderClient

        client = GirderClient('localhost', int(os.environ['GIRDER_PORT']))
        client.authenticate(self.user['login'], self.password)
        extractor = ClientMetadataExtractor(client, self.path, self.item['_id'])
        extractor.extractMetadata()
        sys.path.remove(clientPath)
        item = self.model('item').load(self.item['_id'], user=self.user)
        self.assertEqual(item['name'], self.name)
        self.assertHasKeys(item, ['meta'])
        self.assertEqual(item['meta']['MIME type'], self.mimeType)
Exemplo n.º 34
0
    def testClientMetadataExtractor(self):
        item = Item().load(self.item['_id'], user=self.user)
        self.assertEqual(item['name'], self.name)
        self.assertNotHasKeys(item, ['meta'])
        clientPath = os.path.join(ROOT_DIR, 'clients', 'python')
        sys.path.insert(0, clientPath)

        from girder_client import GirderClient

        client = GirderClient('localhost', int(os.environ['GIRDER_PORT']))
        client.authenticate(self.user['login'], self.password)
        extractor = ClientMetadataExtractor(client, self.path,
                                            self.item['_id'])
        extractor.extractMetadata()
        sys.path.remove(clientPath)
        item = Item().load(self.item['_id'], user=self.user)
        self.assertEqual(item['name'], self.name)
        self.assertHasKeys(item, ['meta'])
        self.assertEqual(item['meta']['MIME type'], self.mimeType)
Exemplo n.º 35
0
def upload_zipped_flat_media_files(
    gc: GirderClient,
    manager: JobManager,
    folderId: str,
    working_directory: Path,
    create_subfolder=False,
):
    """Takes a flat folder of media files and/or annotation and generates a dataset from it"""
    listOfFileNames = os.listdir(working_directory)
    validation = gc.sendRestRequest('POST',
                                    '/dive_dataset/validate_files',
                                    json=listOfFileNames)
    root_folderId = folderId
    default_fps = gc.getFolder(root_folderId).get(
        f"meta.{constants.FPSMarker}", -1)
    if validation.get('ok', False):
        manager.write(f"Annotations: {validation['annotations']}\n")
        manager.write(f"Media: {validation['media']}\n")
        dataset_type = validation['type']
        manager.write(f"Type: {dataset_type}\n")
        if create_subfolder != '':
            sub_folder = gc.createFolder(
                folderId,
                create_subfolder,
                reuseExisting=True,
            )
            root_folderId = str(sub_folder["_id"])

        # Upload all resulting items back into the root folder
        manager.updateStatus(JobStatus.PUSHING_OUTPUT)
        # create a source folder to place the zipFile inside of
        gc.upload(f'{working_directory}/*', root_folderId)
        if dataset_type == constants.ImageSequenceType and default_fps == -1:
            default_fps = 1
        gc.addMetadataToFolder(
            str(root_folderId),
            {
                constants.TypeMarker: dataset_type,
                constants.FPSMarker: default_fps
            },
        )
        # After uploading the default files we do a the postprocess for video conversion now
        gc.sendRestRequest("POST",
                           f"/dive_rpc/postprocess/{str(root_folderId)}")
    else:
        manager.write(f"Message: {validation['message']}\n")
        raise Exception("Could not Validate media Files")
Exemplo n.º 36
0
    def setUp(self):
        url = '%s/api/v1' % self._girder_url
        self._client = GirderClient(apiUrl=url)
        self._client.authenticate(self._girder_user,
                                  self._girder_password)

        user = self._client.get('user/me')
        self._user_id = user['_id']
        r = list(self._client.listFolder(self._user_id, 'user', name='Private'))
        self.assertEqual(len(r), 1)
        self._private_folder_id = r[0]['_id']
Exemplo n.º 37
0
def main(config):
    client = GirderClient(apiUrl=config.girder_api_url)
    client.authenticate(config.girder_user,
                        config.girder_password)

    # Load any parameters
    params = {}
    if config.taskflow_start_params is not None:
        with open(config.taskflow_start_params) as fp:
            params = json.load(fp)

    print params

    try:
        print ('Running %s taskflow ...' % config.taskflow_start_params)
        taskflow_id = create_taskflow(
            client, config.taskflow_class)

        # Start the task flow
        url = 'taskflows/%s/start' % (taskflow_id)
        client.put(url, data=json.dumps(params))

        # Wait for it to complete
        wait_for_complete(client, taskflow_id)

    except HttpError as ex:
        print( ex.responseText)
Exemplo n.º 38
0
def get_girder_client() -> GirderClient:
    client = GirderClient(apiUrl="https://viame.kitware.com/api/v1")
    apikey = os.environ.get('GIRDER_API_KEY', None)
    if apikey:
        client.authenticate(apiKey=apikey)
    else:
        client.authenticate(interactive=True)
    return client
Exemplo n.º 39
0
def download_path(cluster_connection, girder_token, parent, path,
                  assetstore_url, assetstore_id, upload=False, include=None,
                  exclude=None):
    """
    Download a given path on a cluster into an assetstore.

    :params cluster_connection: The cluster connection to access the cluster.
    :params girder_token: The Girder token to use to access Girder.
    :params parent: The target folder to import the path into.
    :params path: The path on the cluster to download.
    :params assetstore_url: The url for the assetstore to use for the import.
    :params assetstore_id: The id of the asseststore to import into.
    :params upload: Indicate if the import should upload the file data or just
                    the metadata, the default is False.
    :params include: List of include regexs
    :params exclude: List of exclude regexs,
    """
    girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl)
    girder_client.token = girder_token

    _import_path(cluster_connection, girder_client, parent, path,
                 assetstore_url, assetstore_id, upload=upload, include=include,
                 exclude=exclude)
Exemplo n.º 40
0
 def __init__(self, username, password, dryrun, blacklist, host="localhost", port=8080, apiRoot=None, scheme="http"):
     """initialization function to create a GirderCli instance, will attempt
     to authenticate with the designated Girder instance.
     :param username: username to authenticate to Girder instance.
     :param password: password to authenticate to Girder instance, leave
         this blank to be prompted.
     :param dryrun: boolean indicating whether to run the command or just
         perform a dryrun showing which files and folders will be uploaded.
     :param blacklist: list of filenames which will be ignored by upload.
     :param host: host used to connect to Girder instance,
         defaults to 'localhost'
     :param port: port used to connect to Girder instance,
         defaults to 8080
     :param apiRoot: The path on the server corresponding to the root of the
         Girder REST API. If None is passed, assumes '/api/v1'.
     :param scheme: scheme used to connect to Girder instance,
         defaults to 'http'; if passing 'https' port should likely be 443.
     """
     GirderClient.__init__(
         self, host=host, port=port, apiRoot=apiRoot, scheme=scheme, dryrun=dryrun, blacklist=blacklist
     )
     interactive = password is None
     self.authenticate(username, password, interactive=interactive)
Exemplo n.º 41
0
def test_extract_download(data):
    fileId, filename = data
    filepath = localDataRoot / str(filename)
    if not filepath.exists():
        gc = GirderClient(apiUrl=source_api_root)
        gc.authenticate(apiKey=os.environ.get('GIRDER_API_KEY'))
        gc.downloadFile(fileId, str(filepath))
        with zipfile.ZipFile(filepath, 'r') as zipref:
            zipref.extractall(localDataRoot)
Exemplo n.º 42
0
def main(args=None):
    parser = argparse.ArgumentParser(
        description='Mount Girder filesystem assetstore.')
    parser.add_argument('--api-url',
                        required=True,
                        default=None,
                        help='full URL to the RESTful API of Girder server')
    parser.add_argument('--username', required=False, default=None)
    parser.add_argument('--password', required=False, default=None)
    parser.add_argument('--api-key', required=False, default=None)
    parser.add_argument('--token', required=False, default=None)
    parser.add_argument('-c',
                        default='remote',
                        choices=['remote', 'direct'],
                        help='command to run')
    parser.add_argument('--foreground', dest='foreground', action='store_true')
    parser.add_argument('--hostns', dest='hostns', action='store_true')
    parser.add_argument('local_folder', help='path to local target folder')
    parser.add_argument('remote_folder', help='Girder\'s folder id')

    args = parser.parse_args()

    gc = GirderClient(apiUrl=args.api_url)
    if args.token:
        gc.token = args.token
    elif args.api_key:
        gc.authenticate(apiKey=args.api_key)
    elif args.username and args.password:
        gc.authenticate(username=args.username, password=args.password)
    else:
        raise RuntimeError("You need to specify apiKey or user/pass")

    if args.hostns:
        targetns = os.path.join(os.environ.get('HOSTDIR', '/'),
                                'proc/1/ns/mnt')
        with open(targetns) as fd:
            setns(fd, CLONE_NEWNS)

    if args.c == 'remote':
        FUSE(RESTGirderFS(args.remote_folder, gc),
             args.local_folder,
             foreground=args.foreground,
             ro=True,
             allow_other=True)
    elif args.c == 'direct':
        FUSE(LocalGirderFS(args.remote_folder, gc),
             args.local_folder,
             foreground=args.foreground,
             ro=True,
             allow_other=True)
    else:
        print('No implementation for command %s' % args.c)
def main():
    """Create the folder hierarchy with metadata in a Girder instance."""
    args = parser.parse_args()
    g = GirderClient(host=args.host, port=args.port, scheme=args.scheme)
    g.authenticate(args.username, args.password)

    def create_folder_on_demand(parent_folder_id, folder_name):
        existing_folders = list(
            g.listFolder(parent_folder_id, name=folder_name))
        if not len(existing_folders):
            sought_folder = g.createFolder(parent_folder_id, name=folder_name)
        else:
            sought_folder = existing_folders[0]
        return sought_folder

    metadata_file = 'metadata.json'
    with open(metadata_file) as json_file:
        metadata = json.load(json_file)
    parent_folder_id = args.parent_folder_id

    for subject_id, subject_metadata in metadata.items():
        subject_folder = create_folder_on_demand(parent_folder_id, subject_id)
        for (scan_time, scan_date, scan_weight) in subject_metadata['scans']:
            create_folder_on_demand(subject_folder['_id'], scan_time)
def upload_benchmark_results(benchmark_bin, api_key=None):
    hostname = socket.gethostname().lower()
    results_dir = os.path.join(benchmark_bin, 'BenchmarkResults', hostname)
    if not os.path.exists(results_dir):
        sys.stderr.write('Expected results directory does not exist: ' +
                         results_dir)
        sys.exit(1)
    from girder_client import GirderClient
    gc = GirderClient(apiUrl='https://data.kitware.com/api/v1')
    gc.authenticate(apiKey=api_key)
    # ITK/PerformanceBenchmarkingResults
    folder_id = '5af50c818d777f06857985e3'
    hostname_folder = gc.loadOrCreateFolder(hostname, folder_id, 'folder')
    gc.upload(os.path.join(results_dir, '*.json'),
              hostname_folder['_id'],
              leafFoldersAsItems=False,
              reuseExisting=True)
def upload_benchmark_results(benchmark_bin, api_key=None):
    hostname = socket.gethostname().lower()
    results_dir = os.path.join(benchmark_bin, 'BenchmarkResults',
            hostname)
    if not os.path.exists(results_dir):
        sys.stderr.write('Expected results directory does not exist: ' + results_dir)
        sys.exit(1)
    from girder_client import GirderClient
    gc = GirderClient(apiUrl='https://data.kitware.com/api/v1')
    gc.authenticate(apiKey=api_key)
    # ITK/PerformanceBenchmarkingResults
    folder_id = '5af50c818d777f06857985e3'
    hostname_folder = gc.loadOrCreateFolder(hostname, folder_id, 'folder')
    gc.upload(os.path.join(results_dir, '*.json'), hostname_folder['_id'],
            leafFoldersAsItems=False, reuseExisting=True)
Exemplo n.º 46
0
def gw_task_prerun(task=None,
                   sender=None,
                   task_id=None,
                   args=None,
                   kwargs=None,
                   **rest):
    """Deserialize the jobInfoSpec passed in through the headers.

    This provides the a JobManager class as an attribute of the
    task before task execution.  decorated functions may bind to
    their task and have access to the job_manager for logging and
    updating their status in girder.
    """
    if is_builtin_celery_task(sender.name):
        return

    try:
        task.job_manager = _job_manager(task.request, task.request.headers)
        _update_status(task, JobStatus.RUNNING)

    except JobSpecNotFound:
        task.job_manager = None
        logger.warn('No jobInfoSpec. Setting job_manager to None.')
    except StateTransitionException:
        # Fetch the current status of the job
        status = task.job_manager.refreshStatus()
        # If we are canceling we want to stay in that state
        if status != JobStatus.CANCELING:
            raise

    try:
        task.girder_client = GirderClient(apiUrl=task.request.girder_api_url)
        task.girder_client.token = task.request.girder_client_token
    except AttributeError:
        task.girder_client = None

    # Deserialize girder_result_hooks if they exist
    if hasattr(task.request, 'girder_result_hooks'):
        u = jsonpickle.unpickler.Unpickler()
        task.request.girder_result_hooks = \
            [u.restore(grh) for grh in task.request.girder_result_hooks]
Exemplo n.º 47
0
def main(args=None):
    parser = argparse.ArgumentParser(
        description='Mount Girder filesystem assetstore.')
    parser.add_argument('--api-url', required=True, default=None,
                        help='full URL to the RESTful API of Girder server')
    parser.add_argument('--username', required=False, default=None)
    parser.add_argument('--password', required=False, default=None)
    parser.add_argument('--api-key', required=False, default=None)
    parser.add_argument('--token', required=False, default=None)
    parser.add_argument('-c', default='remote', choices=['remote', 'direct'],
                        help='command to run')
    parser.add_argument('--foreground', dest='foreground',
                        action='store_true')
    parser.add_argument('--hostns', dest='hostns', action='store_true')
    parser.add_argument('local_folder', help='path to local target folder')
    parser.add_argument('remote_folder', help='Girder\'s folder id')

    args = parser.parse_args()

    gc = GirderClient(apiUrl=args.api_url)
    if args.token:
        gc.token = args.token
    elif args.api_key:
        gc.authenticate(apiKey=args.api_key)
    elif args.username and args.password:
        gc.authenticate(username=args.username, password=args.password)
    else:
        raise RuntimeError("You need to specify apiKey or user/pass")

    if args.hostns:
        targetns = os.path.join(os.environ.get('HOSTDIR', '/'),
                                'proc/1/ns/mnt')
        with open(targetns) as fd:
            setns(fd, CLONE_NEWNS)

    if args.c == 'remote':
        FUSE(RESTGirderFS(args.remote_folder, gc), args.local_folder,
             foreground=args.foreground, ro=True, allow_other=True)
    elif args.c == 'direct':
        FUSE(LocalGirderFS(args.remote_folder, gc), args.local_folder,
             foreground=args.foreground, ro=True, allow_other=True)
    else:
        print('No implementation for command %s' % args.c)
Exemplo n.º 48
0
    def testImportAnalyses(self):
        """
        Test importing a romanesco analysis
        """
        client = GirderClient('localhost', girder_port)
        client.authenticate(self._username, self._password)

        path = os.path.dirname(os.path.realpath(__file__))
        analyses_path = os.path.join(path, 'analyses')

        import_analyses.import_analyses(client, analyses_path)

        # Get the analysis folder
        path = '/minerva_analysis/folder'
        response = self.request(path=path, method='GET', params={}, user=self._user)
        self.assertStatusOk(response)
        analyses_folder = response.json['folder']

        path = '/item'
        params = {
            'folderId': analyses_folder['_id']
        }
        response = self.request(path=path, method='GET', params=params, user=self._user)
        self.assertStatusOk(response)
        self.assertEquals(len(response.json), 2, 'Expecting two analyses')
        analysis = response.json[0]
        self.assertEquals(analysis['name'], 'add', 'Expecting analysis one name to be "add"')
        expected_meta = {
            u'minerva': {
                u'analysis_type': u'add',
                u'analysis_name': u'add',
                u'analysis_id': analysis['_id']
            },
            u'analysis': {
                u'inputs': [{
                    u'default': {
                        u'data': u'0',
                        u'format': u'json'
                    },
                    u'type': u'number',
                    u'name': u'a',
                    u'format': u'number'
                },
                {
                    u'type': u'number',
                    u'name': u'b',
                    u'format':
                    u'number'
                }],
                u'script': u'c = a + b',
                u'mode': u'python',
                u'outputs': [{
                    u'type': u'number',
                    u'name': u'c',
                    u'format': u'number'
                }],
                u'name': u'add'
            }
        }
        self.assertEquals(analysis['meta'], expected_meta, 'Unexpected value for meta data')

        analysis = response.json[1]
        self.assertEquals(analysis['name'], 'local', 'Expecting analysis two name to be "local"')
        expected_meta = {
            u'minerva': {
                u'analysis_type': u'local type',
                u'analysis_name': u'local',
                u'analysis_id': analysis['_id']
            }
        }
        self.assertEquals(analysis['meta'], expected_meta, 'Unexpected value for meta data')
Exemplo n.º 49
0
def _create_girder_client(girder_api_url, girder_token):
    client = GirderClient(apiUrl=girder_api_url)
    client.token = girder_token

    return client
Exemplo n.º 50
0
from girder_client import GirderClient
import json
import os
import pymongo
import sys

if len(sys.argv) < 2:
    print "%s /path/to/arborCollections" % sys.argv[0]
    print "This should be the root of a directory tree containing .json analysis files"
    sys.exit(1)
arbor_collections_path = sys.argv[1]

# Authenticate with Girder.
c = GirderClient(host='localhost', port=8080)
c.authenticate('girder', 'girder')

# Recursively search the specified directory for .json files.
for root, dirs, files in os.walk(arbor_collections_path):
  for file in files:
    if not file.endswith(".json"):
      continue

    # Get the name of this file and the directory that it's in.
    # We use the directory name as the collection name in Girder.
    fullpath = os.path.join(root, file)
    analysis_filename = os.path.basename(fullpath)
    analysis_name = os.path.splitext(analysis_filename)[0]
    analysis_dir = os.path.dirname(fullpath)
    collection_name = os.path.basename(analysis_dir)

    # Create this collection if it doesn't already exist.
Exemplo n.º 51
0
parser = ArgumentParser(description='Initialize the girder environment')
parser.add_argument('--user', help='name of the user to create')
parser.add_argument('--password', help='password of the user to create')
parser.add_argument('--host', help='host to connect to')
parser.add_argument('--port', type=int, help='port to connect to')
parser.add_argument('--data-root', help=("root directory for the girder "
                                         "server's local filesystem data"))
parser.add_argument('--broker', help='romanesco broker URI')
parser.add_argument('--hdfs-user', help='Hadoop HDFS user')
parser.add_argument('--hdfs-namenode', help='Hadoop HDFS namenode')
parser.add_argument('--hdfs-port', type=int, help='Hadoop HDFS port')
parser.add_argument('--web-hdfs-port', type=int, help='Hadoop WebHDFS port')

args = parser.parse_args()

client = GirderClient(host=args.host, port=args.port)
register_and_authenticate(client,
                          login=args.user,
                          password=args.password,
                          email='{}@localhost.com'.format(args.user),
                          firstName='Girder',
                          lastName='Admin')

local_assetstore_name = 'local'

if find_assetstore(local_assetstore_name) is None:
    client.post('assetstore',
                parameters=dict(name=local_assetstore_name,
                                type=str(AssetstoreType.FILESYSTEM),
                                root=os.path.join(args.data_root,
                                                  'assetstores',
Exemplo n.º 52
0
class BaseIntegrationTest(unittest.TestCase):
    def __init__(self, name, girder_url, girder_user, girder_password, job_timeout=60, cleanup=True):
        super(BaseIntegrationTest, self).__init__(name)
        self._job_id = None
        self._script_id = None
        self._output_folder_id = None
        self._input_folder_id = None
        self._girder_url = girder_url
        self._girder_user = girder_user
        self._girder_password = girder_password
        self._job_timeout = job_timeout
        self._data = 'Need more input!'
        self._cleanup = cleanup

    def setUp(self):
        url = '%s/api/v1' % self._girder_url
        self._client = GirderClient(apiUrl=url)
        self._client.authenticate(self._girder_user,
                                  self._girder_password)

        user = self._client.get('user/me')
        self._user_id = user['_id']
        r = list(self._client.listFolder(self._user_id, 'user', name='Private'))
        self.assertEqual(len(r), 1)
        self._private_folder_id = r[0]['_id']

    def tearDown(self):

        if not self._cleanup:
            return

        if self._job_id:
            try:
                url = 'jobs/%s' % self._job_id
                self._client.delete(url)
            except Exception as e:
                traceback.print_exc()

        if self._script_id:
            try:
                url = 'scripts/%s' % self._script_id
                self._client.delete(url)
            except Exception:
                traceback.print_exc()

        if self._output_folder_id:
            try:
                url = 'folder/%s' % self._output_folder_id
                self._client.delete(url)
            except Exception:
                traceback.print_exc()

        if self._input_folder_id:
            try:
                url = 'folder/%s' % self._input_folder_id
                self._client.delete(url)
            except Exception:
                traceback.print_exc()

    def create_script(self, commands=[
                'sleep 10', 'cat CumulusIntegrationTestInput'
            ]):
        body = {
            'commands': commands,
            'name': 'CumulusIntegrationTestLob'
        }

        r = self._client.post('scripts', data=json.dumps(body))
        self._script_id = r['_id']

    def create_input(self, folder_name='CumulusInput'):

        r = self._client.createFolder(self._private_folder_id, folder_name)
        self._input_folder_id = r['_id']
        size = len(self._data)

        item = self._client.uploadFile(self._input_folder_id,
                    StringIO(self._data), 'CumulusIntegrationTestInput', size,
                    parentType='folder')

        self._item_id = item['itemId']

    def create_output_folder(self, folder_name='CumulusOutput'):
        r = self._client.createFolder(self._private_folder_id, folder_name)
        self._output_folder_id = r['_id']

    def create_job(self, job_name='CumulusIntegrationTestJob', tail=None):
        body = {
            'name': job_name,
            'scriptId': self._script_id,
            'output': [{
              'folderId': self._output_folder_id,
              'path': '.'
            }],
            'input': [
              {
                'folderId': self._input_folder_id,
                'path': '.'
              }
            ]
        }

        if tail:
            body['output'].append({
                "path": tail,
                "tail": True
            })

        job = self._client.post('jobs', data=json.dumps(body))
        self._job_id = job['_id']

    def submit_job(self, job_params={}, timeout=None):
        url = 'clusters/%s/job/%s/submit' % (self._cluster_id, self._job_id)

        self._client.put(url, data=json.dumps(job_params))
        start = time.time()
        while True:
            time.sleep(1)
            r = self._client.get('jobs/%s' % self._job_id)

            if r['status'] in ['error', 'unexpectederror']:
                r = self._client.get('jobs/%s/log' % self._job_id)
                self.fail(str(r))
            elif r['status'] == 'complete':
                break

            if time.time() - start > timeout:
                self.fail('Job didn\'t complete in timeout')

    def assert_output(self):
        r = self._client.listItem(self._output_folder_id)
        self.assertEqual(len(r), 4)

        stdout_item = None
        for i in r:
            if i['name'].startswith('CumulusIntegrationTestJob-%s.o' % self._job_id):
                stdout_item = i
                break

        self.assertIsNotNone(stdout_item)
        r = self._client.get('item/%s/files' % i['_id'])
        self.assertEqual(len(r), 1)

        path = os.path.join(tempfile.gettempdir(), self._job_id)
        try:
            self._client.downloadFile(r[0]['_id'], path)
            with open(path, 'rb') as fp:
                self.assertEqual(fp.read(), self._data)

        finally:
            if os.path.exists(path):
                os.remove(path)
Exemplo n.º 53
0
@click.group(context_settings=_CONTEXT_SETTINGS, cls=_Group)
@click.option('--api-url', default=None,
              help='RESTful API URL '
                   '(e.g https://girder.example.com:443/%s)' % GirderClient.DEFAULT_API_ROOT)
@click.option('--api-key', envvar='GIRDER_API_KEY', default=None,
              help='[default: GIRDER_API_KEY env. variable]')
@click.option('--username', default=None)
@click.option('--password', default=None)
# Advanced options
@click.option('--host', default=None,
              cls=_AdvancedOption,
              help="[default: %s]" % GirderClient.DEFAULT_HOST)
@click.option('--scheme', default=None,
              cls=_AdvancedOption,
              help="[default: %s if %s else %s]" % (
                  GirderClient.getDefaultScheme(GirderClient.DEFAULT_HOST),
                  GirderClient.DEFAULT_HOST,
                  GirderClient.getDefaultScheme("girder.example.com")))
@click.option('--port', default=None,
              cls=_AdvancedOption,
              help="[default: %s if %s; %s if %s else %s]" % (
                  GirderClient.DEFAULT_HTTPS_PORT, "https",
                  GirderClient.DEFAULT_LOCALHOST_PORT, "localhost",
                  GirderClient.DEFAULT_HTTP_PORT,
                  ))
@click.option('--api-root', default=None,
              help='relative path to the Girder REST API '
                   '[default: %s]' % GirderClient.DEFAULT_API_ROOT,
              show_default=True,
              cls=_AdvancedOption)
@click.version_option(version=__version__, prog_name='Girder command line interface')
Exemplo n.º 54
0
        offset += limit

    return result

parser = ArgumentParser(description='Initialize the girder environment')
parser.add_argument('--admin', help='name:pass for the admin user')
parser.add_argument('--host', help='host to connect to')
parser.add_argument('--port', type=int, help='port to connect to')
parser.add_argument('--broker', help='girder worker broker URI')
parser.add_argument('--s3', help='name of S3 bucket')
parser.add_argument('--aws-key-id', help='aws key id')
parser.add_argument('--aws-secret-key', help='aws secret key')

args = parser.parse_args()

client = GirderClient(host=args.host, port=args.port)

user, password = args.admin.split(":", 1)

if find_user('girder'):
    client.authenticate('girder', 'girder')

ensure_user(client,
            login=user,
            password=password,
            email='*****@*****.**',
            firstName='Girder',
            lastName='Admin')

client.authenticate(user, password)
Exemplo n.º 55
0
    def testBsveSearchAnalysis(self):
        # create the analysis folder
        path = '/minerva_analysis/folder'
        response = self.request(path=path, method='POST', user=self._user)
        self.assertStatusOk(response)
        analyses_folder = response.json['folder']

        # import the bsve analysis
        client = GirderClient('localhost', girder_port)
        client.authenticate('minervauser', 'password')

        bsve_analysis_path = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../analyses/bsve'))
        import_analyses.import_analyses(client, bsve_analysis_path)

        path = '/item'
        params = {
            'folderId': analyses_folder['_id']
        }
        response = self.request(path=path, method='GET', params=params, user=self._user)
        self.assertStatusOk(response)
        self.assertEquals(len(response.json), 1, 'Expecting only one analysis')
        analysis = response.json[0]
        self.assertEquals(analysis['name'], 'bsve search', 'Expecting analysis name to be "bsve search"')
        expected_meta = {
            u'minerva': {
                u'analysis_type': u'bsve_search',
                u'analysis_name': u'bsve search',
                u'analysis_id': analysis['_id']
            }
        }
        self.assertEquals(analysis['meta'], expected_meta, 'Unexpected value for meta data')

        # create the dataset folder
        path = '/minerva_dataset/folder'
        params = {
            'userId': self._user['_id'],
        }
        response = self.request(path=path, method='POST', params=params, user=self._user)
        self.assertStatusOk(response)

        # mock the calls to bsve search
        @urlmatch(netloc=r'(.*\.)?beta-search.bsvecosystem.net(.*)$')
        def bsve_mock(url, request):
            if url.path.split('/')[-1] == 'request':
                return httmock.response(200, '12345')
            else:
                pluginTestDir = os.path.dirname(os.path.realpath(__file__))
                filepath = os.path.join(pluginTestDir, 'data', 'bsve_search.json')
                with open(filepath) as bsve_search_file:
                    content = {
                        'status': 1,
                        'results': json.load(bsve_search_file)
                    }
                    headers = {
                        'content-length': len(content),
                        'content-type': 'application/json'
                    }
                    return httmock.response(200, content, headers, request=request)

        with HTTMock(bsve_mock):
            response = self.request(
                path='/minerva_analysis/bsve_search',
                method='POST',
                params={
                    'datasetName': 'test dataset',
                    'bsveSearchParams': '{}'
                },
                user=self._user
            )

            # wait for the async job to complete
            searchResultsFinished = False
            count = 0
            while not searchResultsFinished and count < 5:
                # get the dataset and check if it has been updated
                path = '/minerva_dataset/%s/dataset' % str(response.json['dataset_id'])
                response = self.request(
                    path=path,
                    method='GET',
                    user=self._user
                )
                dataset = response.json
                if 'json_row' in dataset:
                    searchResultsFinished = True
                else:
                    time.sleep(2)
                    count += 1

            # ensure the first row of results was added to the dataset
            self.assertTrue('json_row' in dataset, 'json_row expected in dataset')
            self.assertTrue('data' in dataset['json_row'], 'data should be in json_row')
            self.assertTrue('Longitude' in dataset['json_row']['data'], 'data.Longitude should be in json_row')

            # ensure that we can map the Lat/Long to geojson, as this json has
            # unicode values for Lat/Long

            # update the minerva metadata with coordinate mapping
            metadata = {'minerva': dataset}
            metadata['minerva']['mapper'] = {
                "latitudeKeypath": "data.Latitude",
                "longitudeKeypath": "data.Longitude"
            }

            path = '/item/{}/metadata'.format(dataset['dataset_id'])
            response = self.request(
                path=path,
                method='PUT',
                user=self._user,
                body=json.dumps(metadata),
                type='application/json'
            )
            metadata = response.json

            # create geojson in the dataset
            path = '/minerva_dataset/{}/geojson'.format(dataset['dataset_id'])
            response = self.request(
                path=path,
                method='POST',
                user=self._user,
            )
            self.assertHasKeys(response.json, ['geojson_file'])
Exemplo n.º 56
0
def main(config):
    client = GirderClient(apiUrl=config.girder_api_url)
    client.authenticate(config.girder_user,
                        config.girder_password)

    try:
        # First run the simple flow
        print ('Running simple taskflow ...')
        taskflow_id = create_taskflow(
            client, 'cumulus.taskflow.core.test.mytaskflows.SimpleTaskFlow')

        # Start the task flow
        url = 'taskflows/%s/start' % (taskflow_id)
        client.put(url)

        # Wait for it to complete
        wait_for_complete(client, taskflow_id)

        # First run the simple flow
        print ('Running linked taskflow ...')
        taskflow_id = create_taskflow(
            client, 'cumulus.taskflow.core.test.mytaskflows.LinkTaskFlow')

        # Start the task flow
        url = 'taskflows/%s/start' % (taskflow_id)
        client.put(url)

        # Wait for it to complete
        wait_for_complete(client, taskflow_id)

        # Test terminating a simple flow
        print ('Running simple taskflow ...')
        taskflow_id = create_taskflow(
            client, 'cumulus.taskflow.core.test.mytaskflows.SimpleTaskFlow')

        # Start the task flow
        url = 'taskflows/%s/start' % (taskflow_id)
        client.put(url)

        time.sleep(4)

        print ('Terminate the taskflow')
        url = 'taskflows/%s/terminate' % (taskflow_id)
        client.put(url)

        # Wait for it to terminate
        wait_for_terminated(client, taskflow_id)

        # Now delete it
        print ('Delete the taskflow')
        url = 'taskflows/%s' % (taskflow_id)
        try:
            client.delete(url)
        except HttpError as ex:
            if ex.status != 202:
                raise

        # Wait for it to terminate
        wait_for_deletion(client, taskflow_id)


        # Now try something with a chord
        print ('Running taskflow containing a chord ...')
        taskflow_id = create_taskflow(
            client, 'cumulus.taskflow.core.test.mytaskflows.ChordTaskFlow')

        # Start the task flow
        url = 'taskflows/%s/start' % (taskflow_id)
        client.put(url)

        # Wait for it to complete
        wait_for_complete(client, taskflow_id)

        # Now try a workflow that is the two connected together
        print ('Running taskflow that connects to parts together ...')
        taskflow_id = create_taskflow(
            client, 'cumulus.taskflow.core.test.mytaskflows.ConnectTwoTaskFlow')

        # Start the task flow
        url = 'taskflows/%s/start' % (taskflow_id)
        client.put(url)

        # Wait for it to complete
        wait_for_complete(client, taskflow_id)

#      # Now try a composite workflow approach ...
#        print ('Running taskflow that is a composite ...')
#        taskflow_id = create_taskflow(
#            client, 'cumulus.taskflow.core.test.mytaskflows.MyCompositeTaskFlow')
#
#        # Start the task flow
#        url = 'taskflows/%s/start' % (taskflow_id)
#        client.put(url)
#
#        # Wait for it to complete
#        wait_for_complete(client, taskflow_id)

    except HttpError as ex:
        print( ex.responseText)
Exemplo n.º 57
0
import sys


parser = argparse.ArgumentParser()
parser.add_argument("path", type=str,
                    help="path to Arbor web apps")
parser.add_argument("-g", "--girder-host", type=str, default='localhost',
                    help="host to Girder instance")
parser.add_argument("-p", "--girder-port", type=int, default=9000,
                    help="port to Girder instance")

args = parser.parse_args()


# Get the ID for our Analyses folder.
c = GirderClient(host=args.girder_host, port=args.girder_port)
c.authenticate('girder', 'girder')
folderSearch = c.get('resource/search', parameters={
    'q': 'Analyses',
    'types': '["folder"]'
})
folderId = folderSearch['folder'][0]['_id']

# Disable authorization requirements for running romanesco tasks
c.put('system/setting', parameters={
    'key': 'flow.require_auth',
    'value': 'false'
})

# Check if these analyses already exist.  If so, we won't re-upload them.
uploadACR = False
Exemplo n.º 58
0
def upload_path(cluster_connection, girder_token, folder_id, path):
    girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl)
    girder_client.token = girder_token
    cluster_connection.makedirs(path)

    _upload_path(cluster_connection, girder_client, folder_id, path)