def download_path(cluster_connection,
girder_token,
parent,
path,
assetstore_url,
assetstore_id,
upload=False,
include=None,
exclude=None):
"""
Download a given path on a cluster into an assetstore.
:params cluster_connection: The cluster connection to access the cluster.
:params girder_token: The Girder token to use to access Girder.
:params parent: The target folder to import the path into.
:params path: The path on the cluster to download.
:params assetstore_url: The url for the assetstore to use for the import.
:params assetstore_id: The id of the asseststore to import into.
:params upload: Indicate if the import should upload the file data or just
the metadata, the default is False.
:params include: List of include regexs
:params exclude: List of exclude regexs,
"""
girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl)
girder_client.token = girder_token
_import_path(cluster_connection,
girder_client,
parent,
path,
assetstore_url,
assetstore_id,
upload=upload,
include=include,
exclude=exclude)
def __init__(self, girder_url, newt_sessionid):
'''
'''
self._client = None
self._cluster_id = None
self._girder_url = girder_url
self._input_folder_id = None
self._job_folder_id = None
self._job_id = None
self._output_folder_id = None
self._private_folder_id = None
self._script_id = None
self._session = requests.Session()
# Authenticate with Girder using the newt session id
url = '%s/api/v1/newt/authenticate/%s' % \
(self._girder_url, newt_sessionid)
r = self._session.put(url)
if r.status_code != 200:
raise HttpError(r.status_code, r.text, r.url, r.request.method)
# Instantiate Girder client
url = '%s/api/v1' % self._girder_url
self._client = GirderClient(apiUrl=url)
self._client.token = self._session.cookies['girderToken']
user = self._client.get('user/me')
#print 'user', user
user_id = user['_id']
r = self._client.listFolder(user_id, 'user', name='Private')
if len(r) != 1:
raise Exception('Wrong number of users; should be 1 got %s' %
len(r))
self._private_folder_id = r[0]['_id']
print 'private_folder_id', self._private_folder_id
def setUp(self):
# First authenticate with NEWT
self._session = Session()
r = self._session.post('https://newt.nersc.gov/newt/auth', {
'username': self._girder_user,
'password': self._girder_password
})
self.assertEqual(r.status_code, 200)
print r.json()
self._newt_session_id = r.json()['newt_sessionid']
# Now authenticate with Girder using the session id
url = '%s/api/v1/newt/authenticate/%s' % (self._girder_url,
self._newt_session_id)
r = self._session.put(url)
self.assertEqual(r.status_code, 200)
url = '%s/api/v1/newt/authenticate/%s' % (self._girder_url,
self._newt_session_id)
r = self._session.put(url)
self.assertEqual(r.status_code, 200)
url = '%s/api/v1' % self._girder_url
self._client = GirderClient(apiUrl=url)
self._client.token = self._session.cookies['girderToken']
user = self._client.get('user/me')
self._user_id = user['_id']
r = self._client.listFolder(self._user_id, 'user', name='Private')
r = list(r)
self.assertEqual(len(r), 1)
self._private_folder_id = r[0]['_id']
def main():
parser = argparse.ArgumentParser(
description='Import analyses into minerva')
parser.add_argument('--username', required=False, default=None)
parser.add_argument('--password', required=False, default=None)
parser.add_argument('--scheme', required=False, default='http')
parser.add_argument('--host', required=False, default='localhost')
parser.add_argument('--port', required=False, default='8080')
parser.add_argument('--api-root',
required=False,
default='/api/v1',
help='path to the Girder REST API')
parser.add_argument('--path',
required=True,
help='the path to import the analyses from')
config = parser.parse_args()
client = GirderClient(host=config.host,
port=config.port,
apiRoot=config.api_root,
scheme=config.scheme)
client.authenticate(config.username, config.password)
import_analyses(client, config.path)
def main(config):
client = GirderClient(apiUrl=config.girder_api_url)
client.authenticate(config.girder_user,
config.girder_password)
# Load any parameters
params = {}
if config.taskflow_start_params is not None:
with open(config.taskflow_start_params) as fp:
params = json.load(fp)
print params
try:
print ('Running %s taskflow ...' % config.taskflow_start_params)
taskflow_id = create_taskflow(
client, config.taskflow_class)
# Start the task flow
url = 'taskflows/%s/start' % (taskflow_id)
client.put(url, data=json.dumps(params))
# Wait for it to complete
wait_for_complete(client, taskflow_id)
except HttpError as ex:
print( ex.responseText)
def _update_girder(taskflow, body):
taskflow = to_taskflow(taskflow)
taskflow_id = taskflow['id']
girder_token = taskflow['girder_token']
girder_api_url = taskflow['girder_api_url']
client = GirderClient(apiUrl=girder_api_url)
client.token = girder_token
client = _create_girder_client(girder_api_url, girder_token)
# If this is a retry then we have already create a task get it from
# the current tasks headers.
if body['retries'] > 0:
taskflow_task_id = \
current_task.request.headers[TASKFLOW_TASK_ID_HEADER]
# Celery always fires the postrun handler with a state of SUCCESS
# for retries. So we need to save the retries here so we can
# determine in the postrun handler if the task is really complete.
current_task.request.headers[TASKFLOW_RETRY_HEADER] \
= body['retries']
else:
# This is a new task so create a taskflow task instance
body = {'celeryTaskId': body['id'], 'name': body['task']}
url = 'taskflows/%s/tasks' % taskflow_id
r = client.post(url, data=json.dumps(body))
taskflow_task_id = r['_id']
return taskflow, taskflow_task_id
def test_sanity_checks():
gc = GirderClient(apiUrl=source_api_root)
resp = gc.get('/', jsonResp=False)
assert resp.status_code == 200
assert resp.headers['Content-Type'] == 'text/html;charset=utf-8'
resp = gc.get('system/check', jsonResp=False)
assert resp.status_code == 200
def get_girder_client() -> GirderClient:
client = GirderClient(apiUrl="https://viame.kitware.com/api/v1")
apikey = os.environ.get('GIRDER_API_KEY', None)
if apikey:
client.authenticate(apiKey=apikey)
else:
client.authenticate(interactive=True)
return client
def import_calc(config):
try:
target_port = None
if config.port:
target_port = config.port
target_scheme = None
if config.scheme:
target_scheme = config.scheme
target_apiroot = None
if config.apiroot:
target_apiroot = config.apiroot
client = GirderClient(host=config.host, port=target_port,
scheme=target_scheme, apiRoot=target_apiroot)
client.authenticate(apiKey=config.apiKey)
me = client.get('/user/me')
if not me:
print('Error: Girder token invalid, please verify')
return
folderParams = {
'parentId': me['_id'],
'parentType': 'user',
'name': 'Private'
}
# Get the private folder id first
folder = next(client.listResource('folder', folderParams))
folder = next(client.listFolder(me['_id'], 'user', 'Private'))
for file_name in config.datafile:
print ('\nUploading ' + file_name)
file_id = {}
with open(file_name, 'r') as fp:
fileNameBase = os.path.basename(file_name)
size = os.path.getsize(file_name)
file_id = client.uploadFile(folder['_id'], fp, fileNameBase,
size, 'folder')
body = {
'fileId': file_id['_id']
}
if config.public:
body['public'] = True
mol = client.sendRestRequest('POST', 'molecules', data=json.dumps(body))
if mol and '_id' in mol:
config.moleculeId = mol['_id']
print('Molecule ID: ' + mol['_id'])
else:
print(mol)
except HttpError as error:
print(error.responseText, file=sys.stderr)
def _taskflow_task_finished(taskflow, taskflow_task_id):
girder_token = taskflow['girder_token']
girder_api_url = taskflow['girder_api_url']
client = GirderClient(apiUrl=girder_api_url)
client.token = girder_token
url = 'taskflows/%s/tasks/%s/finished' % (taskflow.id, taskflow_task_id)
return client.put(url)
def test_extract_download(data):
fileId, filename = data
filepath = localDataRoot / str(filename)
if not filepath.exists():
gc = GirderClient(apiUrl=source_api_root)
gc.authenticate(apiKey=os.environ.get('GIRDER_API_KEY'))
gc.downloadFile(fileId, str(filepath))
with zipfile.ZipFile(filepath, 'r') as zipref:
zipref.extractall(localDataRoot)
def main(ctx, api_key, api_url):
"""Openchemistry Client
The client can be used to fetch molecules, add molecules, etc.
"""
gc = GirderClient(apiUrl=api_url)
if api_key is not None:
gc.authenticate(apiKey=api_key)
ctx.obj = gc
def setUp(self):
url = '%s/api/v1' % self._girder_url
self._client = GirderClient(apiUrl=url)
self._client.authenticate(self._girder_user, self._girder_password)
user = self._client.get('user/me')
self._user_id = user['_id']
r = list(self._client.listFolder(self._user_id, 'user',
name='Private'))
self.assertEqual(len(r), 1)
self._private_folder_id = r[0]['_id']
def gc_init(api_url='https://girder.hub.yt/api/v1', api_key=None):
from girder_client import GirderClient
if api_key is None:
try:
api_key = os.environ['GIRDER_API_KEY']
except KeyError as err:
msg = 'please define GIRDER_API_KEY environment variable\n'
msg += ' or pass api_key kwargs.'
raise KeyError(msg)
gc = GirderClient(apiUrl=api_url)
gc.authenticate(apiKey=api_key)
return gc
def _ingest(project, composite, dir, channel_map, api_url, api_key):
if dir[-1] != '/':
dir += '/'
gc = GirderClient(apiUrl=api_url)
gc.authenticate(apiKey=api_key)
channel_map = json.load(channel_map)
channel_map = {channel.upper():element.lower() for (channel,element) in channel_map.items()}
experiments = _ingest_runs(gc, project, composite, dir)
#(run_ids, runs) = _ingest_runs(gc, project, composite, dir)
_ingest_samples(gc, project, composite, dir, experiments, channel_map)
def main(args=None):
parser = argparse.ArgumentParser(
description='Mount Girder filesystem assetstore.')
parser.add_argument('--api-url',
required=True,
default=None,
help='full URL to the RESTful API of Girder server')
parser.add_argument('--username', required=False, default=None)
parser.add_argument('--password', required=False, default=None)
parser.add_argument('--api-key', required=False, default=None)
parser.add_argument('--token', required=False, default=None)
parser.add_argument('-c',
default='remote',
choices=['remote', 'direct'],
help='command to run')
parser.add_argument('--foreground', dest='foreground', action='store_true')
parser.add_argument('--hostns', dest='hostns', action='store_true')
parser.add_argument('local_folder', help='path to local target folder')
parser.add_argument('remote_folder', help='Girder\'s folder id')
args = parser.parse_args()
gc = GirderClient(apiUrl=args.api_url)
if args.token:
gc.token = args.token
elif args.api_key:
gc.authenticate(apiKey=args.api_key)
elif args.username and args.password:
gc.authenticate(username=args.username, password=args.password)
else:
raise RuntimeError("You need to specify apiKey or user/pass")
if args.hostns:
targetns = os.path.join(os.environ.get('HOSTDIR', '/'),
'proc/1/ns/mnt')
with open(targetns) as fd:
setns(fd, CLONE_NEWNS)
if args.c == 'remote':
FUSE(RESTGirderFS(args.remote_folder, gc),
args.local_folder,
foreground=args.foreground,
ro=True,
allow_other=True)
elif args.c == 'direct':
FUSE(LocalGirderFS(args.remote_folder, gc),
args.local_folder,
foreground=args.foreground,
ro=True,
allow_other=True)
else:
print('No implementation for command %s' % args.c)
def upload_file(cluster, girder_token, file, path):
"""
Upload a file to a cluster
:param cluster: The cluster to upload to.
:param girder_tokebn: The Grider token for Girder access.
:param file: The Girder file object.
:param path: The path on the cluster to upload to.
"""
girder_client = GirderClient(apiUrl=cumulus.config.girder.baseUrl)
girder_client.token = girder_token
with get_connection(girder_token, cluster) as conn:
conn.makedirs(os.path.dirname(path))
_upload_file(conn, girder_client, file, path)
def testClientMetadataExtractor(testData, user):
item = Item().load(testData['item']['_id'], user=user)
assert item['name'] == testData['name']
del item['meta']
item = Item().save(item)
assert 'meta' not in item
client = GirderClient('localhost', int(os.environ['GIRDER_PORT']))
client.authenticate(user['login'], 'password')
extractor = ClientMetadataExtractor(client, testData['path'], testData['item']['_id'])
extractor.extractMetadata()
item = Item().load(testData['item']['_id'], user=user)
assert item['name'] == testData['name']
assert 'meta' in item
assert item['meta']['MIME type'] == testData['mimeType']
def init_girder(api_key=None, api_url='https://girder.hub.yt/api/v1'):
"""Initialize girder client, rely on environment variable:
GIRDER_API_KEY, connect to yt Hub.
Args:
api_key (str, optional): use GIRDER_API_KEY env. var. by default
api_url (str, optional): use yt Hub v1 by default
Return:
GirderClient: initialized girder client
"""
from girder_client import GirderClient
if api_key is None:
import os
api_key = os.environ['GIRDER_API_KEY']
gc = GirderClient(apiUrl=api_url)
gc.authenticate(apiKey=api_key)
return gc
def upload_benchmark_results(benchmark_bin, api_key=None):
hostname = socket.gethostname().lower()
results_dir = os.path.join(benchmark_bin, 'BenchmarkResults', hostname)
if not os.path.exists(results_dir):
sys.stderr.write('Expected results directory does not exist: ' +
results_dir)
sys.exit(1)
from girder_client import GirderClient
gc = GirderClient(apiUrl='https://data.kitware.com/api/v1')
gc.authenticate(apiKey=api_key)
# ITK/PerformanceBenchmarkingResults
folder_id = '5af50c818d777f06857985e3'
hostname_folder = gc.loadOrCreateFolder(hostname, folder_id, 'folder')
gc.upload(os.path.join(results_dir, '*.json'),
hostname_folder['_id'],
leafFoldersAsItems=False,
reuseExisting=True)
def create_movie(id):
token = request.headers.get('girderToken', '')
if not token or not id:
return Response('Invalid token or parameter ID.', status=400)
gc = GirderClient(apiUrl=girder_url)
gc.setToken(token)
with tempfile.TemporaryDirectory() as tmpdir:
gc.downloadItem(id, tmpdir)
item_name = os.listdir(tmpdir)[0]
path_name = os.path.join(tmpdir, item_name, '*.svg')
output_file = tempfile.NamedTemporaryFile(suffix='.mp4')
(ffmpeg
.input(path_name, pattern_type='glob', framerate=10)
.output(output_file.name)
.overwrite_output()
.run())
return send_file(output_file, mimetype='mp4')
def testClientMetadataExtractor(self):
item = Item().load(self.item['_id'], user=self.user)
self.assertEqual(item['name'], self.name)
self.assertNotHasKeys(item, ['meta'])
clientPath = os.path.join(ROOT_DIR, 'clients', 'python')
sys.path.insert(0, clientPath)
from girder_client import GirderClient
client = GirderClient('localhost', int(os.environ['GIRDER_PORT']))
client.authenticate(self.user['login'], self.password)
extractor = ClientMetadataExtractor(client, self.path,
self.item['_id'])
extractor.extractMetadata()
sys.path.remove(clientPath)
item = Item().load(self.item['_id'], user=self.user)
self.assertEqual(item['name'], self.name)
self.assertHasKeys(item, ['meta'])
self.assertEqual(item['meta']['MIME type'], self.mimeType)
def gw_task_prerun(task=None,
sender=None,
task_id=None,
args=None,
kwargs=None,
**rest):
"""Deserialize the jobInfoSpec passed in through the headers.
This provides the a JobManager class as an attribute of the
task before task execution. decorated functions may bind to
their task and have access to the job_manager for logging and
updating their status in girder.
"""
if is_builtin_celery_task(sender.name):
return
try:
task.job_manager = _job_manager(task.request, task.request.headers)
_update_status(task, JobStatus.RUNNING)
except JobSpecNotFound:
task.job_manager = None
logger.warn('No jobInfoSpec. Setting job_manager to None.')
except StateTransitionException:
# Fetch the current status of the job
status = task.job_manager.refreshStatus()
# If we are canceling we want to stay in that state
if status != JobStatus.CANCELING:
raise
try:
task.girder_client = GirderClient(apiUrl=task.request.girder_api_url)
task.girder_client.token = task.request.girder_client_token
except AttributeError:
task.girder_client = None
# Deserialize girder_result_hooks if they exist
if hasattr(task.request, 'girder_result_hooks'):
u = jsonpickle.unpickler.Unpickler()
task.request.girder_result_hooks = \
[u.restore(grh) for grh in task.request.girder_result_hooks]
def test_upload_zip_data(dataset: dict):
user = zipUser
client = GirderClient(apiUrl='http://localhost:8010/api/v1')
client.authenticate(username=user['login'], password=user['password'])
dsPath = localDataRoot / str(dataset['path'])
privateFolder = getTestFolder(client)
newDatasetFolder = client.createFolder(
privateFolder['_id'],
dataset['name'],
metadata={
'fps': dataset['fps'],
'type': dataset['type'],
},
)
if Path(dsPath).is_file():
client.uploadFileToFolder(newDatasetFolder['_id'], str(dsPath))
client.post(f'dive_rpc/postprocess/{newDatasetFolder["_id"]}')
wait_for_jobs(client,
max_wait_timeout=30,
expected_status=dataset['job_status'])
resultFolder = client.getFolder(newDatasetFolder['_id'])
# verify sub datasets if they exist
if dataset.get('subDatasets', False):
folders = list(client.listFolder(newDatasetFolder['_id']))
for item in dataset["subDatasets"]:
matches = [x for x in folders if x["name"] == item["name"]]
if len(matches) > 0:
meta = matches[0].get("meta", {})
assert meta.get("fps", -1) == item["fps"]
assert meta.get("type", "") == item["type"]
assert meta.get("annotate", False)
elif dataset['job_status'] == JobStatus.SUCCESS:
assert resultFolder['meta'].get("annotate", False)
assert type(resultFolder['meta'].get("fps")) in [int, float]
assert type(resultFolder['meta'].get("type")) == str
else:
assert resultFolder['meta'].get("annotate", None) is None
def __init__(self, *args, **kwargs):
gc = kwargs.pop('gc', None)
try:
if gc is None:
# We need to resolve Girder's API URL, but girder_worker can
# specify a different value than what Girder gets from a rest
# request.
# Girder 3
try:
from girder_worker.girder_plugin.utils import getWorkerApiUrl
except ImportError:
# Girder 2
try:
from girder.plugins.worker.utils import getWorkerApiUrl
# Fall back if the worker plugin is unavailble
except ImportError:
from girder.api.rest import getApiUrl as getWorkerApiUrl
self.gc = GirderClient(apiUrl=getWorkerApiUrl())
from girder.api.rest import getCurrentUser
if getCurrentUser():
from girder.constants import TokenScope
from girder.models.token import Token
token = Token().createToken(
days=7,
scope=[TokenScope.DATA_READ, TokenScope.DATA_WRITE],
user=getCurrentUser(),
)['_id']
else:
from girder.api.rest import getCurrentToken
token = getCurrentToken()['_id']
self.gc.token = token
else:
self.gc = gc
except ImportError:
self.gc = None
from girder_client import GirderClient
import json
import pymongo
import sys
if len(sys.argv) < 2:
print "%s /path/to/ArborWebApps" % sys.argv[0]
sys.exit(1)
arborWebAppsPath = sys.argv[1]
# Get the ID for our Analyses folder.
c = GirderClient(host='localhost', port=9000)
c.authenticate('girder', 'girder')
folderSearch = c.get('resource/search',
parameters={
'q': 'Analyses',
'types': '["folder"]'
})
folderId = folderSearch['folder'][0]['_id']
# Disable authorization requirements for running romanesco tasks
c.put('system/setting',
parameters={
'key': 'romanesco.require_auth',
'value': 'false'
})
# Check if these analyses already exist. If so, we won't re-upload them.
uploadACR = False
uploadPGS = False
parser.add_argument("path", type=str, help="path to Arbor web apps")
parser.add_argument("-g",
"--girder-host",
type=str,
default='localhost',
help="host to Girder instance")
parser.add_argument("-p",
"--girder-port",
type=int,
default=9000,
help="port to Girder instance")
args = parser.parse_args()
# Get the ID for our Analyses folder.
c = GirderClient(host=args.girder_host, port=args.girder_port)
c.authenticate('girder', 'girder')
folderSearch = c.get('resource/search',
parameters={
'q': 'Analyses',
'types': '["folder"]'
})
folderId = folderSearch['folder'][0]['_id']
# Disable authorization requirements for running romanesco tasks
c.put('system/setting',
parameters={
'key': 'flow.require_auth',
'value': 'false'
})
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################
from girder_client import GirderClient
if __name__ == "__main__":
login = '******'
password = '******'
gc = GirderClient(apiUrl='http://*****:*****@admin.com',
firstName='admin',
lastName='admin',
password=password,
admin=True)
gc.authenticate(username=login, password=password)
# Create an assetstore
gc.post('assetstore',
parameters={
'name': 'TestAssetstore',
'type': 0,
def girder_client(request, api_url):
username, password = request.param
client = GirderClient(apiUrl=api_url)
client.authenticate(username, password)
yield client
return result
parser = ArgumentParser(description='Initialize the girder environment')
parser.add_argument('--admin', help='name:pass for the admin user')
parser.add_argument('--host', help='host to connect to')
parser.add_argument('--port', type=int, help='port to connect to')
parser.add_argument('--broker', help='girder worker broker URI')
parser.add_argument('--s3', help='name of S3 bucket')
parser.add_argument('--aws-key-id', help='aws key id')
parser.add_argument('--aws-secret-key', help='aws secret key')
args = parser.parse_args()
client = GirderClient(host=args.host, port=args.port)
user, password = args.admin.split(":", 1)
if find_user('girder'):
client.authenticate('girder', 'girder')
ensure_user(client,
login=user,
password=password,
email='*****@*****.**',
firstName='Girder',
lastName='Admin')
client.authenticate(user, password)
