def upload_job_output_to_folder(cluster, job, log_write_url=None, job_dir=None,
girder_token=None):
status_url = '%s/jobs/%s' % (cumulus.config.girder.baseUrl, job['_id'])
headers = {'Girder-Token': girder_token}
assetstore_base_url = get_assetstore_url_base(cluster)
assetstore_id = get_assetstore_id(girder_token, cluster)
if not job_dir:
job_dir = job['dir']
try:
with get_connection(girder_token, cluster) as conn:
for output in job['output']:
if 'folderId' in output and 'path' in output:
folder_id = output['folderId']
path = os.path.join(job_dir, output['path'])
download_path(conn, girder_token, folder_id, path,
assetstore_base_url, assetstore_id)
except HttpError as e:
job['status'] = JobState.ERROR
url = '%s/jobs/%s/log' % (cumulus.config.girder.baseUrl, job['_id'])
logger = get_post_logger('job', girder_token, url)
logger.exception(e.responseText)
r = requests.patch(status_url, headers=headers,
json={'status': JobState.ERROR})
check_status(r)
if _get_on_complete(job) == 'terminate':
cluster_log_url = '%s/clusters/%s/log' % \
(cumulus.config.girder.baseUrl, cluster['_id'])
command.send_task(
'cumulus.tasks.cluster.terminate_cluster',
args=(cluster,), kwargs={'log_write_url': cluster_log_url,
'girder_token': girder_token})
# If we where uploading move job to the complete state
if job['status'] == JobState.UPLOADING:
job_status = from_string(job['status'], task=None,
cluster=cluster, job=job,
log_write_url=log_write_url,
girder_token=girder_token,
conn=conn)
job_status = Complete(job_status)
job_status = job_status.next(JobQueueState.COMPLETE)
job_status.run()
r = requests.patch(status_url, headers=headers,
json={'status': str(job_status)})
check_status(r)
def upload_job_output_to_folder(cluster, job, log_write_url=None, job_dir=None,
girder_token=None):
status_url = '%s/jobs/%s' % (cumulus.config.girder.baseUrl, job['_id'])
headers = {'Girder-Token': girder_token}
assetstore_base_url = get_assetstore_url_base(cluster)
assetstore_id = get_assetstore_id(girder_token, cluster)
if not job_dir:
job_dir = job['dir']
try:
with get_connection(girder_token, cluster) as conn:
for output in job['output']:
if 'folderId' in output and 'path' in output:
folder_id = output['folderId']
path = os.path.join(job_dir, output['path'])
download_path(conn, girder_token, folder_id, path,
assetstore_base_url, assetstore_id)
except HttpError as e:
job['status'] = JobState.ERROR
url = '%s/jobs/%s/log' % (cumulus.config.girder.baseUrl, job['_id'])
logger = get_post_logger('job', girder_token, url)
logger.exception(e.responseText)
r = requests.patch(status_url, headers=headers,
json={'status': JobState.ERROR})
check_status(r)
if _get_on_complete(job) == 'terminate':
cluster_log_url = '%s/clusters/%s/log' % \
(cumulus.config.girder.baseUrl, cluster['_id'])
command.send_task(
'cumulus.tasks.cluster.terminate_cluster',
args=(cluster,), kwargs={'log_write_url': cluster_log_url,
'girder_token': girder_token})
# If we where uploading move job to the complete state
if job['status'] == JobState.UPLOADING:
job_status = from_string(job['status'], task=None,
cluster=cluster, job=job,
log_write_url=log_write_url,
girder_token=girder_token,
conn=conn)
job_status = Complete(job_status)
job_status = job_status.next(JobQueueState.COMPLETE)
job_status.run()
r = requests.patch(status_url, headers=headers,
json={'status': str(job_status)})
check_status(r)
def test_download(self):
cluster_connection = mock.MagicMock()
cluster_connection.list.side_effect = self.paths
download_path(cluster_connection,
self._token['_id'],
self._folder['_id'],
'/test',
'sftp_assetstores',
self._assetstore['_id'],
upload=False)
files = []
folders = []
for (p, _) in self.model('folder').fileList(self._folder,
user=self._user,
path='/',
data=False):
files.append(p)
def traverse_folders(folder, path='/Public'):
for f in self.model('folder').childFolders(folder,
'folder',
user=self._user):
print('trav: path: %s' % (path))
current_path = os.path.join(path, f['name'])
print('trav: path: %s' % (path))
folders.append(current_path)
traverse_folders(f, path=current_path)
traverse_folders(self._folder)
files.sort()
folders.sort()
expected_path = os.path.abspath(
'plugins/cumulus/plugin_tests/fixtures/expected_paths.json')
with open(expected_path) as fp:
expected = json.load(fp)
self.assertEqual(files, expected['files'])
self.assertEqual(folders, expected['folders'])
def test_import_path(self):
file = {
'name': 'test.txt',
'mode': 1,
'size': 123
}
folder = {
'name': 'folder',
'mode': stat.S_IFDIR,
'size': 1234
}
cluster_connection = mock.MagicMock()
cluster_connection.list.side_effect = [[file, folder], [file]]
girder_token = 'dummy'
parent = {
'_id': 'dummy_id'
}
path = '/my/path'
assetstore_id = 'dummy_id'
# Mock create item
def _create_item(url, request):
content = {
'_id': 'dummy'
}
content = json.dumps(content).encode('utf8')
headers = {
'content-length': len(content),
'content-type': 'application/json'
}
self._item_requests.append(request)
return httmock.response(200, content, headers, request=request)
item_url = '/api/v1/item'
create_item = httmock.urlmatch(
path=r'^%s$' % item_url, method='POST')(_create_item)
# Mock create file
def _create_file(url, request):
content = {
'_id': 'dummy'
}
content = json.dumps(content).encode('utf8')
headers = {
'content-length': len(content),
'content-type': 'application/json'
}
self._file_requests.append(request)
return httmock.response(200, content, headers, request=request)
file_url = '/api/v1/sftp_assetstores/%s/files' % assetstore_id
create_file = httmock.urlmatch(
path=r'^%s$' % file_url, method='POST')(_create_file)
# Mock create folder
def _create_folder(url, request):
content = {
'_id': 'dummy'
}
content = json.dumps(content).encode('utf8')
headers = {
'content-length': len(content),
'content-type': 'application/json'
}
self._folder_requests.append(request)
return httmock.response(200, content, headers, request=request)
folder_url = '/api/v1/folder'
create_folder = httmock.urlmatch(
path=r'^%s$' % folder_url, method='POST')(_create_folder)
# Mock list folder
def _list_folder(url, request):
content = []
content = json.dumps(content).encode('utf8')
headers = {
'content-length': len(content),
'content-type': 'application/json'
}
return httmock.response(200, content, headers, request=request)
folder_url = '/api/v1/folder'
list_folder = httmock.urlmatch(
path=r'^%s$' % folder_url, method='GET')(_list_folder)
with httmock.HTTMock(create_item, create_file, create_folder, list_folder):
download_path(cluster_connection, girder_token, parent, path,
'sftp_assetstores', assetstore_id, upload=False)
self.assertEqual(len(self._item_requests), 2)
self.assertEqual(len(self._file_requests), 2)
self.assertEqual(len(self._folder_requests), 1)
