def copy_datafiles_to_collab_storage(request, job, local_dir, relative_paths):
# upload local files to collab storage
from bbp_client.oidc.client import BBPOIDCClient
from bbp_client.document_service.client import Client as DocClient
import bbp_services.client as bsc
services = bsc.get_services()
access_token = get_access_token(request.user.social_auth.get())
oidc_client = BBPOIDCClient.bearer_auth(
services['oidc_service']['prod']['url'], access_token)
doc_client = DocClient(services['document_service']['prod']['url'],
oidc_client)
project = doc_client.get_project_by_collab_id(job.collab_id)
root = doc_client.get_path_by_id(project["_uuid"])
collab_folder = root + "/job_{}".format(job.pk)
folder_id = doc_client.mkdir(collab_folder)
collab_paths = []
for relative_path in relative_paths:
collab_path = os.path.join(collab_folder, relative_path)
if os.path.dirname(relative_path): # if there are subdirectories...
doc_client.makedirs(os.path.dirname(collab_path))
local_path = os.path.join(local_dir, relative_path)
id = doc_client.upload_file(local_path, collab_path)
collab_paths.append(collab_path)
content_type = mimetypes.guess_type(local_path)[0]
if content_type:
doc_client.set_standard_attr(collab_path,
{'_contentType': content_type})
return collab_paths
def copy_datafiles_with_unicore(request, target, job, local_dir,
relative_paths):
from simqueue import unicore_client
url = unicore_client.get_site(target)['url']
access_token = get_access_token(request.user.social_auth.get())
auth = unicore_client.get_oidc_auth("Bearer {}".format(access_token))
headers_query = auth.copy()
headers_query['Accept'] = 'application/json'
storages = requests.get(url + '/storages',
headers=headers_query,
verify=False).json()['storages']
home_url = [st for st in storages if "home" in st.lower()][0] + '/files'
headers_upload = auth.copy()
headers_upload['Content-Type'] = "application/octet-stream"
# upload files
remote_dir = home_url + "/neuromorphic/job_{}/".format(job.pk)
remote_paths = []
for relative_path in relative_paths:
remote_path = os.path.join(remote_dir, relative_path)
local_path = os.path.join(local_dir, relative_path)
with open(local_path, 'rb') as fp:
file_contents = fp.read()
response = requests.put(remote_path,
headers=headers_upload,
data=file_contents,
verify=False)
if response.status_code != 204:
# bail out early
break
remote_paths.append(remote_path)
return remote_paths
def get_related_data(self, user):
# assume for now that data is in collab
from bbp_client.oidc.client import BBPOIDCClient
from bbp_client.document_service.client import Client as DocClient
import bbp_services.client as bsc
services = bsc.get_services()
access_token = get_access_token(user.social_auth.get())
oidc_client = BBPOIDCClient.bearer_auth(
services['oidc_service']['prod']['url'], access_token)
doc_client = DocClient(services['document_service']['prod']['url'],
oidc_client)
parse_result = urlparse(self.object.results_storage)
if parse_result.scheme == "collab":
collab_folder = parse_result.path
#return doc_client.listdir(collab_folder)
folder_uuid = doc_client.get_standard_attr(collab_folder)['_uuid']
data = {
"folder": {
"path": collab_folder,
}
}
if self.object.project:
data["folder"]["url"] = self.get_collab_storage_url(
) + "?state=uuid={}".format(folder_uuid)
return data
else:
print("Storage not yet supported")
return {}
def copy_code_file_from_collab_storage(self, bundle):
"""
Download code from Collab storage
"""
from hbp_service_client.document_service.client import Client as DocClient
joinp = os.path.join
access_token = get_access_token(bundle.request.user.social_auth.get())
doc_client = DocClient.new(access_token)
entity_id = bundle.data.get("code")
metadata = doc_client.get_entity_details(entity_id)
ext = metadata['name'].split('.')[-1]
entity_name = "{}_{}".format(entity_id, metadata["name"])
entity_type = metadata["entity_type"]
local_dir = settings.TMP_FILE_ROOT
if entity_type == 'file':
# put the file content into the "code" field directly
etag, content = doc_client.download_file_content(entity_id)
if ext == 'ipynb':
content = self.filter_ipynb_content(content)
elif ext in ('zip', 'tgz', 'gz'):
with open(os.path.join(local_dir, entity_name), 'wb') as fp:
fp.write(content)
temporary_url = bundle.request.build_absolute_uri(settings.TMP_FILE_URL + entity_name)
content = temporary_url
# logger.debug(content)
return content
elif entity_type == 'folder':
# create a zip archive and put its url into the "code" field
def download_folder(entity, local_dir):
folder_content = doc_client.list_folder_content(entity["uuid"])
sub_dir = joinp(local_dir, entity["name"])
if os.path.exists(sub_dir):
shutil.rmtree(sub_dir)
os.mkdir(sub_dir)
for child in folder_content['results']:
entity_type = child["entity_type"]
if entity_type == 'file':
with open(joinp(sub_dir, child["name"]), "wb") as fp:
etag, content = doc_client.download_file_content(child["uuid"])
fp.write(content)
elif entity_type == 'folder':
download_folder(child, sub_dir)
else:
logger.warning("Cannot handle entity type '{}'".format(entity_type))
return sub_dir
folder_root = download_folder(metadata, local_dir)
shutil.make_archive(joinp(local_dir, entity_name), 'zip',
root_dir=local_dir,
base_dir=metadata["name"])
filename = entity_name + ".zip"
shutil.rmtree(folder_root)
temporary_url = bundle.request.build_absolute_uri(settings.TMP_FILE_URL + filename)
return temporary_url
else:
raise ValueError("Can't handle entity type '{}'".format(entity_type))
def __get_client_token(request):
try:
social_auth = request.user.social_auth.get()
return {
'access_token': get_access_token(social_auth),
'token_type': get_token_type(social_auth),
'expires_in': __get_session_expiry_age(request),
}
except UserSocialAuth.DoesNotExist:
raise exceptions.UserTypeException(request.user)
def copy_datafiles_to_collab_storage(request, job, local_dir, relative_paths):
# upload local files to collab storage
#from bbp_client.oidc.client import BBPOIDCClient
#from bbp_client.document_service.client import Client as DocClient
#import bbp_services.client as bsc
import hbp_service_client.document_service.client as doc_service_client
#services = bsc.get_services()
access_token = get_access_token(request.user.social_auth.get())
#oidc_client = BBPOIDCClient.bearer_auth(services['oidc_service']['prod']['url'], access_token)
#doc_client = DocClient(services['document_service']['prod']['url'], oidc_client)
#dsc = doc_service_client.Client.__init__()
dsc = doc_service_client.Client.new(access_token)
#project = doc_client.get_project_by_collab_id(job.collab_id)
project_dict = dsc.list_projects(None, None, None, job.collab_id)
project = project_dict['results']
#root = doc_client.get_path_by_id(project["_uuid"])
root = dsc.get_entity_path(project[0]['uuid'])
collab_folder = root + "/job_{}".format(job.pk)
collab_folder_2 = "job_{}".format(job.pk)
#folder_id = doc_client.mkdir(collab_folder)
folder = dsc.create_folder(collab_folder_2, str(project[0]['uuid']))
folder_id = folder['uuid']
collab_paths = []
for relative_path in relative_paths:
collab_path = os.path.join(collab_folder, relative_path)
cps = collab_path.split('/')
collab_path_2 = cps[3]
if os.path.dirname(relative_path): # if there are subdirectories...
#doc_client.makedirs(os.path.dirname(collab_path))
collab_path_id = dsc.create_folder(collab_path_2, folder_id)
local_path = os.path.join(local_dir, relative_path)
#id = doc_client.upload_file(local_path, collab_path)
#file = dsc.create_file("file_to_upload", "plain/text", collab_path_id['uuid'])
file = dsc.create_file(relative_path, "plain/text", folder_id)
with open(local_path, 'rb') as fp:
file_contents = fp.read()
id = dsc.upload_file_content(file['uuid'], None, None, file_contents)
#logging.warning("successful upload content")
collab_paths.append(collab_path)
content_type = mimetypes.guess_type(local_path)[0]
if content_type:
#doc_client.set_standard_attr(collab_path, {'_contentType': content_type})
os.path.normpath(os.path.join('/', str(collab_path)))
return collab_paths
def upload_zip_file_to_storage(request):
access_token = get_access_token(request.user.social_auth.get())
# retrieve data from request.session
headers = request.session['headers']
collab_id = request.session['collab_id']
st_rel_user_results_folder = request.session['st_rel_user_results_folder']
st_rel_user_uploaded_folder = request.session[
'st_rel_user_uploaded_folder']
crr_user_folder = request.session['time_info']
output_path = request.session['result_file_zip']
context = request.session['context']
services = bsc.get_services()
# get clients from bbp python packages
oidc_client = BBPOIDCClient.bearer_auth(\
services['oidc_service']['prod']['url'], access_token)
bearer_client = BBPOIDCClient.bearer_auth('prod', access_token)
doc_client = DocClient(\
services['document_service']['prod']['url'], oidc_client)
context = request.session['context']
#logout(request)
nextUrl = urllib.quote('%s?ctx=%s' % (request.path, context))
# extract project from collab_id
project = doc_client.get_project_by_collab_id(collab_id)
# extract collab storage root path
storage_root = doc_client.get_path_by_id(project["_uuid"])
crr_collab_storage_folder = os.path.join(storage_root, \
st_rel_user_results_folder)
if not doc_client.exists(crr_collab_storage_folder):
doc_client.makedirs(crr_collab_storage_folder)
# final zip collab storage path
zip_collab_storage_path = os.path.join(crr_collab_storage_folder, \
crr_user_folder + '_results.zip')
# bypassing uploading data to collab storage
if not doc_client.exists(zip_collab_storage_path):
doc_client.upload_file(output_path, zip_collab_storage_path)
# render to html page
return HttpResponse("")
def config(request):
'''Render the config file'''
res = requests.get(settings.HBP_ENV_URL)
config = res.json()
# Use this app client ID
config['auth']['clientId'] = settings.SOCIAL_AUTH_HBP_KEY
# Add user token information
config['auth']['token'] = {
'access_token': get_access_token(request.user.social_auth.get()),
'token_type': get_token_type(request.user.social_auth.get()),
'expires_in': request.session.get_expiry_age(),
}
config['build'] = settings.BUILD_INFO
return HttpResponse(json.dumps(config), content_type='application/json')
def config(request):
'''Render the config file'''
res = requests.get(settings.HBP_ENV_URL)
config = res.json()
# Use this app client ID
config['auth']['clientId'] = settings.SOCIAL_AUTH_HBP_KEY
# Add user token informations
request.user.social_auth.get().extra_data
config['auth']['token'] = {
'access_token': get_access_token(request.user.social_auth.get()),
'token_type': get_token_type(request.user.social_auth.get()),
'expires_in': request.session.get_expiry_age(),
}
config['build'] = settings.BUILD_INFO
return HttpResponse(json.dumps(config), content_type='application/json')