def CreateInstrument(settingsModel, facility, name):
"""
Create instrument.
"""
myTardisUrl = settingsModel.GetMyTardisUrl()
myTardisUsername = settingsModel.GetUsername()
myTardisApiKey = settingsModel.GetApiKey()
url = myTardisUrl + "/api/v1/instrument/"
headers = {
"Authorization":
"ApiKey %s:%s" % (myTardisUsername, myTardisApiKey),
"Content-Type": "application/json",
"Accept": "application/json"
}
instrumentJson = {"facility": facility.GetResourceUri(), "name": name}
data = json.dumps(instrumentJson)
response = requests.post(headers=headers, url=url, data=data)
content = response.text
if response.status_code >= 200 and response.status_code < 300:
instrumentJson = response.json()
return InstrumentModel(settingsModel=settingsModel,
name=name,
instrumentJson=instrumentJson)
else:
if response.status_code == 401:
message = "Couldn't create instrument \"%s\" " \
"in facility \"%s\"." \
% (name, facility.GetName())
message += "\n\n"
message += "Please ask your MyTardis administrator to " \
"check the permissions of the \"%s\" " \
"user account." % myTardisUsername
raise Unauthorized(message)
if response.status_code == 404:
raise Exception("HTTP 404 (Not Found) received for: " + url)
logger.error("Status code = " + str(response.status_code))
logger.error("URL = " + url)
raise Exception(content)
def ShareExperimentWithGroup(experiment, group):
"""
Grants read access to experiment to group.
"""
logger.debug("\nSharing via ObjectACL with group \"" +
group.GetName() + "\"...\n")
settingsModel = experiment.GetSettingsModel()
myTardisUrl = settingsModel.GetMyTardisUrl()
myTardisDefaultUsername = settingsModel.GetUsername()
myTardisDefaultUserApiKey = settingsModel.GetApiKey()
objectAclJson = {
"pluginId": "django_group",
"entityId": str(group.GetId()),
"content_object":
experiment.GetResourceUri().replace("mydata_", ""),
"content_type": "experiment",
"object_id": experiment.GetId(),
"aclOwnershipType": 1,
"isOwner": True,
"canRead": True,
"canWrite": True,
"canDelete": False,
"effectiveDate": None,
"expiryDate": None
}
headers = {
"Authorization":
"ApiKey %s:%s" %
(myTardisDefaultUsername, myTardisDefaultUserApiKey),
"Content-Type":
"application/json",
"Accept":
"application/json"
}
url = myTardisUrl + "/api/v1/objectacl/"
response = requests.post(headers=headers,
url=url,
data=json.dumps(objectAclJson))
if response.status_code == 201:
logger.debug("Shared experiment with group " + group.GetName() +
".")
else:
logger.debug(url)
logger.debug(response.text)
logger.debug("response.status_code = " + str(response.status_code))
if response.status_code == 401:
message = "Couldn't create ObjectACL for " \
"experiment \"%s\"." % experiment.GetTitle()
message += "\n\n"
message += "Please ask your MyTardis administrator " \
"to check the permissions of the \"%s\" " \
"user account." % myTardisDefaultUsername
raise Unauthorized(message)
elif response.status_code == 404:
message = "Couldn't create ObjectACL for " \
"experiment \"%s\"." % experiment.GetTitle()
message += "\n\n"
message += "A 404 (Not Found) error occurred while " \
"attempting to create the ObjectACL.\n\n" \
"Please ask your MyTardis administrator " \
"to check that a User Profile record " \
"exists for the \"%s\" user account." \
% myTardisDefaultUsername
raise DoesNotExist(message, modelClass=UserProfileModel)
def GetDataFiles(self):
# pylint: disable=too-many-nested-blocks
if not self.datafiles:
try:
self.getDatasetFilesThreadingLock.acquire()
if self.datafiles:
return self.datafiles
myTardisUrl = self.settingsModel.GetMyTardisUrl()
myTardisUsername = self.settingsModel.GetUsername()
myTardisApiKey = self.settingsModel.GetApiKey()
# limit=0 can still encounter a limit of 1000 unless
# API_LIMIT_PER_PAGE is set to 0 in MyTardis's settings.py
limit = 0
url = "%s/api/v1/dataset/%d/files/?format=json&limit=%d" \
% (myTardisUrl, self.GetId(), limit)
headers = {
"Authorization":
"ApiKey %s:%s" % (myTardisUsername, myTardisApiKey)
}
logger.debug(url)
response = requests.get(headers=headers, url=url)
if response.status_code >= 200 and response.status_code < 300:
from .datafile import DataFileModel
self.datafiles = []
datafilesJson = response.json()['objects']
for datafileJson in datafilesJson:
self.datafiles.append(
DataFileModel(self.settingsModel, self,
datafileJson))
offset = 0
while response.json()['meta']['next']:
# We should be able to use
# response.json()['meta']['next'] in the URL,
# instead of manually constructing the next
# URL using offset.
# But response.json()['meta']['next'] seems to give
# the wrong URL for /api/v1/dataset/%d/files/
offset += 1
url = "%s/api/v1/dataset/%d/files/?format=json" \
"&limit=%d&offset=%d" % (myTardisUrl, self.GetId(),
limit, offset)
logger.debug(url)
response = requests.get(headers=headers, url=url)
if response.status_code >= 200 and \
response.status_code < 300:
datafilesJson = response.json()['objects']
for datafileJson in datafilesJson:
self.datafiles\
.append(DataFileModel(self.settingsModel,
self, datafileJson))
else:
logger.error(url)
logger.error("response.status_code = " +
str(response.status_code))
logger.error(response.text)
else:
logger.error(url)
logger.error("response.status_code = " +
str(response.status_code))
logger.error(response.text)
if response.status_code == 401:
message = "Couldn't list files for dataset \"%s\". " \
% (self.GetDescription())
message += "\n\n"
message += "Please ask your MyTardis administrator " \
"to check the permissions of the \"%s\" " \
"user account." % myTardisUsername
self.getDatasetFilesThreadingLock.release()
raise Unauthorized(message)
self.getDatasetFilesThreadingLock.release()
raise Exception(response.text)
finally:
self.getDatasetFilesThreadingLock.release()
return self.datafiles
def CreateDatasetIfNecessary(folderModel, testRun=False):
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
# pylint: disable=too-many-locals
description = folderModel.GetFolder()
settingsModel = folderModel.settingsModel
myTardisUrl = settingsModel.GetMyTardisUrl()
myTardisUsername = settingsModel.GetUsername()
myTardisApiKey = settingsModel.GetApiKey()
experiment = folderModel.GetExperiment()
if experiment: # Could be None in test run
url = myTardisUrl + "/api/v1/dataset/?format=json" + \
"&experiments__id=" + str(experiment.GetId())
url = url + "&description=" + urllib.quote(description)
headers = {
"Authorization":
"ApiKey %s:%s" % (myTardisUsername, myTardisApiKey)
}
response = requests.get(headers=headers, url=url)
existingMatchingDatasets = response.json()
numExistingMatchingDatasets = \
existingMatchingDatasets['meta']['total_count']
if numExistingMatchingDatasets == 1:
logger.debug("Found existing dataset for folder " +
description)
elif numExistingMatchingDatasets > 1:
logger.debug("WARNING: Found multiple datasets for folder " +
description)
else:
numExistingMatchingDatasets = 0
if numExistingMatchingDatasets == 0:
logger.debug("Creating dataset record for folder: " + description)
description = folderModel.GetFolder()
if experiment:
experimentUri = experiment.GetResourceUri()
else:
experimentUri = None
immutable = False
datasetJson = {
"instrument": settingsModel.GetInstrument().GetResourceUri(),
"description": description,
"experiments": [experimentUri],
"immutable": immutable
}
data = json.dumps(datasetJson)
headers = {
"Authorization":
"ApiKey %s:%s" % (myTardisUsername, myTardisApiKey),
"Content-Type":
"application/json",
"Accept":
"application/json"
}
url = myTardisUrl + "/api/v1/dataset/"
if testRun:
message = "CREATING NEW DATASET FOR FOLDER: %s\n" \
" Description: %s" \
% (folderModel.GetRelPath(),
description)
if experiment: # Could be None in test run.
message += "\n In Experiment: %s/%s" \
% (folderModel.settingsModel.GetMyTardisUrl(),
experiment.GetViewUri())
logger.testrun(message)
return
response = requests.post(headers=headers, url=url, data=data)
if response.status_code >= 200 and response.status_code < 300:
newDatasetJson = response.json()
return DatasetModel(settingsModel, newDatasetJson)
else:
logger.error(url)
logger.error("response.status_code = " +
str(response.status_code))
logger.error(response.text)
if response.status_code == 401:
message = "Couldn't create dataset \"%s\" " \
"for folder \"%s\"." \
% (description, folderModel.GetFolder())
message += "\n\n"
message += "Please ask your MyTardis administrator to " \
"check the permissions of the \"%s\" user " \
"account." % myTardisUsername
raise Unauthorized(message)
elif response.status_code == 500:
message = "Couldn't create dataset \"%s\" " \
"for folder \"%s\"." \
% (description, folderModel.GetFolder())
message += "\n\n"
message += "An Internal Server Error occurred."
message += "\n\n"
message += "If running MyTardis in DEBUG mode, " \
"more information may be available below. " \
"Otherwise, please ask your MyTardis " \
"administrator to check in their logs " \
"for more information."
message += "\n\n"
try:
message += "ERROR: \"%s\"" \
% response.json()['error_message']
except: # pylint: disable=bare-except
message += response.text
raise InternalServerError(message)
raise Exception(response.text)
else:
existingDatasetJson = \
existingMatchingDatasets['objects'][0]
if testRun:
description = existingDatasetJson['description']
datasetId = existingDatasetJson['id']
viewUri = "dataset/%s" % datasetId
message = "ADDING TO EXISTING DATASET FOR FOLDER: %s\n" \
" URL: %s/%s\n" \
" Description: %s\n" \
" In Experiment: %s/%s" \
% (folderModel.GetRelPath(),
folderModel.settingsModel.GetMyTardisUrl(),
viewUri,
description,
folderModel.settingsModel.GetMyTardisUrl(),
folderModel.GetExperiment().GetViewUri())
logger.testrun(message)
return DatasetModel(settingsModel,
existingMatchingDatasets['objects'][0])
def CreateExperimentForFolder(folderModel, testRun=False):
# pylint: disable=too-many-branches
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
settingsModel = folderModel.GetSettingsModel()
userFolderName = folderModel.GetUserFolderName()
hostname = settingsModel.GetUploaderModel().GetHostname()
location = folderModel.GetLocation()
groupFolderName = folderModel.GetGroupFolderName()
owner = folderModel.GetOwner()
ownerUsername = folderModel.GetOwner().GetUsername()
# pylint: disable=bare-except
try:
ownerUserId = folderModel.GetOwner().GetJson()['id']
except:
ownerUserId = None
uploaderName = settingsModel.GetUploaderModel().GetName()
uploaderUuid = settingsModel.GetUploaderModel().GetUuid()
experimentTitle = folderModel.GetExperimentTitle()
myTardisUrl = settingsModel.GetMyTardisUrl()
myTardisDefaultUsername = settingsModel.GetUsername()
myTardisDefaultUserApiKey = settingsModel.GetApiKey()
if userFolderName:
message = "Creating experiment for uploader '%s', " \
"user folder '%s'." % (uploaderName, userFolderName)
if groupFolderName:
message += ", group folder : '%s'" % groupFolderName
elif groupFolderName:
message = "Creating experiment for uploader '%s', " \
"user group folder '%s'." % (uploaderName, groupFolderName)
else:
message = "Creating experiment for uploader '%s'" % uploaderName
logger.info(message)
if userFolderName:
description = ("Uploader: %s\n"
"User folder name: %s\n"
"Uploaded from: %s:%s" %
(uploaderName, userFolderName, hostname, location))
if groupFolderName:
description += "\nGroup folder name: %s" % groupFolderName
else:
description = ("Uploader: %s\n"
"Group folder name: %s\n"
"Uploaded from: %s:%s" %
(uploaderName, groupFolderName, hostname, location))
if testRun:
message = "CREATING NEW EXPERIMENT FOR FOLDER: %s\n" \
" Title: %s\n" \
" Description: \n" \
" Uploader: %s\n" \
" User folder name: %s\n" \
" Owner: %s" \
% (folderModel.GetRelPath(),
experimentTitle, uploaderName, userFolderName,
ownerUsername)
logger.testrun(message)
return
experimentJson = {
"title":
experimentTitle,
"description":
description,
"immutable":
False,
"parameter_sets": [{
"schema":
"http://mytardis.org/schemas"
"/mydata/defaultexperiment",
"parameters": [{
"name": "uploader",
"value": uploaderUuid
}, {
"name": "user_folder_name",
"value": userFolderName
}]
}]
}
if groupFolderName:
experimentJson["parameter_sets"][0]["parameters"].append({
"name":
"group_folder_name",
"value":
groupFolderName
})
headers = {
"Authorization":
"ApiKey %s:%s" %
(myTardisDefaultUsername, myTardisDefaultUserApiKey),
"Content-Type":
"application/json",
"Accept":
"application/json"
}
url = myTardisUrl + "/api/v1/mydata_experiment/"
response = requests.post(headers=headers,
url=url,
data=json.dumps(experimentJson))
# pylint: disable=bare-except
try:
createdExperimentJson = response.json()
createdExperiment = ExperimentModel(settingsModel,
createdExperimentJson)
logger.debug(url)
except:
logger.error(url)
logger.error(response.text)
logger.error("response.status_code = " + str(response.status_code))
if response.status_code == 401:
message = "Couldn't create experiment \"%s\" " \
"for folder \"%s\"." \
% (experimentTitle, folderModel.GetFolder())
message += "\n\n"
message += "Please ask your MyTardis administrator to " \
"check the permissions of the \"%s\" user " \
"account." % myTardisDefaultUsername
raise Unauthorized(message)
elif response.status_code == 404:
message = "Couldn't create experiment \"%s\" " \
"for folder \"%s\"." \
% (experimentTitle, folderModel.GetFolder())
message += "\n\n"
message += "A 404 (Not Found) error occurred while " \
"attempting to create the experiment.\n\n" \
"Please ask your MyTardis administrator to " \
"check that a User Profile record exists " \
"for the \"%s\" user account." \
% myTardisDefaultUsername
raise DoesNotExist(message)
raise
if response.status_code == 201:
message = "Succeeded in creating experiment '%s' for uploader " \
"\"%s\" and user folder \"%s\"" \
% (experimentTitle, uploaderName, userFolderName)
if groupFolderName:
message += " and group folder \"%s\"" % groupFolderName
logger.debug(message)
facilityManagersGroup = settingsModel.GetFacility(
).GetManagerGroup()
ObjectAclModel.ShareExperimentWithGroup(createdExperiment,
facilityManagersGroup)
# Avoid creating a duplicate ObjectACL if the user folder's
# username matches the facility manager's username.
# Don't attempt to create an ObjectACL record for an
# invalid user (without a MyTardis user ID).
if myTardisDefaultUsername != ownerUsername and \
ownerUserId is not None:
ObjectAclModel.ShareExperimentWithUser(createdExperiment,
owner)
if folderModel.GetGroup() is not None and \
folderModel.GetGroup().GetId() != \
facilityManagersGroup.GetId():
ObjectAclModel.ShareExperimentWithGroup(
createdExperiment, folderModel.GetGroup())
else:
message = "Failed to create experiment for uploader " \
"\"%s\" and user folder \"%s\"" \
% (uploaderName, userFolderName)
if groupFolderName:
message += " and group folder \"%s\"" % groupFolderName
logger.error(message)
logger.error(headers)
logger.error(url)
logger.error(response.text)
logger.error("response.status_code = " + str(response.status_code))
if response.status_code == 401:
message = "Couldn't create experiment \"%s\" " \
"for folder \"%s\"." \
% (experimentTitle, folderModel.GetFolder())
message += "\n\n"
message += "Please ask your MyTardis administrator to " \
"check the permissions of the \"%s\" user " \
"account." % myTardisDefaultUsername
raise Unauthorized(message)
elif response.status_code == 404:
message = "Couldn't create experiment \"%s\" " \
"for folder \"%s\"." \
% (experimentTitle, folderModel.GetFolder())
message += "\n\n"
modelClassOfObjectNotFound = None
# pylint: disable=bare-except
try:
errorResponse = response.json()
if errorResponse['error_message'] == \
"UserProfile matching query does not exist.":
modelClassOfObjectNotFound = UserProfileModel
elif errorResponse['error_message'] == \
"Schema matching query does not exist.":
modelClassOfObjectNotFound = SchemaModel
elif errorResponse['error_message'] == \
"Sorry, this request could not be processed. " \
"Please try again later.":
raise Exception("TASTYPIE_CANNED_ERROR")
message += "A 404 (Not Found) error occurred while " \
"attempting to create an experiment " \
"record:\n\n" \
" %s\n\n" % errorResponse['error_message']
except:
message += "A 404 (Not Found) error occurred while " \
"attempting to create an experiment " \
"record. This could be caused by a missing " \
"UserProfile record for user \"%s\" or it " \
"could be caused by a missing Schema record " \
"(see https://github.com/wettenhj/" \
"mytardis-app-mydata/blob/master/README.md)" \
"\n\n" \
"Turning on DEBUG mode on the MyTardis " \
"server could help to isolate the problem." \
% myTardisDefaultUsername
if modelClassOfObjectNotFound == UserProfileModel:
message += "Please ask your MyTardis administrator to " \
"ensure that a User Profile record exists " \
"for the \"%s\" user account." \
% myTardisDefaultUsername
elif modelClassOfObjectNotFound == SchemaModel:
message += "Please ask your MyTardis administrator to " \
"create the experiment metadata schema " \
"described in the \"MyTardis Prerequisites\" " \
"section of the MyData documentation:\n\n" \
"http://mydata.readthedocs.org/en/latest/" \
"mytardis-prerequisites.html"
raise DoesNotExist(message,
modelClass=modelClassOfObjectNotFound)
return createdExperiment
else:
raise Exception(
"Only %d of %d bytes were uploaded for %s"
% (bytesUploaded, dataFileSize, dataFilePath))
if not postSuccess and not self.existingUnverifiedDatafile:
if response.status_code == 401:
message = "Couldn't create datafile \"%s\" " \
"for folder \"%s\"." \
% (dataFileName,
self.folderModel.GetFolder())
message += "\n\n"
message += \
"Please ask your MyTardis administrator to " \
"check the permissions of the \"%s\" user " \
"account." % myTardisUsername
raise Unauthorized(message)
elif response.status_code == 404:
message = "Encountered a 404 (Not Found) error " \
"while attempting to create a datafile " \
"record for \"%s\" in folder \"%s\"." \
% (dataFileName,
self.folderModel.GetFolder())
message += "\n\n"
message += \
"Please ask your MyTardis administrator to " \
"check whether an appropriate staging " \
"storage box exists."
raise DoesNotExist(message)
elif response.status_code == 500:
message = "Couldn't create datafile \"%s\" " \
"for folder \"%s\"." \