def save(username, project_name, experiment_name, file):
"""Save file to username/project name/experiment_name in data store."""
exp_dir = os.path.join(
experiment_data_storage.get_valid_name(username),
experiment_data_storage.get_valid_name(project_name),
experiment_data_storage.get_valid_name(experiment_name))
# file.name may be full path, so get just the name of the file
file_name = os.path.basename(file.name)
file_path = os.path.join(exp_dir,
experiment_data_storage.get_valid_name(file_name))
input_file_name = experiment_data_storage.save(file_path, file)
input_file_fullpath = experiment_data_storage.path(input_file_name)
# Create DataProductModel instance with DataReplicaLocationModel
data_product = DataProductModel()
data_product.gatewayId = settings.GATEWAY_ID
data_product.ownerName = username
data_product.productName = file_name
data_product.dataProductType = DataProductType.FILE
data_replica_location = DataReplicaLocationModel()
data_replica_location.storageResourceId = \
settings.GATEWAY_DATA_STORE_RESOURCE_ID
data_replica_location.replicaName = \
"{} gateway data store copy".format(file_name)
data_replica_location.replicaLocationCategory = \
ReplicaLocationCategory.GATEWAY_DATA_STORE
data_replica_location.replicaPersistentType = \
ReplicaPersistentType.TRANSIENT
data_replica_location.filePath = \
"file://{}:{}".format(settings.GATEWAY_DATA_STORE_HOSTNAME,
input_file_fullpath)
data_product.replicaLocations = [data_replica_location]
return data_product
def test_copy_input_file_upload(self):
"Test copy input file upload copies data product"
with tempfile.TemporaryDirectory() as tmpdirname, \
self.settings(GATEWAY_DATA_STORE_DIR=tmpdirname,
GATEWAY_DATA_STORE_HOSTNAME="gateway.com"):
# path is just the user directory in gateway storage
source_path = os.path.join(tmpdirname, self.user.username,
"foo.ext")
os.makedirs(os.path.dirname(source_path))
with open(source_path, 'wb') as f:
f.write(b"123")
data_product = DataProductModel()
data_product.productUri = f"airavata-dp://{uuid.uuid4()}"
data_product.gatewayId = GATEWAY_ID
data_product.ownerName = self.user.username
data_product.productName = "foo.ext"
data_product.dataProductType = DataProductType.FILE
data_product.productMetadata = {
'mime-type': 'application/some-app'
}
replica_category = ReplicaLocationCategory.GATEWAY_DATA_STORE
replica_path = f"file://gateway.com:{source_path}"
data_product.replicaLocations = [
DataReplicaLocationModel(
filePath=replica_path,
replicaLocationCategory=replica_category)
]
data_product_copy = data_products_helper.copy_input_file_upload(
self.request, data_product)
self.request.airavata_client.registerDataProduct.\
assert_called_once()
self.assertIsNot(data_product_copy, data_product)
self.assertNotEqual(data_product_copy.productUri,
data_product.productUri)
self.assertDictEqual(data_product_copy.productMetadata,
data_product.productMetadata)
self.assertEqual(data_product_copy.productName,
data_product.productName)
self.assertEqual(data_product_copy.dataProductType,
data_product.dataProductType)
replica_copy_path = data_product_copy.replicaLocations[0].filePath
self.assertNotEqual(replica_copy_path, replica_path)
replica_copy_filepath = urlparse(replica_copy_path).path
self.assertEqual(os.path.dirname(replica_copy_filepath),
os.path.join(tmpdirname, self.user.username,
"tmp"),
msg="Verify input file copied to user's tmp dir")
def _create_data_product(username, full_path, name=None, content_type=None):
data_product = DataProductModel()
data_product.gatewayId = settings.GATEWAY_ID
data_product.ownerName = username
if name is not None:
file_name = name
else:
file_name = os.path.basename(full_path)
data_product.productName = file_name
data_product.dataProductType = DataProductType.FILE
final_content_type = _determine_content_type(full_path, content_type)
if final_content_type is not None:
data_product.productMetadata = {'mime-type': final_content_type}
data_replica_location = _create_replica_location(full_path, file_name)
data_product.replicaLocations = [data_replica_location]
return data_product
def upload_input_file(request):
try:
# Save input file to username/project name/exp name/filename
username = request.user.username
project_id = request.POST['project-id']
project = request.airavata_client.getProject(request.authz_token,
project_id)
exp_name = request.POST['experiment-name']
input_file = request.FILES['file']
exp_dir = os.path.join(
experiment_data_storage.get_valid_name(username),
experiment_data_storage.get_valid_name(project.name),
experiment_data_storage.get_valid_name(exp_name))
file_path = os.path.join(
exp_dir, experiment_data_storage.get_valid_name(input_file.name))
input_file_name = experiment_data_storage.save(file_path, input_file)
input_file_fullpath = experiment_data_storage.path(input_file_name)
# Register DataProductModel with DataReplicaLocationModel
data_product = DataProductModel()
data_product.gatewayId = settings.GATEWAY_ID
data_product.ownerName = username
data_product.productName = input_file.name
data_product.dataProductType = DataProductType.FILE
data_replica_location = DataReplicaLocationModel()
data_replica_location.storageResourceId = \
settings.GATEWAY_DATA_STORE_RESOURCE_ID
data_replica_location.replicaName = \
"{} gateway data store copy".format(input_file.name)
data_replica_location.replicaLocationCategory = \
ReplicaLocationCategory.GATEWAY_DATA_STORE
data_replica_location.replicaPersistentType = \
ReplicaPersistentType.TRANSIENT
hostname = urlparse(request.build_absolute_uri()).hostname
data_replica_location.filePath = \
"file://{}:{}".format(hostname, input_file_fullpath)
data_product.replicaLocations = [data_replica_location]
data_product_uri = request.airavata_client.registerDataProduct(
request.authz_token, data_product)
return JsonResponse({
'uploaded': True,
'data-product-uri': data_product_uri
})
except Exception as e:
resp = JsonResponse({'uploaded': False, 'error': str(e)})
resp.status_code = 500
return resp
def register_input_file(self, file_identifier, storage_name, storageId,
input_file_name, uploaded_storage_path):
dataProductModel = DataProductModel()
dataProductModel.gatewayId = self.gateway_id
dataProductModel.ownerName = self.username
dataProductModel.productName = file_identifier
dataProductModel.dataProductType = DataProductType.FILE
replicaLocation = DataReplicaLocationModel()
replicaLocation.storageResourceId = storageId
replicaLocation.replicaName = "{} gateway data store copy".format(
input_file_name)
replicaLocation.replicaLocationCategory = ReplicaLocationCategory.GATEWAY_DATA_STORE
replicaLocation.filePath = "file://{}:{}".format(
storage_name, uploaded_storage_path + input_file_name)
dataProductModel.replicaLocations = [replicaLocation]
return self.api_server_client.register_data_product(
self.token, dataProductModel)
def _create_data_product(username, full_path):
data_product = DataProductModel()
data_product.gatewayId = settings.GATEWAY_ID
data_product.ownerName = username
file_name = os.path.basename(full_path)
data_product.productName = file_name
data_product.dataProductType = DataProductType.FILE
data_replica_location = DataReplicaLocationModel()
data_replica_location.storageResourceId = \
settings.GATEWAY_DATA_STORE_RESOURCE_ID
data_replica_location.replicaName = \
"{} gateway data store copy".format(file_name)
data_replica_location.replicaLocationCategory = \
ReplicaLocationCategory.GATEWAY_DATA_STORE
data_replica_location.replicaPersistentType = \
ReplicaPersistentType.TRANSIENT
data_replica_location.filePath = \
"file://{}:{}".format(settings.GATEWAY_DATA_STORE_HOSTNAME,
full_path)
data_product.replicaLocations = [data_replica_location]
return data_product
