def do_provider(self, loc):
provider = loc.provider
base_url = loc.url
datafile, replica = generate_datafile("1/1/3", self.dataset, "Hi mum")
self.assertEquals(replica.verify(allowEmptyChecksums=True), True)
target_replica = Replica()
target_replica.datafile = datafile
target_replica.location = loc
url = provider.generate_url(target_replica)
self.assertEquals(url, base_url + '1/1/3')
target_replica.url = url
provider.put_file(replica, target_replica)
self.assertEqual(replica.location.provider.get_file(replica).read(),
"Hi mum")
self.assertEqual(provider.get_file(target_replica).read(), "Hi mum")
self.assertEqual(provider.get_length(target_replica), 6)
try:
self.maxDiff = None
self.assertEqual(provider.get_metadata(target_replica),
{'sha512sum' : '2274cc8c16503e3d182ffaa835c543b' +
'ce278bc8fc971f3bf38b94b4d9db44cd89c8f36d4006e' +
'5abea29bc05f7f0ea662cb4b0e805e56bbce97f00f94e' +
'a6e6498',
'md5sum' : '3b6b51114c3d0ad347e20b8e79765951',
'length' : '6'})
except NotImplementedError:
pass
provider.remove_file(target_replica)
with self.assertRaises(TransferError):
provider.get_length(target_replica)
def process_enclosure(self, dataset, enclosure):
filename = getattr(enclosure, 'title', basename(enclosure.href))
datafile = Dataset_File(filename=filename, dataset=dataset)
try:
datafile.mimetype = enclosure.mime
except AttributeError:
pass
try:
datafile.size = enclosure.length
except AttributeError:
pass
try:
hash = enclosure.hash
# Split on white space, then ':' to get tuples to feed into dict
hashdict = dict([s.partition(':')[::2] for s in hash.split()])
# Set SHA-512 sum
datafile.sha512sum = hashdict['sha-512']
except AttributeError:
pass
datafile.save()
url = enclosure.href
# This means we will allow the atom feed to feed us any enclosure
# URL that matches a registered location. Maybe we should restrict
# this to a specific location.
location = Location.get_location_for_url(url)
if not location:
logger.error('Rejected ingestion for unknown location %s' % url)
return
replica = Replica(datafile=datafile, url=url,
location=location)
replica.protocol = enclosure.href.partition('://')[0]
replica.save()
self.make_local_copy(replica)
def remove_experiment_data(exp, archive_url, archive_location):
"""Remove the online Replicas for an Experiment that are not shared with
other Experiments. When Replicas are removed, they are replaced with
offline replicas whose 'url' consists of the archive_url, with the
archive pathname for the datafile as a url fragment id.
"""
for ds in Dataset.objects.filter(experiments=exp):
if ds.experiments.count() == 1:
for df in Dataset_File.objects.filter(dataset=ds):
replicas = Replica.objects.filter(datafile=df,
location__type='online')
if replicas.count() > 0:
for replica in replicas:
location = Location.get_location(replica.location.name)
location.provider.remove_file(replica)
if archive_url:
old_replica = replicas[0]
path_in_archive = '%s/%s/%s' % (
exp.id, ds.id, df.filename)
new_replica_url = '%s#%s' % (
archive_url, quote(path_in_archive))
new_replica = Replica(datafile=old_replica.datafile,
url=new_replica_url,
protocol=old_replica.protocol,
verified=True,
stay_remote=False,
location=archive_location)
new_replica.save()
replicas.delete()
def process_enclosure(self, dataset, enclosure):
filename = getattr(enclosure, 'title', basename(enclosure.href))
datafile = Dataset_File(filename=filename, dataset=dataset)
try:
datafile.mimetype = enclosure.mime
except AttributeError:
pass
try:
datafile.size = enclosure.length
except AttributeError:
pass
try:
hash = enclosure.hash
# Split on white space, then ':' to get tuples to feed into dict
hashdict = dict([s.partition(':')[::2] for s in hash.split()])
# Set SHA-512 sum
datafile.sha512sum = hashdict['sha-512']
except AttributeError:
pass
datafile.save()
url = enclosure.href
# This means we will allow the atom feed to feed us any enclosure
# URL that matches a registered location. Maybe we should restrict
# this to a specific location.
location = Location.get_location_for_url(url)
if not location:
logger.error('Rejected ingestion for unknown location %s' % url)
return
replica = Replica(datafile=datafile, url=url, location=location)
replica.protocol = enclosure.href.partition('://')[0]
replica.save()
self.make_local_copy(replica)
def _build(dataset, filename, url, protocol):
from tardis.tardis_portal.models import \
Dataset_File, Replica, Location
datafile = Dataset_File(dataset=dataset, filename=filename)
datafile.save()
replica = Replica(datafile=datafile, url=url,
protocol=protocol,
location=Location.get_default_location())
replica.save()
return datafile
def _build(dataset, filename, url, protocol):
from tardis.tardis_portal.models import \
Dataset_File, Replica, Location
datafile = Dataset_File(dataset=dataset, filename=filename)
datafile.save()
replica = Replica(datafile=datafile, url=url,
protocol=protocol,
location=Location.get_default_location())
replica.save()
return datafile
def _create_test_dataset(nosDatafiles):
ds_ = Dataset(description='happy snaps of plumage')
ds_.save()
for i in range (0, nosDatafiles) :
df_ = Dataset_File(dataset=ds_, size='21', sha512sum='bogus')
df_.save()
rep_ = Replica(datafile=df_,
url='http://planet-python.org/' + str(_next_id()),
location=Location.get_default_location())
rep_.save()
ds_.save()
return ds_
def _create_test_dataset(nosDatafiles):
ds_ = Dataset(description='happy snaps of plumage')
ds_.save()
for i in range(0, nosDatafiles):
df_ = Dataset_File(dataset=ds_, size='21', sha512sum='bogus')
df_.save()
rep_ = Replica(datafile=df_,
url='http://planet-python.org/' + str(_next_id()),
location=Location.get_default_location())
rep_.save()
ds_.save()
return ds_
def generate_datafile(path, dataset, content=None, size=-1,
verify=True, verified=True):
'''Generates a datafile AND a replica to hold its contents'''
from tardis.tardis_portal.models import Dataset_File, Replica, Location
saved = settings.REQUIRE_DATAFILE_CHECKSUMS
settings.REQUIRE_DATAFILE_CHECKSUMS = False
try:
datafile = Dataset_File()
if content:
datafile.size = str(len(content))
else:
datafile.size = str(size)
# Normally we use any old string for the datafile path, but some
# tests require the path to be the same as what 'staging' would use
if path == None:
datafile.dataset_id = dataset.id
datafile.save()
path = "%s/%s/%s" % (dataset.get_first_experiment().id,
dataset.id, datafile.id)
filepath = os.path.normpath(FILE_STORE_PATH + '/' + path)
if content:
try:
os.makedirs(os.path.dirname(filepath))
os.remove(filepath)
except:
pass
file = open(filepath, 'wb+')
file.write(content)
file.close()
datafile.mimetype = "application/unspecified"
datafile.filename = os.path.basename(filepath)
datafile.dataset_id = dataset.id
datafile.save()
location = _infer_location(path)
replica = Replica(datafile=datafile, url=path, protocol='',
location=location)
if verify and content:
if not replica.verify(allowEmptyChecksums=True):
raise RuntimeError('verify failed!?!')
else:
replica.verified = verified
replica.save()
return (datafile, replica)
finally:
settings.REQUIRE_DATAFILE_CHECKSUMS = saved
def testMirror(self):
dest = Location.get_location('test')
datafile, replica = generate_datafile(None, self.dataset, "Hi granny")
path = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path))
dummy_replica = Replica()
dummy_replica.datafile = datafile
dummy_replica.location = Location.objects.get(name='test')
dummy_replica.url = dummy_replica.generate_default_url()
with self.assertRaises(TransferError):
dest.provider.get_length(dummy_replica)
self.assertTrue(migrate_replica(replica, dest, mirror=True))
datafile = Dataset_File.objects.get(id=datafile.id)
self.assertTrue(datafile.is_local())
self.assertEquals(dest.provider.get_length(dummy_replica), 9)
def create_datafile(index):
testfile = path.join(path.dirname(__file__), 'fixtures',
'middleware_test%d.txt' % index)
size, sha512sum = get_size_and_sha512sum(testfile)
datafile = Dataset_File(dataset=dataset,
filename=path.basename(testfile),
size=size,
sha512sum=sha512sum)
datafile.save()
base_url = 'file://' + path.abspath(path.dirname(testfile))
location = Location.load_location({
'name': 'test-middleware',
'url': base_url,
'type': 'external',
'priority': 10,
'transfer_provider': 'local'
})
replica = Replica(datafile=datafile,
url='file://' + path.abspath(testfile),
protocol='file',
location=location)
replica.save()
if index != 1:
replica.verified = False
replica.save(update_fields=['verified'])
return Dataset_File.objects.get(pk=datafile.pk)
def testRemoteFile(self):
content = urandom(1024)
with NamedTemporaryFile() as f:
# Create new Datafile
datafile = Dataset_File(dataset=self.dataset)
datafile.filename = 'background_task_testfile'
datafile.size = len(content)
datafile.sha512sum = hashlib.sha512(content).hexdigest()
datafile.save()
url = 'file://' + path.abspath(f.name)
base_url = 'file://' + path.dirname(path.abspath(f.name))
location = self._get_or_create_local_location(
'test-staging-xxx', base_url, 'external', 10)
replica = Replica(datafile=datafile, location=location, url=url)
replica.save()
def get_replica(replica):
try:
return Replica.objects.get(id=replica.id)
except Replica.DoesNotExist:
return None
def get_new_replica(datafile):
location = Location.get_default_location()
return Replica.objects.get(datafile=datafile.id,
location=location)
# Check that it won't verify as it stands
expect(get_replica(replica).verified).to_be(False)
verify_files()
expect(get_replica(replica).verified).to_be(False)
expect(get_replica(replica).is_local()).to_be(False)
# Fill in the content
f.write(content)
f.flush()
# Check it now verifies
verify_files()
expect(get_replica(replica).id).to_be(
get_new_replica(datafile).id)
expect(get_new_replica(datafile).verified).to_be(True)
expect(get_new_replica(datafile).is_local()).to_be(True)
def create_datafile(file_path):
testfile = path.join(path.dirname(__file__), 'fixtures', file_path)
size, sha512sum = get_size_and_sha512sum(testfile)
datafile = Dataset_File(dataset=dataset,
filename=path.basename(testfile),
size=size,
sha512sum=sha512sum)
datafile.save()
base_url = 'file://' + path.abspath(path.dirname(testfile))
location = Location.load_location({
'name': 'test-flexstation', 'url': base_url, 'type': 'external',
'priority': 10, 'transfer_provider': 'local'})
replica = Replica(datafile=datafile,
url='file://'+ path.abspath(testfile),
protocol='file',
location=location)
replica.verify()
replica.save()
return Dataset_File.objects.get(pk=datafile.pk)
def create_datafile(file_path):
testfile = path.join(path.dirname(__file__), 'fixtures', file_path)
size, sha512sum = get_size_and_sha512sum(testfile)
datafile = Dataset_File(dataset=dataset,
filename=path.basename(testfile),
size=size,
sha512sum=sha512sum)
datafile.save()
base_url = 'file://' + path.abspath(path.dirname(testfile))
location = Location.load_location({
'name': 'test-flexstation',
'url': base_url,
'type': 'external',
'priority': 10,
'transfer_provider': 'local'
})
replica = Replica(datafile=datafile,
url='file://' + path.abspath(testfile),
protocol='file',
location=location)
replica.verify()
replica.save()
return Dataset_File.objects.get(pk=datafile.pk)
def setUp(self):
"""
setting up essential objects, copied from tests above
"""
Location.force_initialize()
self.location = Location.get_location('local')
user = '******'
pwd = 'secret'
email = ''
self.user = User.objects.create_user(user, email, pwd)
self.userProfile = UserProfile(user=self.user).save()
self.exp = Experiment(title='test exp1',
institution_name='monash',
created_by=self.user)
self.exp.save()
self.acl = ObjectACL(
pluginId=django_user,
entityId=str(self.user.id),
content_object=self.exp,
canRead=True,
isOwner=True,
aclOwnershipType=ObjectACL.OWNER_OWNED,
)
self.acl.save()
self.dataset = Dataset(description='dataset description...')
self.dataset.save()
self.dataset.experiments.add(self.exp)
self.dataset.save()
self.dataset_file = Dataset_File(dataset=self.dataset,
size=42,
filename="foo",
md5sum="junk")
self.dataset_file.save()
self.replica = Replica(datafile=self.dataset_file,
url="http://foo",
location=self.location,
verified=False)
self.replica.save()
def testLocalFile(self):
content = urandom(1024)
cf = ContentFile(content, 'background_task_testfile')
# Create new Datafile
datafile = Dataset_File(dataset=self.dataset)
datafile.filename = cf.name
datafile.size = len(content)
datafile.sha512sum = hashlib.sha512(content).hexdigest()
datafile.save()
replica = Replica(datafile=datafile,
url=write_uploaded_file_to_dataset(self.dataset, cf),
location=Location.get_default_location())
replica.save()
def get_replica(datafile):
return Replica.objects.get(datafile=datafile)
# undo auto-verify:
replica.verified = False
replica.save(update_fields=['verified'])
# Check that it's not currently verified
expect(get_replica(datafile).verified).to_be(False)
# Check it verifies
verify_files()
expect(get_replica(datafile).verified).to_be(True)
def stage_replica(replica):
from django.core.files.uploadedfile import TemporaryUploadedFile
from tardis.tardis_portal.models import Replica, Location
if not replica.location.type == 'external':
raise ValueError('Only external replicas can be staged')
with TemporaryUploadedFile(replica.datafile.filename,
None, None, None) as tf:
if replica.verify(tempfile=tf.file):
if not replica.stay_remote:
tf.file.flush()
target_replica = Replica(
datafile=replica.datafile,
url=write_uploaded_file_to_dataset(\
replica.datafile.dataset, tf),
location=Location.get_default_location(),
verified=True,
protocol='')
target_replica.save()
replica.delete()
return True
else:
return False
def _build_datafile(self, testfile, filename, dataset, url,
protocol='', checksum=None, size=None, mimetype=''):
filesize, sha512sum = get_size_and_sha512sum(testfile)
datafile = Dataset_File(dataset=dataset, filename=filename,
mimetype=mimetype,
size=str(size if size != None else filesize),
sha512sum=(checksum if checksum else sha512sum))
datafile.save()
if urlparse.urlparse(url).scheme == '':
location = Location.get_location('local')
else:
location = Location.get_location_for_url(url)
if not location:
location = Location.load_location({
'name': filename, 'url': urlparse.urljoin(url, '.'),
'type': 'external',
'priority': 10, 'transfer_provider': 'local'})
replica = Replica(datafile=datafile, protocol=protocol, url=url,
location=location)
replica.verify()
replica.save()
return Dataset_File.objects.get(pk=datafile.pk)
def do_provider(self, loc):
provider = loc.provider
base_url = loc.url
datafile, replica = generate_datafile("1/1/3", self.dataset, "Hi mum")
self.assertEquals(replica.verify(allowEmptyChecksums=True), True)
target_replica = Replica()
target_replica.datafile = datafile
target_replica.location = loc
url = provider.generate_url(target_replica)
self.assertEquals(url, base_url + '1/1/3')
target_replica.url = url
provider.put_file(replica, target_replica)
self.assertEqual(
replica.location.provider.get_file(replica).read(), "Hi mum")
self.assertEqual(provider.get_file(target_replica).read(), "Hi mum")
self.assertEqual(provider.get_length(target_replica), 6)
try:
self.maxDiff = None
self.assertEqual(
provider.get_metadata(target_replica), {
'sha512sum':
'2274cc8c16503e3d182ffaa835c543b' +
'ce278bc8fc971f3bf38b94b4d9db44cd89c8f36d4006e' +
'5abea29bc05f7f0ea662cb4b0e805e56bbce97f00f94e' +
'a6e6498',
'md5sum':
'3b6b51114c3d0ad347e20b8e79765951',
'length':
'6'
})
except NotImplementedError:
pass
provider.remove_file(target_replica)
with self.assertRaises(TransferError):
provider.get_length(target_replica)
def generate_datafile(path, dataset, content=None, size=-1,
verify=True, verified=True, verify_checksums_req=False):
'''Generates a datafile AND a replica to hold its contents'''
from tardis.tardis_portal.models import Dataset_File, Replica, Location
saved = settings.REQUIRE_DATAFILE_CHECKSUMS
settings.REQUIRE_DATAFILE_CHECKSUMS = False
try:
datafile = Dataset_File()
if content:
datafile.size = str(len(content))
else:
datafile.size = str(size)
# Normally we use any old string for the datafile path, but some
# tests require the path to be the same as what 'staging' would use
if path == None:
datafile.dataset_id = dataset.id
datafile.save()
path = "%s/%s/%s" % (dataset.get_first_experiment().id,
dataset.id, datafile.id)
filepath = os.path.normpath(settings.FILE_STORE_PATH + '/' + path)
if content:
try:
os.makedirs(os.path.dirname(filepath))
os.remove(filepath)
except:
pass
gen_file = open(filepath, 'wb+')
gen_file.write(content)
gen_file.close()
datafile.mimetype = "application/unspecified"
datafile.filename = os.path.basename(filepath)
datafile.dataset_id = dataset.id
datafile.save()
settings.REQUIRE_DATAFILE_CHECKSUMS = verify_checksums_req
location = _infer_location(path)
replica = Replica(datafile=datafile, url=path, protocol='',
location=location)
if verify and content:
if not replica.verify():
raise RuntimeError('verify failed!?!')
replica.save()
replica.verified = verified
replica.save(update_fields=['verified']) # force no verification
return (datafile, replica)
finally:
settings.REQUIRE_DATAFILE_CHECKSUMS = saved
def create_staging_datafile(filepath, username, dataset_id):
from tardis.tardis_portal.models import Dataset_File, Dataset, Replica, \
Location
dataset = Dataset.objects.get(id=dataset_id)
url, size = get_staging_url_and_size(username, filepath)
datafile = Dataset_File(dataset=dataset,
filename=path.basename(filepath),
size=size)
replica = Replica(datafile=datafile,
protocol='staging',
url=url,
location=Location.get_location('staging'))
replica.verify(allowEmptyChecksums=True)
datafile.save()
replica.datafile = datafile
replica.save()
def create_staging_datafile(filepath, username, dataset_id):
dataset = Dataset.objects.get(id=dataset_id)
url, size = get_staging_url_and_size(username, filepath)
datafile = Dataset_File(dataset=dataset,
filename=path.basename(filepath),
size=size)
replica = Replica(datafile=datafile,
protocol='staging',
url=url,
location=Location.get_location('staging'))
replica.verify(allowEmptyChecksums=True)
datafile.save()
replica.datafile = datafile
replica.save()
def _create_datafile():
user = User.objects.create_user('testuser', '*****@*****.**', 'pwd')
user.save()
UserProfile(user=user).save()
Location.force_initialize()
full_access = Experiment.PUBLIC_ACCESS_FULL
experiment = Experiment.objects.create(title="IIIF Test",
created_by=user,
public_access=full_access)
experiment.save()
ObjectACL(content_object=experiment,
pluginId='django_user',
entityId=str(user.id),
isOwner=True,
canRead=True,
canWrite=True,
canDelete=True,
aclOwnershipType=ObjectACL.OWNER_OWNED).save()
dataset = Dataset()
dataset.save()
dataset.experiments.add(experiment)
dataset.save()
# Create new Datafile
tempfile = TemporaryUploadedFile('iiif_stored_file', None, None, None)
with Image(filename='magick:rose') as img:
img.format = 'tiff'
img.save(file=tempfile.file)
tempfile.file.flush()
datafile = Dataset_File(dataset=dataset,
size=os.path.getsize(tempfile.file.name),
filename='iiif_named_file')
replica = Replica(datafile=datafile,
url=write_uploaded_file_to_dataset(dataset, tempfile),
location=Location.get_default_location())
replica.verify(allowEmptyChecksums=True)
datafile.save()
replica.datafile = datafile
replica.save()
return datafile
def fpupload(request, dataset_id):
"""
Uploads all files picked by filepicker to the dataset
:param request: a HTTP Request instance
:type request: :class:`django.http.HttpRequest`
:param dataset_id: the dataset_id
:type dataset_id: integer
:returns: boolean true if successful
:rtype: bool
"""
dataset = Dataset.objects.get(id=dataset_id)
logger.debug('called fpupload')
if request.method == 'POST':
logger.debug('got POST')
for key, val in request.POST.items():
splits = val.split(",")
for url in splits:
try:
fp = FilepickerFile(url)
except ValueError:
pass
else:
picked_file = fp.get_file()
filepath = write_uploaded_file_to_dataset(
dataset, picked_file)
datafile = Dataset_File(dataset=dataset,
filename=picked_file.name,
size=picked_file.size)
replica = Replica(datafile=datafile,
url=filepath,
protocol='',
location=Location.get_default_location())
replica.verify(allowEmptyChecksums=True)
datafile.save()
replica.datafile = datafile
replica.save()
return HttpResponse(json.dumps({"result": True}))
def testMirror(self):
dest = Location.get_location('test')
datafile, replica = generate_datafile(None, self.dataset, "Hi granny")
path = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path))
dummy_replica = Replica()
dummy_replica.datafile = datafile
dummy_replica.location = Location.objects.get(name='test')
dummy_replica.url = dummy_replica.generate_default_url()
with self.assertRaises(TransferError):
dest.provider.get_length(dummy_replica)
self.assertTrue(migrate_replica(replica, dest, mirror=True))
datafile = Dataset_File.objects.get(id=datafile.id)
self.assertTrue(datafile.is_local())
self.assertEquals(dest.provider.get_length(dummy_replica), 9)
def setUp(self):
"""
setting up essential objects, copied from tests above
"""
Location.force_initialize()
self.location = Location.get_location('local')
user = '******'
pwd = 'secret'
email = ''
self.user = User.objects.create_user(user, email, pwd)
self.userProfile = UserProfile(user=self.user).save()
self.exp = Experiment(title='test exp1',
institution_name='monash', created_by=self.user)
self.exp.save()
self.acl = ObjectACL(
pluginId=django_user,
entityId=str(self.user.id),
content_object=self.exp,
canRead=True,
isOwner=True,
aclOwnershipType=ObjectACL.OWNER_OWNED,
)
self.acl.save()
self.dataset = Dataset(description='dataset description...')
self.dataset.save()
self.dataset.experiments.add(self.exp)
self.dataset.save()
self.dataset_file = Dataset_File(dataset=self.dataset,
size=42, filename="foo",
md5sum="junk")
self.dataset_file.save()
self.replica = Replica(datafile=self.dataset_file,
url="http://foo",
location=self.location,
verified=False)
self.replica.save()
def _create_datafile():
user = User.objects.create_user("testuser", "*****@*****.**", "pwd")
user.save()
UserProfile(user=user).save()
Location.force_initialize()
full_access = Experiment.PUBLIC_ACCESS_FULL
experiment = Experiment.objects.create(title="IIIF Test", created_by=user, public_access=full_access)
experiment.save()
ObjectACL(
content_object=experiment,
pluginId="django_user",
entityId=str(user.id),
isOwner=True,
canRead=True,
canWrite=True,
canDelete=True,
aclOwnershipType=ObjectACL.OWNER_OWNED,
).save()
dataset = Dataset()
dataset.save()
dataset.experiments.add(experiment)
dataset.save()
# Create new Datafile
tempfile = TemporaryUploadedFile("iiif_stored_file", None, None, None)
with Image(filename="magick:rose") as img:
img.format = "tiff"
img.save(file=tempfile.file)
tempfile.file.flush()
datafile = Dataset_File(dataset=dataset, size=os.path.getsize(tempfile.file.name), filename="iiif_named_file")
replica = Replica(
datafile=datafile,
url=write_uploaded_file_to_dataset(dataset, tempfile),
location=Location.get_default_location(),
)
replica.verify(allowEmptyChecksums=True)
datafile.save()
replica.datafile = datafile
replica.save()
return datafile
def fpupload(request, dataset_id):
"""
Uploads all files picked by filepicker to the dataset
:param request: a HTTP Request instance
:type request: :class:`django.http.HttpRequest`
:param dataset_id: the dataset_id
:type dataset_id: integer
:returns: boolean true if successful
:rtype: bool
"""
dataset = Dataset.objects.get(id=dataset_id)
logger.debug('called fpupload')
if request.method == 'POST':
logger.debug('got POST')
for key, val in request.POST.items():
splits = val.split(",")
for url in splits:
try:
fp = FilepickerFile(url)
except ValueError:
pass
else:
picked_file = fp.get_file()
filepath = write_uploaded_file_to_dataset(dataset,
picked_file)
datafile = Dataset_File(dataset=dataset,
filename=picked_file.name,
size=picked_file.size)
replica = Replica(datafile=datafile,
url=filepath,
protocol='',
location=Location.get_default_location())
replica.verify(allowEmptyChecksums=True)
datafile.save()
replica.datafile = datafile
replica.save()
return HttpResponse(json.dumps({"result": True}))
def _build_datafile(self,
testfile,
filename,
dataset,
url,
protocol='',
checksum=None,
size=None,
mimetype=''):
filesize, sha512sum = get_size_and_sha512sum(testfile)
datafile = Dataset_File(
dataset=dataset,
filename=filename,
mimetype=mimetype,
size=str(size if size != None else filesize),
sha512sum=(checksum if checksum else sha512sum))
datafile.save()
if urlparse.urlparse(url).scheme == '':
location = Location.get_location('local')
else:
location = Location.get_location_for_url(url)
if not location:
location = Location.load_location({
'name':
filename,
'url':
urlparse.urljoin(url, '.'),
'type':
'external',
'priority':
10,
'transfer_provider':
'local'
})
replica = Replica(datafile=datafile,
protocol=protocol,
url=url,
location=location)
replica.verify()
replica.save()
return Dataset_File.objects.get(pk=datafile.pk)
def create_datafile(index):
testfile = path.join(path.dirname(__file__), 'fixtures',
'middleware_test%d.txt' % index)
size, sha512sum = get_size_and_sha512sum(testfile)
datafile = Dataset_File(dataset=dataset,
filename=path.basename(testfile),
size=size,
sha512sum=sha512sum)
datafile.save()
base_url = 'file://' + path.abspath(path.dirname(testfile))
location = Location.load_location({
'name': 'test-middleware', 'url': base_url, 'type': 'external',
'priority': 10, 'transfer_provider': 'local'})
replica = Replica(datafile=datafile,
url='file://'+path.abspath(testfile),
protocol='file',
location=location)
replica.save()
if index != 1:
replica.verified = False
replica.save(update_fields=['verified'])
return Dataset_File.objects.get(pk=datafile.pk)
def migrate_replica(replica, location, noRemove=False, mirror=False):
"""
Migrate the replica to a different storage location. The overall
effect will be that the datafile will be stored at the new location and
removed from the current location, and the datafile metadata will be
updated to reflect this.
"""
from tardis.tardis_portal.models import Replica, Location
with transaction.commit_on_success():
replica = Replica.objects.select_for_update().get(pk=replica.pk)
source = Location.get_location(replica.location.name)
if not replica.verified or location.provider.trust_length:
raise MigrationError('Only verified datafiles can be migrated' \
' to this destination')
filename = replica.get_absolute_filepath()
try:
newreplica = Replica.objects.get(datafile=replica.datafile,
location=location)
created_replica = False
# We've most likely mirrored this file previously. But if
# we are about to delete the source Replica, we need to check
# that the target Replica still verifies.
if not mirror and not check_file_transferred(newreplica, location):
raise MigrationError('Previously mirrored / migrated Replica' \
' no longer verifies locally!')
except Replica.DoesNotExist:
newreplica = Replica()
newreplica.location = location
newreplica.datafile = replica.datafile
newreplica.protocol = ''
newreplica.stay_remote = location != Location.get_default_location(
)
newreplica.verified = False
url = location.provider.generate_url(newreplica)
if newreplica.url == url:
# We should get here ...
raise MigrationError('Cannot migrate a replica to its' \
' current location')
newreplica.url = url
location.provider.put_file(replica, newreplica)
verified = False
try:
verified = check_file_transferred(newreplica, location)
except:
# FIXME - should we always do this?
location.provider.remove_file(newreplica)
raise
newreplica.verified = verified
newreplica.save()
logger.info('Transferred file %s for replica %s' %
(filename, replica.id))
created_replica = True
if mirror:
return created_replica
# FIXME - do this more reliably ...
replica.delete()
if not noRemove:
source.provider.remove_file(replica)
logger.info('Removed local file %s for replica %s' %
(filename, replica.id))
return True
class ViewTemplateContextsTest(TestCase):
def setUp(self):
"""
setting up essential objects, copied from tests above
"""
Location.force_initialize()
self.location = Location.get_location('local')
user = '******'
pwd = 'secret'
email = ''
self.user = User.objects.create_user(user, email, pwd)
self.userProfile = UserProfile(user=self.user).save()
self.exp = Experiment(title='test exp1',
institution_name='monash', created_by=self.user)
self.exp.save()
self.acl = ObjectACL(
pluginId=django_user,
entityId=str(self.user.id),
content_object=self.exp,
canRead=True,
isOwner=True,
aclOwnershipType=ObjectACL.OWNER_OWNED,
)
self.acl.save()
self.dataset = Dataset(description='dataset description...')
self.dataset.save()
self.dataset.experiments.add(self.exp)
self.dataset.save()
self.dataset_file = Dataset_File(dataset=self.dataset,
size=42, filename="foo",
md5sum="junk")
self.dataset_file.save()
self.replica = Replica(datafile=self.dataset_file,
url="http://foo",
location=self.location,
verified=False)
self.replica.save()
def tearDown(self):
self.user.delete()
self.exp.delete()
self.dataset.delete()
self.dataset_file.delete()
self.acl.delete()
def testExperimentView(self):
"""
test some template context parameters for an experiment view
"""
from tardis.tardis_portal.views import view_experiment
from tardis.tardis_portal.shortcuts import render_response_index
from django.http import HttpRequest
from django.template import Context
import sys
# Default behavior
views_module = flexmock(sys.modules['tardis.tardis_portal.views'])
request = HttpRequest()
request.user=self.user
request.groups=[]
context = {'organization': ['classic', 'test', 'test2'],
'default_organization': 'classic',
'default_format': 'zip',
'protocol': [['zip', '/download/experiment/1/zip/'],
['tar', '/download/experiment/1/tar/']]}
views_module.should_call('render_response_index'). \
with_args(_AnyMatcher(), "tardis_portal/view_experiment.html",
_ContextMatcher(context))
response = view_experiment(request, experiment_id=self.exp.id)
self.assertEqual(response.status_code, 200)
# Behavior with USER_AGENT_SENSING enabled and a request.user_agent
saved_setting = getattr(settings, "USER_AGENT_SENSING", None)
try:
setattr(settings, "USER_AGENT_SENSING", True)
request = HttpRequest()
request.user=self.user
request.groups=[]
mock_agent = _MiniMock(os=_MiniMock(family="Macintosh"))
setattr(request, 'user_agent', mock_agent);
context = {'organization': ['classic', 'test', 'test2'],
'default_organization': 'classic',
'default_format': 'tar',
'protocol': [['tar', '/download/experiment/1/tar/']]}
views_module.should_call('render_response_index'). \
with_args(_AnyMatcher(), "tardis_portal/view_experiment.html",
_ContextMatcher(context))
response = view_experiment(request, experiment_id=self.exp.id)
self.assertEqual(response.status_code, 200)
finally:
if saved_setting != None:
setattr(settings, "USER_AGENT_SENSING", saved_setting)
else:
delattr(settings, "USER_AGENT_SENSING")
def testDatasetView(self):
"""
test some context parameters for a dataset view
"""
from tardis.tardis_portal.views import view_dataset
from tardis.tardis_portal.shortcuts import render_response_index
from django.http import HttpRequest
from django.template import Context
import sys
views_module = flexmock(sys.modules['tardis.tardis_portal.views'])
request = HttpRequest()
request.user=self.user
request.groups=[]
context = {'default_organization': 'classic',
'default_format': 'zip'}
views_module.should_call('render_response_index'). \
with_args(_AnyMatcher(), "tardis_portal/view_dataset.html",
_ContextMatcher(context))
response = view_dataset(request, dataset_id=self.dataset.id)
self.assertEqual(response.status_code, 200)
# Behavior with USER_AGENT_SENSING enabled and a request.user_agent
saved_setting = getattr(settings, "USER_AGENT_SENSING", None)
try:
setattr(settings, "USER_AGENT_SENSING", True)
request = HttpRequest()
request.user=self.user
request.groups=[]
mock_agent = _MiniMock(os=_MiniMock(family="Macintosh"))
setattr(request, 'user_agent', mock_agent);
context = {'default_organization': 'classic',
'default_format': 'tar'}
views_module.should_call('render_response_index'). \
with_args(_AnyMatcher(), "tardis_portal/view_dataset.html",
_ContextMatcher(context))
response = view_dataset(request, dataset_id=self.dataset.id)
self.assertEqual(response.status_code, 200)
finally:
if saved_setting != None:
setattr(settings, "USER_AGENT_SENSING", saved_setting)
else:
delattr(settings, "USER_AGENT_SENSING")
def migrate_replica(replica, location, noRemove=False, mirror=False):
"""
Migrate the replica to a different storage location. The overall
effect will be that the datafile will be stored at the new location and
removed from the current location, and the datafile metadata will be
updated to reflect this.
"""
from tardis.tardis_portal.models import Replica, Location
with transaction.commit_on_success():
replica = Replica.objects.select_for_update().get(pk=replica.pk)
source = Location.get_location(replica.location.name)
if not replica.verified or location.provider.trust_length:
raise MigrationError('Only verified datafiles can be migrated' \
' to this destination')
filename = replica.get_absolute_filepath()
try:
newreplica = Replica.objects.get(datafile=replica.datafile,
location=location)
created_replica = False
# We've most likely mirrored this file previously. But if
# we are about to delete the source Replica, we need to check
# that the target Replica still verifies.
if not mirror and not check_file_transferred(newreplica, location):
raise MigrationError('Previously mirrored / migrated Replica' \
' no longer verifies locally!')
except Replica.DoesNotExist:
newreplica = Replica()
newreplica.location = location
newreplica.datafile = replica.datafile
newreplica.protocol = ''
newreplica.stay_remote = location != Location.get_default_location()
newreplica.verified = False
url = location.provider.generate_url(newreplica)
if newreplica.url == url:
# We should get here ...
raise MigrationError('Cannot migrate a replica to its' \
' current location')
newreplica.url = url
location.provider.put_file(replica, newreplica)
verified = False
try:
verified = check_file_transferred(newreplica, location)
except:
# FIXME - should we always do this?
location.provider.remove_file(newreplica)
raise
newreplica.verified = verified
newreplica.save()
logger.info('Transferred file %s for replica %s' %
(filename, replica.id))
created_replica = True
if mirror:
return created_replica
# FIXME - do this more reliably ...
replica.delete()
if not noRemove:
source.provider.remove_file(replica)
logger.info('Removed local file %s for replica %s' %
(filename, replica.id))
return True
class ViewTemplateContextsTest(TestCase):
def setUp(self):
"""
setting up essential objects, copied from tests above
"""
Location.force_initialize()
self.location = Location.get_location('local')
user = '******'
pwd = 'secret'
email = ''
self.user = User.objects.create_user(user, email, pwd)
self.userProfile = UserProfile(user=self.user).save()
self.exp = Experiment(title='test exp1',
institution_name='monash',
created_by=self.user)
self.exp.save()
self.acl = ObjectACL(
pluginId=django_user,
entityId=str(self.user.id),
content_object=self.exp,
canRead=True,
isOwner=True,
aclOwnershipType=ObjectACL.OWNER_OWNED,
)
self.acl.save()
self.dataset = Dataset(description='dataset description...')
self.dataset.save()
self.dataset.experiments.add(self.exp)
self.dataset.save()
self.dataset_file = Dataset_File(dataset=self.dataset,
size=42,
filename="foo",
md5sum="junk")
self.dataset_file.save()
self.replica = Replica(datafile=self.dataset_file,
url="http://foo",
location=self.location,
verified=False)
self.replica.save()
def tearDown(self):
self.user.delete()
self.exp.delete()
self.dataset.delete()
self.dataset_file.delete()
self.acl.delete()
def testExperimentView(self):
"""
test some template context parameters for an experiment view
"""
from tardis.tardis_portal.views import view_experiment
from tardis.tardis_portal.shortcuts import render_response_index
from django.http import HttpRequest
from django.template import Context
import sys
# Default behavior
views_module = flexmock(sys.modules['tardis.tardis_portal.views'])
request = HttpRequest()
request.user = self.user
request.groups = []
context = {
'organization': ['test', 'test2'],
'default_organization':
'test',
'default_format':
'tar',
'protocol': [['tgz', '/download/experiment/1/tgz/'],
['tar', '/download/experiment/1/tar/']]
}
views_module.should_call('render_response_index'). \
with_args(_AnyMatcher(), "tardis_portal/view_experiment.html",
_ContextMatcher(context))
response = view_experiment(request, experiment_id=self.exp.id)
self.assertEqual(response.status_code, 200)
# Behavior with USER_AGENT_SENSING enabled and a request.user_agent
saved_setting = getattr(settings, "USER_AGENT_SENSING", None)
try:
setattr(settings, "USER_AGENT_SENSING", True)
request = HttpRequest()
request.user = self.user
request.groups = []
mock_agent = _MiniMock(os=_MiniMock(family="Macintosh"))
setattr(request, 'user_agent', mock_agent)
context = {
'organization': ['classic', 'test', 'test2'],
'default_organization': 'classic',
'default_format': 'tar',
'protocol': [['tar', '/download/experiment/1/tar/']]
}
views_module.should_call('render_response_index'). \
with_args(_AnyMatcher(), "tardis_portal/view_experiment.html",
_ContextMatcher(context))
response = view_experiment(request, experiment_id=self.exp.id)
self.assertEqual(response.status_code, 200)
finally:
if saved_setting != None:
setattr(settings, "USER_AGENT_SENSING", saved_setting)
else:
delattr(settings, "USER_AGENT_SENSING")
def testDatasetView(self):
"""
test some context parameters for a dataset view
"""
from tardis.tardis_portal.views import view_dataset
from tardis.tardis_portal.shortcuts import render_response_index
from django.http import HttpRequest
from django.template import Context
import sys
views_module = flexmock(sys.modules['tardis.tardis_portal.views'])
request = HttpRequest()
request.user = self.user
request.groups = []
context = {'default_organization': 'test', 'default_format': 'tar'}
views_module.should_call('render_response_index'). \
with_args(_AnyMatcher(), "tardis_portal/view_dataset.html",
_ContextMatcher(context))
response = view_dataset(request, dataset_id=self.dataset.id)
self.assertEqual(response.status_code, 200)
# Behavior with USER_AGENT_SENSING enabled and a request.user_agent
saved_setting = getattr(settings, "USER_AGENT_SENSING", None)
try:
setattr(settings, "USER_AGENT_SENSING", True)
request = HttpRequest()
request.user = self.user
request.groups = []
mock_agent = _MiniMock(os=_MiniMock(family="Macintosh"))
setattr(request, 'user_agent', mock_agent)
context = {
'default_organization': 'classic',
'default_format': 'tar'
}
views_module.should_call('render_response_index'). \
with_args(_AnyMatcher(), "tardis_portal/view_dataset.html",
_ContextMatcher(context))
response = view_dataset(request, dataset_id=self.dataset.id)
self.assertEqual(response.status_code, 200)
finally:
if saved_setting != None:
setattr(settings, "USER_AGENT_SENSING", saved_setting)
else:
delattr(settings, "USER_AGENT_SENSING")
