def stage_replica(replica): from django.core.files.uploadedfile import TemporaryUploadedFile from tardis.tardis_portal.models import Replica, Location if not replica.location.type == 'external': raise ValueError('Only external replicas can be staged') if getattr(settings, "DEEP_DATASET_STORAGE", False): relurl = path.relpath(replica.url[7:], settings.SYNC_TEMP_PATH) spliturl = relurl.split(os.sep)[1:] subdir = path.dirname(path.join(*spliturl)) else: subdir = None with TemporaryUploadedFile(replica.datafile.filename, None, None, None) as tf: if replica.verify(tempfile=tf.file): if not replica.stay_remote: tf.file.flush() target_replica = { 'datafile': replica.datafile, 'url': write_uploaded_file_to_dataset(replica.datafile.dataset, tf, subdir=subdir), 'location': Location.get_default_location(), 'verified': True, 'protocol': '' } Replica.objects.filter(id=replica.id).update(**target_replica) return True else: return False
def test_location(self): from tardis.tardis_portal.models import Location self.assertEquals(Location.get_default_location().name, 'local') self.assertEquals(Location.get_location('staging').name, 'staging') self.assertEquals(len(Location.objects.all()), 6)
def stage_replica(replica): from django.core.files.uploadedfile import TemporaryUploadedFile from tardis.tardis_portal.models import Replica, Location if not replica.location.type == 'external': raise ValueError('Only external replicas can be staged') if getattr(settings, "DEEP_DATASET_STORAGE", False): relurl = path.relpath(replica.url[7:], settings.SYNC_TEMP_PATH) spliturl = relurl.split(os.sep)[1:] subdir = path.dirname(path.join(*spliturl)) else: subdir = None with TemporaryUploadedFile(replica.datafile.filename, None, None, None) as tf: if replica.verify(tempfile=tf.file): if not replica.stay_remote: tf.file.flush() target_replica = { 'datafile': replica.datafile, 'url': write_uploaded_file_to_dataset( replica.datafile.dataset, tf, subdir=subdir), 'location': Location.get_default_location(), 'verified': True, 'protocol': ''} Replica.objects.filter(id=replica.id).update(**target_replica) return True else: return False
def testLocalFile(self): content = urandom(1024) cf = ContentFile(content, 'background_task_testfile') # Create new Datafile datafile = Dataset_File(dataset=self.dataset) datafile.filename = cf.name datafile.size = len(content) datafile.sha512sum = hashlib.sha512(content).hexdigest() datafile.save() replica = Replica(datafile=datafile, url=write_uploaded_file_to_dataset(self.dataset, cf), location=Location.get_default_location()) replica.save() def get_replica(datafile): return Replica.objects.get(datafile=datafile) # undo auto-verify: replica.verified = False replica.save(update_fields=['verified']) # Check that it's not currently verified expect(get_replica(datafile).verified).to_be(False) # Check it verifies verify_files() expect(get_replica(datafile).verified).to_be(True)
def _infer_location(path): if urlparse.urlparse(path).scheme == '': loc = Location.get_default_location() else: loc = Location.get_location_for_url(path) if loc: return loc else: raise Exception('Cannot infer a location for %s' % path)
def _build(dataset, filename, url, protocol): from tardis.tardis_portal.models import \ Dataset_File, Replica, Location datafile = Dataset_File(dataset=dataset, filename=filename) datafile.save() replica = Replica(datafile=datafile, url=url, protocol=protocol, location=Location.get_default_location()) replica.save() return datafile
def _create_test_dataset(nosDatafiles): ds_ = Dataset(description='happy snaps of plumage') ds_.save() for i in range(0, nosDatafiles): df_ = Dataset_File(dataset=ds_, size='21', sha512sum='bogus') df_.save() rep_ = Replica(datafile=df_, url='http://planet-python.org/' + str(_next_id()), location=Location.get_default_location()) rep_.save() ds_.save() return ds_
def _create_test_dataset(nosDatafiles): ds_ = Dataset(description='happy snaps of plumage') ds_.save() for i in range (0, nosDatafiles) : df_ = Dataset_File(dataset=ds_, size='21', sha512sum='bogus') df_.save() rep_ = Replica(datafile=df_, url='http://planet-python.org/' + str(_next_id()), location=Location.get_default_location()) rep_.save() ds_.save() return ds_
def _create_datafile(): user = User.objects.create_user("testuser", "*****@*****.**", "pwd") user.save() UserProfile(user=user).save() Location.force_initialize() full_access = Experiment.PUBLIC_ACCESS_FULL experiment = Experiment.objects.create(title="IIIF Test", created_by=user, public_access=full_access) experiment.save() ObjectACL( content_object=experiment, pluginId="django_user", entityId=str(user.id), isOwner=True, canRead=True, canWrite=True, canDelete=True, aclOwnershipType=ObjectACL.OWNER_OWNED, ).save() dataset = Dataset() dataset.save() dataset.experiments.add(experiment) dataset.save() # Create new Datafile tempfile = TemporaryUploadedFile("iiif_stored_file", None, None, None) with Image(filename="magick:rose") as img: img.format = "tiff" img.save(file=tempfile.file) tempfile.file.flush() datafile = Dataset_File(dataset=dataset, size=os.path.getsize(tempfile.file.name), filename="iiif_named_file") replica = Replica( datafile=datafile, url=write_uploaded_file_to_dataset(dataset, tempfile), location=Location.get_default_location(), ) replica.verify(allowEmptyChecksums=True) datafile.save() replica.datafile = datafile replica.save() return datafile
def _create_datafile(): user = User.objects.create_user('testuser', '*****@*****.**', 'pwd') user.save() UserProfile(user=user).save() Location.force_initialize() full_access = Experiment.PUBLIC_ACCESS_FULL experiment = Experiment.objects.create(title="IIIF Test", created_by=user, public_access=full_access) experiment.save() ObjectACL(content_object=experiment, pluginId='django_user', entityId=str(user.id), isOwner=True, canRead=True, canWrite=True, canDelete=True, aclOwnershipType=ObjectACL.OWNER_OWNED).save() dataset = Dataset() dataset.save() dataset.experiments.add(experiment) dataset.save() # Create new Datafile tempfile = TemporaryUploadedFile('iiif_stored_file', None, None, None) with Image(filename='magick:rose') as img: img.format = 'tiff' img.save(file=tempfile.file) tempfile.file.flush() datafile = Dataset_File(dataset=dataset, size=os.path.getsize(tempfile.file.name), filename='iiif_named_file') replica = Replica(datafile=datafile, url=write_uploaded_file_to_dataset(dataset, tempfile), location=Location.get_default_location()) replica.verify(allowEmptyChecksums=True) datafile.save() replica.datafile = datafile replica.save() return datafile
def fpupload(request, dataset_id): """ Uploads all files picked by filepicker to the dataset :param request: a HTTP Request instance :type request: :class:`django.http.HttpRequest` :param dataset_id: the dataset_id :type dataset_id: integer :returns: boolean true if successful :rtype: bool """ dataset = Dataset.objects.get(id=dataset_id) logger.debug('called fpupload') if request.method == 'POST': logger.debug('got POST') for key, val in request.POST.items(): splits = val.split(",") for url in splits: try: fp = FilepickerFile(url) except ValueError: pass else: picked_file = fp.get_file() filepath = write_uploaded_file_to_dataset(dataset, picked_file) datafile = Dataset_File(dataset=dataset, filename=picked_file.name, size=picked_file.size) replica = Replica(datafile=datafile, url=filepath, protocol='', location=Location.get_default_location()) replica.verify(allowEmptyChecksums=True) datafile.save() replica.datafile = datafile replica.save() return HttpResponse(json.dumps({"result": True}))
def fpupload(request, dataset_id): """ Uploads all files picked by filepicker to the dataset :param request: a HTTP Request instance :type request: :class:`django.http.HttpRequest` :param dataset_id: the dataset_id :type dataset_id: integer :returns: boolean true if successful :rtype: bool """ dataset = Dataset.objects.get(id=dataset_id) logger.debug('called fpupload') if request.method == 'POST': logger.debug('got POST') for key, val in request.POST.items(): splits = val.split(",") for url in splits: try: fp = FilepickerFile(url) except ValueError: pass else: picked_file = fp.get_file() filepath = write_uploaded_file_to_dataset( dataset, picked_file) datafile = Dataset_File(dataset=dataset, filename=picked_file.name, size=picked_file.size) replica = Replica(datafile=datafile, url=filepath, protocol='', location=Location.get_default_location()) replica.verify(allowEmptyChecksums=True) datafile.save() replica.datafile = datafile replica.save() return HttpResponse(json.dumps({"result": True}))
def stage_replica(replica): from django.core.files.uploadedfile import TemporaryUploadedFile from tardis.tardis_portal.models import Replica, Location if not replica.location.type == 'external': raise ValueError('Only external replicas can be staged') with TemporaryUploadedFile(replica.datafile.filename, None, None, None) as tf: if replica.verify(tempfile=tf.file): if not replica.stay_remote: tf.file.flush() target_replica = Replica( datafile=replica.datafile, url=write_uploaded_file_to_dataset(\ replica.datafile.dataset, tf), location=Location.get_default_location(), verified=True, protocol='') target_replica.save() replica.delete() return True else: return False
def migrate_replica(replica, location, noRemove=False, mirror=False): """ Migrate the replica to a different storage location. The overall effect will be that the datafile will be stored at the new location and removed from the current location, and the datafile metadata will be updated to reflect this. """ from tardis.tardis_portal.models import Replica, Location with transaction.commit_on_success(): replica = Replica.objects.select_for_update().get(pk=replica.pk) source = Location.get_location(replica.location.name) if not replica.verified or location.provider.trust_length: raise MigrationError('Only verified datafiles can be migrated' \ ' to this destination') filename = replica.get_absolute_filepath() try: newreplica = Replica.objects.get(datafile=replica.datafile, location=location) created_replica = False # We've most likely mirrored this file previously. But if # we are about to delete the source Replica, we need to check # that the target Replica still verifies. if not mirror and not check_file_transferred(newreplica, location): raise MigrationError('Previously mirrored / migrated Replica' \ ' no longer verifies locally!') except Replica.DoesNotExist: newreplica = Replica() newreplica.location = location newreplica.datafile = replica.datafile newreplica.protocol = '' newreplica.stay_remote = location != Location.get_default_location() newreplica.verified = False url = location.provider.generate_url(newreplica) if newreplica.url == url: # We should get here ... raise MigrationError('Cannot migrate a replica to its' \ ' current location') newreplica.url = url location.provider.put_file(replica, newreplica) verified = False try: verified = check_file_transferred(newreplica, location) except: # FIXME - should we always do this? location.provider.remove_file(newreplica) raise newreplica.verified = verified newreplica.save() logger.info('Transferred file %s for replica %s' % (filename, replica.id)) created_replica = True if mirror: return created_replica # FIXME - do this more reliably ... replica.delete() if not noRemove: source.provider.remove_file(replica) logger.info('Removed local file %s for replica %s' % (filename, replica.id)) return True
def migrate_replica(replica, location, noRemove=False, mirror=False): """ Migrate the replica to a different storage location. The overall effect will be that the datafile will be stored at the new location and removed from the current location, and the datafile metadata will be updated to reflect this. """ from tardis.tardis_portal.models import Replica, Location with transaction.commit_on_success(): replica = Replica.objects.select_for_update().get(pk=replica.pk) source = Location.get_location(replica.location.name) if not replica.verified or location.provider.trust_length: raise MigrationError('Only verified datafiles can be migrated' \ ' to this destination') filename = replica.get_absolute_filepath() try: newreplica = Replica.objects.get(datafile=replica.datafile, location=location) created_replica = False # We've most likely mirrored this file previously. But if # we are about to delete the source Replica, we need to check # that the target Replica still verifies. if not mirror and not check_file_transferred(newreplica, location): raise MigrationError('Previously mirrored / migrated Replica' \ ' no longer verifies locally!') except Replica.DoesNotExist: newreplica = Replica() newreplica.location = location newreplica.datafile = replica.datafile newreplica.protocol = '' newreplica.stay_remote = location != Location.get_default_location( ) newreplica.verified = False url = location.provider.generate_url(newreplica) if newreplica.url == url: # We should get here ... raise MigrationError('Cannot migrate a replica to its' \ ' current location') newreplica.url = url location.provider.put_file(replica, newreplica) verified = False try: verified = check_file_transferred(newreplica, location) except: # FIXME - should we always do this? location.provider.remove_file(newreplica) raise newreplica.verified = verified newreplica.save() logger.info('Transferred file %s for replica %s' % (filename, replica.id)) created_replica = True if mirror: return created_replica # FIXME - do this more reliably ... replica.delete() if not noRemove: source.provider.remove_file(replica) logger.info('Removed local file %s for replica %s' % (filename, replica.id)) return True
def get_new_replica(datafile): location = Location.get_default_location() return Replica.objects.get(datafile=datafile.id, location=location)