def setUp(self): from tardis.tardis_portal import models from tempfile import mkdtemp, mktemp from django.conf import settings from os import path import os # Disconnect post_save signal from django.db.models.signals import post_save from tardis.tardis_portal.models import \ staging_hook, Dataset_File, Replica, Location post_save.disconnect(staging_hook, sender=Replica) from django.contrib.auth.models import User user = '******' pwd = 'secret' email = '' self.user = User.objects.create_user(user, email, pwd) try: os.makedirs(settings.GET_FULL_STAGING_PATH_TEST) except OSError: pass self.temp = mkdtemp(dir=settings.GET_FULL_STAGING_PATH_TEST) self.file = mktemp(dir=self.temp) content = 'test file' with open(self.file, "w+b") as f: f.write(content) Location.force_initialize() # make datafile exp = models.Experiment(title='test exp1', institution_name='monash', created_by=self.user) exp.save() # make dataset dataset = models.Dataset(description="dataset description...") dataset.save() dataset.experiments.add(exp) dataset.save() # create datafile df = models.Dataset_File(dataset=dataset, size = len(content), filename = path.basename(self.file), md5sum='f20d9f2072bbeb6691c0f9c5099b01f3') df.save() # create replica base_url = 'file://' + settings.GET_FULL_STAGING_PATH_TEST location = Location.load_location({ 'name': 'staging-test-yyy', 'url': base_url, 'type': 'external', 'priority': 10, 'transfer_provider': 'local'}) replica = models.Replica(datafile=df, url='file://'+self.file, protocol="staging",location=location) replica.verify() replica.save() self.replica = replica
def create_datafile(index): testfile = path.join(path.dirname(__file__), 'fixtures', 'jeol_sem_test%d.txt' % index) size, sha512sum = get_size_and_sha512sum(testfile) datafile = Dataset_File(dataset=dataset, filename=path.basename(testfile), size=size, sha512sum=sha512sum) datafile.save() base_url = 'file://' + path.abspath(path.dirname(testfile)) location = Location.load_location({ 'name': 'test-jeol', 'url': base_url, 'type': 'external', 'priority': 10, 'transfer_provider': 'local' }) replica = Replica(datafile=datafile, url='file://' + path.abspath(testfile), protocol='file', location=location) replica.verify() replica.save() return Dataset_File.objects.get(pk=datafile.pk)
def setUpClass(cls): cls.priorcwd = os.getcwd() os.chdir(os.path.dirname(__file__)+'/atom_test') cls.server = TestWebServer() cls.server.start() Location.force_initialize() Location.load_location({ 'name': 'test-atom', 'transfer_provider': 'http', 'url': 'http://localhost:4272/files/', 'type': 'external', 'priority': 10}) Location.load_location({ 'name': 'test-atom2', 'transfer_provider': 'http', 'url': 'http://mydatagrabber.cmm.uq.edu.au/files', 'type': 'external', 'priority': 10}) files = path.realpath(path.join(path.dirname(__file__), 'atom_test', 'files')) Location.load_location({ 'name': 'test-atom3', 'transfer_provider': 'local', 'url': 'file://' + files, 'type': 'external', 'priority': 10})
def setUpClass(cls): cls.priorcwd = os.getcwd() os.chdir(os.path.dirname(__file__) + '/atom_test') cls.server = TestWebServer() cls.server.start() Location.force_initialize() Location.load_location({ 'name': 'test-atom', 'transfer_provider': 'http', 'url': 'http://localhost:4272/files/', 'type': 'external', 'priority': 10 }) Location.load_location({ 'name': 'test-atom2', 'transfer_provider': 'http', 'url': 'http://mydatagrabber.cmm.uq.edu.au/files', 'type': 'external', 'priority': 10 }) files = path.realpath( path.join(path.dirname(__file__), 'atom_test', 'files')) Location.load_location({ 'name': 'test-atom3', 'transfer_provider': 'local', 'url': 'file://' + files, 'type': 'external', 'priority': 10 })
def _build_datafile(self, testfile, filename, dataset, url, protocol='', checksum=None, size=None, mimetype=''): filesize, sha512sum = get_size_and_sha512sum(testfile) datafile = Dataset_File( dataset=dataset, filename=filename, mimetype=mimetype, size=str(size if size != None else filesize), sha512sum=(checksum if checksum else sha512sum)) datafile.save() if urlparse.urlparse(url).scheme == '': location = Location.get_location('local') else: location = Location.get_location_for_url(url) if not location: location = Location.load_location({ 'name': filename, 'url': urlparse.urljoin(url, '.'), 'type': 'external', 'priority': 10, 'transfer_provider': 'local' }) replica = Replica(datafile=datafile, protocol=protocol, url=url, location=location) replica.verify() replica.save() return Dataset_File.objects.get(pk=datafile.pk)
def create_datafile(file_path): testfile = path.join(path.dirname(__file__), 'fixtures', file_path) size, sha512sum = get_size_and_sha512sum(testfile) datafile = Dataset_File(dataset=dataset, filename=path.basename(testfile), size=size, sha512sum=sha512sum) datafile.save() base_url = 'file://' + path.abspath(path.dirname(testfile)) location = Location.load_location({ 'name': 'test-flexstation', 'url': base_url, 'type': 'external', 'priority': 10, 'transfer_provider': 'local'}) replica = Replica(datafile=datafile, url='file://'+ path.abspath(testfile), protocol='file', location=location) replica.verify() replica.save() return Dataset_File.objects.get(pk=datafile.pk)
def _build_datafile(self, testfile, filename, dataset, url, protocol='', checksum=None, size=None, mimetype=''): filesize, sha512sum = get_size_and_sha512sum(testfile) datafile = Dataset_File(dataset=dataset, filename=filename, mimetype=mimetype, size=str(size if size != None else filesize), sha512sum=(checksum if checksum else sha512sum)) datafile.save() if urlparse.urlparse(url).scheme == '': location = Location.get_location('local') else: location = Location.get_location_for_url(url) if not location: location = Location.load_location({ 'name': filename, 'url': urlparse.urljoin(url, '.'), 'type': 'external', 'priority': 10, 'transfer_provider': 'local'}) replica = Replica(datafile=datafile, protocol=protocol, url=url, location=location) replica.verify() replica.save() return Dataset_File.objects.get(pk=datafile.pk)
def create_datafile(index): testfile = path.join(path.dirname(__file__), 'fixtures', 'middleware_test%d.txt' % index) size, sha512sum = get_size_and_sha512sum(testfile) datafile = Dataset_File(dataset=dataset, filename=path.basename(testfile), size=size, sha512sum=sha512sum) datafile.save() base_url = 'file://' + path.abspath(path.dirname(testfile)) location = Location.load_location({ 'name': 'test-middleware', 'url': base_url, 'type': 'external', 'priority': 10, 'transfer_provider': 'local'}) replica = Replica(datafile=datafile, url='file://'+path.abspath(testfile), protocol='file', location=location) replica.save() if index != 1: replica.verified = False replica.save(update_fields=['verified']) return Dataset_File.objects.get(pk=datafile.pk)
def setUpClass(cls): cls.priorcwd = os.getcwd() os.chdir(os.path.dirname(__file__) + "/atom_test") cls.server = TestWebServer() cls.server.start() Location.force_initialize() Location.load_location( { "name": "test-atom", "transfer_provider": "http", "url": "http://localhost:4272/files/", "type": "external", "priority": 10, } ) Location.load_location( { "name": "test-atom2", "transfer_provider": "http", "url": "http://mydatagrabber.cmm.uq.edu.au/files", "type": "external", "priority": 10, } ) files = path.realpath(path.join(path.dirname(__file__), "atom_test", "files")) Location.load_location( { "name": "test-atom3", "transfer_provider": "local", "url": "file://" + files, "type": "external", "priority": 10, } )
def setUp(self): from tardis.tardis_portal import models from tempfile import mkdtemp, mktemp from django.conf import settings from os import path import os # Disconnect post_save signal from django.db.models.signals import post_save from tardis.tardis_portal.models import \ staging_hook, Dataset_File, Replica, Location post_save.disconnect(staging_hook, sender=Replica) from django.contrib.auth.models import User user = '******' pwd = 'secret' email = '' self.user = User.objects.create_user(user, email, pwd) try: os.makedirs(settings.GET_FULL_STAGING_PATH_TEST) except OSError: pass self.temp = mkdtemp(dir=settings.GET_FULL_STAGING_PATH_TEST) self.file = mktemp(dir=self.temp) content = 'test file' with open(self.file, "w+b") as f: f.write(content) Location.force_initialize() # make datafile exp = models.Experiment(title='test exp1', institution_name='monash', created_by=self.user) exp.save() # make dataset dataset = models.Dataset(description="dataset description...") dataset.save() dataset.experiments.add(exp) dataset.save() # create datafile df = models.Dataset_File(dataset=dataset, size=len(content), filename=path.basename(self.file), md5sum='f20d9f2072bbeb6691c0f9c5099b01f3') df.save() # create replica base_url = 'file://' + settings.GET_FULL_STAGING_PATH_TEST location = Location.load_location({ 'name': 'staging-test-yyy', 'url': base_url, 'type': 'external', 'priority': 10, 'transfer_provider': 'local' }) replica = models.Replica(datafile=df, url='file://' + self.file, protocol="staging", location=location) replica.verify() replica.save() self.replica = replica
def _get_or_create_local_location(self, name, url, type, priority): return Location.load_location({ 'name': name, 'url': url, 'type': type, 'priority': priority, 'transfer_provider': 'local'})