def setUp(self):
from tardis.tardis_portal import models
from tempfile import mkdtemp, mktemp
from django.conf import settings
from os import path
import os
# Disconnect post_save signal
from django.db.models.signals import post_save
from tardis.tardis_portal.models import \
staging_hook, Dataset_File, Replica, Location
post_save.disconnect(staging_hook, sender=Replica)
from django.contrib.auth.models import User
user = '******'
pwd = 'secret'
email = ''
self.user = User.objects.create_user(user, email, pwd)
try:
os.makedirs(settings.GET_FULL_STAGING_PATH_TEST)
except OSError:
pass
self.temp = mkdtemp(dir=settings.GET_FULL_STAGING_PATH_TEST)
self.file = mktemp(dir=self.temp)
content = 'test file'
with open(self.file, "w+b") as f:
f.write(content)
Location.force_initialize()
# make datafile
exp = models.Experiment(title='test exp1',
institution_name='monash',
created_by=self.user)
exp.save()
# make dataset
dataset = models.Dataset(description="dataset description...")
dataset.save()
dataset.experiments.add(exp)
dataset.save()
# create datafile
df = models.Dataset_File(dataset=dataset, size = len(content),
filename = path.basename(self.file),
md5sum='f20d9f2072bbeb6691c0f9c5099b01f3')
df.save()
# create replica
base_url = 'file://' + settings.GET_FULL_STAGING_PATH_TEST
location = Location.load_location({
'name': 'staging-test-yyy', 'url': base_url, 'type': 'external',
'priority': 10, 'transfer_provider': 'local'})
replica = models.Replica(datafile=df, url='file://'+self.file,
protocol="staging",location=location)
replica.verify()
replica.save()
self.replica = replica
def create_datafile(index):
testfile = path.join(path.dirname(__file__), 'fixtures',
'jeol_sem_test%d.txt' % index)
size, sha512sum = get_size_and_sha512sum(testfile)
datafile = Dataset_File(dataset=dataset,
filename=path.basename(testfile),
size=size,
sha512sum=sha512sum)
datafile.save()
base_url = 'file://' + path.abspath(path.dirname(testfile))
location = Location.load_location({
'name': 'test-jeol',
'url': base_url,
'type': 'external',
'priority': 10,
'transfer_provider': 'local'
})
replica = Replica(datafile=datafile,
url='file://' + path.abspath(testfile),
protocol='file',
location=location)
replica.verify()
replica.save()
return Dataset_File.objects.get(pk=datafile.pk)
def setUpClass(cls):
cls.priorcwd = os.getcwd()
os.chdir(os.path.dirname(__file__)+'/atom_test')
cls.server = TestWebServer()
cls.server.start()
Location.force_initialize()
Location.load_location({
'name': 'test-atom',
'transfer_provider': 'http',
'url': 'http://localhost:4272/files/',
'type': 'external',
'priority': 10})
Location.load_location({
'name': 'test-atom2',
'transfer_provider': 'http',
'url': 'http://mydatagrabber.cmm.uq.edu.au/files',
'type': 'external',
'priority': 10})
files = path.realpath(path.join(path.dirname(__file__),
'atom_test', 'files'))
Location.load_location({
'name': 'test-atom3',
'transfer_provider': 'local',
'url': 'file://' + files,
'type': 'external',
'priority': 10})
def setUpClass(cls):
cls.priorcwd = os.getcwd()
os.chdir(os.path.dirname(__file__) + '/atom_test')
cls.server = TestWebServer()
cls.server.start()
Location.force_initialize()
Location.load_location({
'name': 'test-atom',
'transfer_provider': 'http',
'url': 'http://localhost:4272/files/',
'type': 'external',
'priority': 10
})
Location.load_location({
'name': 'test-atom2',
'transfer_provider': 'http',
'url': 'http://mydatagrabber.cmm.uq.edu.au/files',
'type': 'external',
'priority': 10
})
files = path.realpath(
path.join(path.dirname(__file__), 'atom_test', 'files'))
Location.load_location({
'name': 'test-atom3',
'transfer_provider': 'local',
'url': 'file://' + files,
'type': 'external',
'priority': 10
})
def _build_datafile(self,
testfile,
filename,
dataset,
url,
protocol='',
checksum=None,
size=None,
mimetype=''):
filesize, sha512sum = get_size_and_sha512sum(testfile)
datafile = Dataset_File(
dataset=dataset,
filename=filename,
mimetype=mimetype,
size=str(size if size != None else filesize),
sha512sum=(checksum if checksum else sha512sum))
datafile.save()
if urlparse.urlparse(url).scheme == '':
location = Location.get_location('local')
else:
location = Location.get_location_for_url(url)
if not location:
location = Location.load_location({
'name':
filename,
'url':
urlparse.urljoin(url, '.'),
'type':
'external',
'priority':
10,
'transfer_provider':
'local'
})
replica = Replica(datafile=datafile,
protocol=protocol,
url=url,
location=location)
replica.verify()
replica.save()
return Dataset_File.objects.get(pk=datafile.pk)
def create_datafile(file_path):
testfile = path.join(path.dirname(__file__), 'fixtures', file_path)
size, sha512sum = get_size_and_sha512sum(testfile)
datafile = Dataset_File(dataset=dataset,
filename=path.basename(testfile),
size=size,
sha512sum=sha512sum)
datafile.save()
base_url = 'file://' + path.abspath(path.dirname(testfile))
location = Location.load_location({
'name': 'test-flexstation', 'url': base_url, 'type': 'external',
'priority': 10, 'transfer_provider': 'local'})
replica = Replica(datafile=datafile,
url='file://'+ path.abspath(testfile),
protocol='file',
location=location)
replica.verify()
replica.save()
return Dataset_File.objects.get(pk=datafile.pk)
def _build_datafile(self, testfile, filename, dataset, url,
protocol='', checksum=None, size=None, mimetype=''):
filesize, sha512sum = get_size_and_sha512sum(testfile)
datafile = Dataset_File(dataset=dataset, filename=filename,
mimetype=mimetype,
size=str(size if size != None else filesize),
sha512sum=(checksum if checksum else sha512sum))
datafile.save()
if urlparse.urlparse(url).scheme == '':
location = Location.get_location('local')
else:
location = Location.get_location_for_url(url)
if not location:
location = Location.load_location({
'name': filename, 'url': urlparse.urljoin(url, '.'),
'type': 'external',
'priority': 10, 'transfer_provider': 'local'})
replica = Replica(datafile=datafile, protocol=protocol, url=url,
location=location)
replica.verify()
replica.save()
return Dataset_File.objects.get(pk=datafile.pk)
def create_datafile(index):
testfile = path.join(path.dirname(__file__), 'fixtures',
'middleware_test%d.txt' % index)
size, sha512sum = get_size_and_sha512sum(testfile)
datafile = Dataset_File(dataset=dataset,
filename=path.basename(testfile),
size=size,
sha512sum=sha512sum)
datafile.save()
base_url = 'file://' + path.abspath(path.dirname(testfile))
location = Location.load_location({
'name': 'test-middleware', 'url': base_url, 'type': 'external',
'priority': 10, 'transfer_provider': 'local'})
replica = Replica(datafile=datafile,
url='file://'+path.abspath(testfile),
protocol='file',
location=location)
replica.save()
if index != 1:
replica.verified = False
replica.save(update_fields=['verified'])
return Dataset_File.objects.get(pk=datafile.pk)
def setUpClass(cls):
cls.priorcwd = os.getcwd()
os.chdir(os.path.dirname(__file__) + "/atom_test")
cls.server = TestWebServer()
cls.server.start()
Location.force_initialize()
Location.load_location(
{
"name": "test-atom",
"transfer_provider": "http",
"url": "http://localhost:4272/files/",
"type": "external",
"priority": 10,
}
)
Location.load_location(
{
"name": "test-atom2",
"transfer_provider": "http",
"url": "http://mydatagrabber.cmm.uq.edu.au/files",
"type": "external",
"priority": 10,
}
)
files = path.realpath(path.join(path.dirname(__file__), "atom_test", "files"))
Location.load_location(
{
"name": "test-atom3",
"transfer_provider": "local",
"url": "file://" + files,
"type": "external",
"priority": 10,
}
)
def setUp(self):
from tardis.tardis_portal import models
from tempfile import mkdtemp, mktemp
from django.conf import settings
from os import path
import os
# Disconnect post_save signal
from django.db.models.signals import post_save
from tardis.tardis_portal.models import \
staging_hook, Dataset_File, Replica, Location
post_save.disconnect(staging_hook, sender=Replica)
from django.contrib.auth.models import User
user = '******'
pwd = 'secret'
email = ''
self.user = User.objects.create_user(user, email, pwd)
try:
os.makedirs(settings.GET_FULL_STAGING_PATH_TEST)
except OSError:
pass
self.temp = mkdtemp(dir=settings.GET_FULL_STAGING_PATH_TEST)
self.file = mktemp(dir=self.temp)
content = 'test file'
with open(self.file, "w+b") as f:
f.write(content)
Location.force_initialize()
# make datafile
exp = models.Experiment(title='test exp1',
institution_name='monash',
created_by=self.user)
exp.save()
# make dataset
dataset = models.Dataset(description="dataset description...")
dataset.save()
dataset.experiments.add(exp)
dataset.save()
# create datafile
df = models.Dataset_File(dataset=dataset,
size=len(content),
filename=path.basename(self.file),
md5sum='f20d9f2072bbeb6691c0f9c5099b01f3')
df.save()
# create replica
base_url = 'file://' + settings.GET_FULL_STAGING_PATH_TEST
location = Location.load_location({
'name': 'staging-test-yyy',
'url': base_url,
'type': 'external',
'priority': 10,
'transfer_provider': 'local'
})
replica = models.Replica(datafile=df,
url='file://' + self.file,
protocol="staging",
location=location)
replica.verify()
replica.save()
self.replica = replica
def _get_or_create_local_location(self, name, url, type, priority):
return Location.load_location({
'name': name, 'url': url, 'type': type, 'priority': priority,
'transfer_provider': 'local'})
