def setUp(self):
self.root_dir = os.path.abspath('kusehgcfscuzhfqizuchgsireugvcsi')
if os.path.exists(self.root_dir):
shutil.rmtree(self.root_dir)
assert not os.path.exists(self.root_dir)
self.ds = FileSystemDataStore(self.root_dir)
self.now = datetime.datetime.now()
os.mkdir(os.path.join(self.root_dir, 'test_dir'))
self.test_files = set(
['test_file1', 'test_file2', 'test_dir/test_file3'])
self.test_data = b'licgsnireugcsenrigucsic\ncrgqgjch,kgch'
for filename in self.test_files:
with open(os.path.join(self.root_dir, filename), 'wb') as f:
f.write(self.test_data)
def test__get_data_store__should_return_DataStore_object(self):
root_dir = 'kuqeyfgneuqygvn'
ds = FileSystemDataStore(root_dir)
self.assert_(
isinstance(
get_data_store('FileSystemDataStore', {'root': root_dir}),
DataStore))
if os.path.exists(root_dir):
os.rmdir(root_dir)
class TestFileSystemDataStore(unittest.TestCase):
def setUp(self):
self.root_dir = os.path.abspath('kusehgcfscuzhfqizuchgsireugvcsi')
if os.path.exists(self.root_dir):
shutil.rmtree(self.root_dir)
assert not os.path.exists(self.root_dir)
self.ds = FileSystemDataStore(self.root_dir)
self.now = datetime.datetime.now()
os.mkdir(os.path.join(self.root_dir, 'test_dir'))
self.test_files = set(
['test_file1', 'test_file2', 'test_dir/test_file3'])
self.test_data = b'licgsnireugcsenrigucsic\ncrgqgjch,kgch'
for filename in self.test_files:
with open(os.path.join(self.root_dir, filename), 'wb') as f:
f.write(self.test_data)
def tearDown(self):
shutil.rmtree(self.root_dir)
del self.ds
def test__init__should_create_root_if_it_doesnt_exist(self):
self.assert_(os.path.exists(self.root_dir))
def test__str__should_return_root(self):
self.assertEqual(str(self.ds), self.root_dir)
def test__get_state__should_return_dict_containing_root(self):
self.assertEqual(self.ds.__getstate__(), {'root': self.root_dir})
def test__find_new_data__should_return_list_of_keys_matching_new_files(
self):
self.assertEqual(
set(key.path for key in self.ds.find_new_data(self.now)),
self.test_files)
def test__find_new_data_with_future_timestamp__should_return_empty_list(
self):
tomorrow = self.now + datetime.timedelta(1)
self.assertEqual(set(self.ds.find_new_data(tomorrow)), set([]))
def test__get_content__should_return_short_file_content(self):
digest = hashlib.sha1(self.test_data).hexdigest()
key = DataKey('test_file1', digest)
content = self.ds.get_content(key)
self.assertEqual(content, self.test_data)
def test__get_content__should_truncate_long_files(self):
digest = hashlib.sha1(self.test_data).hexdigest()
key = DataKey('test_file1', digest)
content = self.ds.get_content(key, max_length=10)
self.assertEqual(content, self.test_data[:10])
def test__delete__should_remove_files(self):
assert os.path.exists(os.path.join(self.root_dir, 'test_file1'))
digest = hashlib.sha1(self.test_data).hexdigest()
keys = [DataKey(path, digest) for path in self.test_files]
self.ds.delete(*keys)
self.assert_(
not os.path.exists(os.path.join(self.root_dir, 'test_file1')))
class TestFileSystemDataStore(unittest.TestCase):
def setUp(self):
self.root_dir = os.path.abspath('kusehgcfscuzhfqizuchgsireugvcsi')
if os.path.exists(self.root_dir):
shutil.rmtree(self.root_dir)
assert not os.path.exists(self.root_dir)
self.ds = FileSystemDataStore(self.root_dir)
self.now = datetime.datetime.now()
os.mkdir(os.path.join(self.root_dir, 'test_dir'))
self.test_files = set(['test_file1', 'test_file2', 'test_dir/test_file3'])
self.test_data = b'licgsnireugcsenrigucsic\ncrgqgjch,kgch'
for filename in self.test_files:
with open(os.path.join(self.root_dir, filename), 'wb') as f:
f.write(self.test_data)
def tearDown(self):
shutil.rmtree(self.root_dir)
del self.ds
def test__init__should_create_root_if_it_doesnt_exist(self):
self.assert_(os.path.exists(self.root_dir))
def test__str__should_return_root(self):
self.assertEqual(str(self.ds), self.root_dir)
def test__get_state__should_return_dict_containing_root(self):
self.assertEqual(self.ds.__getstate__(), {'root': self.root_dir})
def test__find_new_data__should_return_list_of_keys_matching_new_files(self):
self.assertEqual(set(key.path for key in self.ds.find_new_data(self.now)),
self.test_files)
def test__find_new_data_with_future_timestamp__should_return_empty_list(self):
tomorrow = self.now + datetime.timedelta(1)
self.assertEqual(set(self.ds.find_new_data(tomorrow)),
set([]))
def test__get_content__should_return_short_file_content(self):
digest = hashlib.sha1(self.test_data).hexdigest()
key = DataKey('test_file1', digest)
content = self.ds.get_content(key)
self.assertEqual(content, self.test_data)
def test__get_content__should_truncate_long_files(self):
digest = hashlib.sha1(self.test_data).hexdigest()
key = DataKey('test_file1', digest)
content = self.ds.get_content(key, max_length=10)
self.assertEqual(content, self.test_data[:10])
def test__delete__should_remove_files(self):
assert os.path.exists(os.path.join(self.root_dir, 'test_file1'))
digest = hashlib.sha1(self.test_data).hexdigest()
keys = [DataKey(path, digest) for path in self.test_files]
self.ds.delete(*keys)
self.assert_(not os.path.exists(os.path.join(self.root_dir, 'test_file1')))
def setUp(self):
self.root_dir = os.path.abspath('kusehgcfscuzhfqizuchgsireugvcsi')
if os.path.exists(self.root_dir):
shutil.rmtree(self.root_dir)
assert not os.path.exists(self.root_dir)
self.ds = FileSystemDataStore(self.root_dir)
self.now = datetime.datetime.now()
os.mkdir(os.path.join(self.root_dir, 'test_dir'))
self.test_files = set(['test_file1', 'test_file2', 'test_dir/test_file3'])
self.test_data = b'licgsnireugcsenrigucsic\ncrgqgjch,kgch'
for filename in self.test_files:
with open(os.path.join(self.root_dir, filename), 'wb') as f:
f.write(self.test_data)
from __future__ import unicode_literals
from builtins import range
from sumatra.projects import Project
from sumatra.records import Record
from sumatra.recordstore import django_store
from sumatra.programs import PythonExecutable
from sumatra.launch import SerialLaunchMode
from sumatra.datastore import FileSystemDataStore
from sumatra.parameters import SimpleParameterSet
from sumatra.versioncontrol._git import GitRepository
import random
serial = SerialLaunchMode()
executable = PythonExecutable("/usr/bin/python", version="2.7")
repos = GitRepository('.')
datastore = FileSystemDataStore("/path/to/datastore")
project = Project("test_project",
default_executable=executable,
default_repository=repos,
default_launch_mode=serial,
data_store=datastore,
record_store=django_store.DjangoRecordStore())
parameters = SimpleParameterSet({'a': 2, 'b': 3})
for i in range(50):
record = Record(executable=executable,
repository=repos,
main_file="main.py",
version="99863a9dc5f",
launch_mode=serial,
datastore=datastore,
class TestFileSystemDataStore(unittest.TestCase):
def setUp(self):
self.root_dir = os.path.abspath('kusehgcfscuzhfqizuchgsireugvcsi')
if os.path.exists(self.root_dir):
shutil.rmtree(self.root_dir)
assert not os.path.exists(self.root_dir)
self.ds = FileSystemDataStore(self.root_dir)
self.now = datetime.datetime.now()
os.mkdir(os.path.join(self.root_dir, 'test_dir'))
self.test_files = set(
['test_file1', 'test_file2', 'test_dir/test_file3'])
self.test_data = b'licgsnireugcsenrigucsic\ncrgqgjch,kgch'
for filename in self.test_files:
with open(os.path.join(self.root_dir, filename), 'wb') as f:
f.write(self.test_data)
def tearDown(self):
shutil.rmtree(self.root_dir)
del self.ds
def test__init__should_create_root_if_it_doesnt_exist(self):
self.assert_(os.path.exists(self.root_dir))
def test__str__should_return_root(self):
self.assertEqual(str(self.ds), self.root_dir)
def test__setting_root_with_str_should_work(self):
# This is a regression test for Python2 where str(...) returns an instance
# of type 'future.types.newstr.newstr' which makes Pathlib choke if passed
# directly (because it tries to intern the string). Here we check that the
# class FileSystemDataStore knows how to deal with this.
self.ds.root = str('/tmp/foo/bar')
def test__get_state__should_return_dict_containing_root(self):
self.assertEqual(self.ds.__getstate__(), {'root': self.root_dir})
def test__find_new_data__should_return_list_of_keys_matching_new_files(
self):
self.assertEqual(
set(key.path for key in self.ds.find_new_data(self.now)),
self.test_files)
def test__find_new_data_with_future_timestamp__should_return_empty_list(
self):
tomorrow = self.now + datetime.timedelta(1)
self.assertEqual(set(self.ds.find_new_data(tomorrow)), set([]))
def test__get_content__should_return_short_file_content(self):
digest = hashlib.sha1(self.test_data).hexdigest()
key = DataKey('test_file1', digest, creation=None)
content = self.ds.get_content(key)
self.assertEqual(content, self.test_data)
def test__get_content__should_truncate_long_files(self):
digest = hashlib.sha1(self.test_data).hexdigest()
key = DataKey('test_file1', digest, creation=self.now)
content = self.ds.get_content(key, max_length=10)
self.assertEqual(content, self.test_data[:10])
def test__delete__should_remove_files(self):
assert os.path.exists(os.path.join(self.root_dir, 'test_file1'))
digest = hashlib.sha1(self.test_data).hexdigest()
keys = [
DataKey(path, digest, creation=None) for path in self.test_files
]
self.ds.delete(*keys)
self.assert_(
not os.path.exists(os.path.join(self.root_dir, 'test_file1')))
class TestFileSystemDataStore(unittest.TestCase):
def setUp(self):
self.root_dir = os.path.abspath('kusehgcfscuzhfqizuchgsireugvcsi')
if os.path.exists(self.root_dir):
shutil.rmtree(self.root_dir)
assert not os.path.exists(self.root_dir)
self.ds = FileSystemDataStore(self.root_dir)
self.now = datetime.datetime.now()
os.mkdir(os.path.join(self.root_dir, 'test_dir'))
self.test_files = set(['test_file1', 'test_file2', 'test_dir/test_file3'])
self.test_data = b'licgsnireugcsenrigucsic\ncrgqgjch,kgch'
for filename in self.test_files:
with open(os.path.join(self.root_dir, filename), 'wb') as f:
f.write(self.test_data)
def tearDown(self):
shutil.rmtree(self.root_dir)
del self.ds
def test__init__should_create_root_if_it_doesnt_exist(self):
self.assert_(os.path.exists(self.root_dir))
def test__str__should_return_root(self):
self.assertEqual(str(self.ds), self.root_dir)
def test__setting_root_with_str_should_work(self):
# This is a regression test for Python2 where str(...) returns an instance
# of type 'future.types.newstr.newstr' which makes Pathlib choke if passed
# directly (because it tries to intern the string). Here we check that the
# class FileSystemDataStore knows how to deal with this.
self.ds.root = str('/tmp/foo/bar')
def test__get_state__should_return_dict_containing_root(self):
self.assertEqual(self.ds.__getstate__(), {'root': self.root_dir})
def test__find_new_data__should_return_list_of_keys_matching_new_files(self):
self.assertEqual(set(key.path for key in self.ds.find_new_data(self.now)),
self.test_files)
def test__find_new_data_with_future_timestamp__should_return_empty_list(self):
tomorrow = self.now + datetime.timedelta(1)
self.assertEqual(set(self.ds.find_new_data(tomorrow)),
set([]))
def test__get_content__should_return_short_file_content(self):
digest = hashlib.sha1(self.test_data).hexdigest()
key = DataKey('test_file1', digest, creation=None)
content = self.ds.get_content(key)
self.assertEqual(content, self.test_data)
def test__get_content__should_truncate_long_files(self):
digest = hashlib.sha1(self.test_data).hexdigest()
key = DataKey('test_file1', digest, creation=self.now)
content = self.ds.get_content(key, max_length=10)
self.assertEqual(content, self.test_data[:10])
def test__delete__should_remove_files(self):
assert os.path.exists(os.path.join(self.root_dir, 'test_file1'))
digest = hashlib.sha1(self.test_data).hexdigest()
keys = [DataKey(path, digest, creation=None) for path in self.test_files]
self.ds.delete(*keys)
self.assert_(not os.path.exists(os.path.join(self.root_dir, 'test_file1')))