class TestPrepareTarget(TestCase):
def setUp(self):
self.dir = TempDirectory()
self.addCleanup(self.dir.cleanup)
replace = Replacer()
replace('workfront.generate.TARGET_ROOT', self.dir.path)
self.addCleanup(replace.restore)
self.session = Session('test')
def test_from_scratch(self):
path = prepare_target(self.session)
compare(path, expected=self.dir.getpath('unsupported.py'))
self.dir.compare(expected=[])
def test_everything(self):
self.dir.write('unsupported.py', b'yy')
path = prepare_target(self.session)
compare(path, expected=self.dir.getpath('unsupported.py'))
self.dir.compare(expected=['unsupported.py'])
compare(self.dir.read('unsupported.py'), b"yy")
def test_dots_in_version(self):
path = prepare_target(Session('test', api_version='v4.0'))
compare(path, expected=self.dir.getpath('v40.py'))
self.dir.compare(expected=[])
class MultiFileMultiRegexTestCase(unittest.TestCase):
def setUp(self):
self.d1 = TempDirectory()
self.d1_path = self.d1.getpath('')
self.d1.write('foo.txt', 'some nonsense 1')
self.d1.write('bar.txt', 'some nonsense 1')
self.d1.write('baz.txt', 'some nonsense 2')
self.all_files = [self.d1_path + i for i in os.listdir(self.d1_path)]
def test_2_files_match(self):
mfmr = MultiFileMultiRegex(self.all_files, ['some nonsense 1'])
match_groups = mfmr.find_all()
self.assertItemsEqual(
match_groups.keys(),
[self.d1_path + i for i in ['foo.txt', 'bar.txt']])
self.assertItemsEqual(match_groups.values(), [{
'some nonsense 1': ['some nonsense 1']
}, {
'some nonsense 1': ['some nonsense 1']
}])
def test_all_files_match(self):
mfmr = MultiFileMultiRegex(self.all_files, ['some nonsense \d'])
match_groups = mfmr.find_all()
self.assertItemsEqual(match_groups.keys(), self.all_files)
self.assertItemsEqual(match_groups.values(), [{
'some nonsense \d': ['some nonsense 1']
}, {
'some nonsense \d': ['some nonsense 1']
}, {
'some nonsense \d': ['some nonsense 2']
}])
def test_evaluate_read_same(self):
dir = TempDirectory()
dir.write('foo', 'content')
d = TestContainer('parsed', FileBlock('foo', 'content', 'read'))
d.evaluate_with(Files('td'), globs={'td': dir})
compare([C(FileResult, passed=True, expected=None, actual=None)],
[r.evaluated for r in d])
class TestPrepareTarget(TestCase):
def setUp(self):
self.dir = TempDirectory()
self.addCleanup(self.dir.cleanup)
replace = Replacer()
replace('workfront.generate.TARGET_ROOT', self.dir.path)
self.addCleanup(replace.restore)
self.session = Session('test')
def test_from_scratch(self):
path = prepare_target(self.session)
compare(path, expected=self.dir.getpath('unsupported.py'))
self.dir.compare(expected=[])
def test_everything(self):
self.dir.write('unsupported.py', b'yy')
path = prepare_target(self.session)
compare(path, expected=self.dir.getpath('unsupported.py'))
self.dir.compare(expected=['unsupported.py'])
compare(self.dir.read('unsupported.py'), b"yy")
def test_dots_in_version(self):
path = prepare_target(Session('test', api_version='v4.0'))
compare(path, expected=self.dir.getpath('v40.py'))
self.dir.compare(expected=[])
class TestGetLanes(unittest.TestCase):
def setUp(self):
self.tempdir = TempDirectory()
self.tempdir.write('fake_file1.txt', b'some foo thing')
self.tempdir.write('fake_tmp_files/folder/afile.txt', b'the text')
self.tempdir.write('fake_directory/fake_file2.txt', b'the text')
self.tempdir.write('fake_directory/afile.bam', b'the text')
self.tempdir.write('fake_directory/afile.sam', b'the text')
self.tempdir.write('fake_directory/afile.fastq.gz', b'the text')
self.tempdir.makedir('empty_directory')
self.tempdir_path = self.tempdir.path
def tearDown(self):
self.tempdir.cleanup()
pass
def test_get_lanes_returns_data(self):
data_list = bytes(str(self.tempdir.path + '/fake_tmp_files/folder/afile.txt\n'+self.tempdir.path + '/fake_directory/fake_file2.txt\n' + 'fake_path\n'),'ascii')
with mock.patch('archive_project.GetLanes.check_output', return_value=data_list) as co:
actual, message = get_lanes("study")
co.assert_called_once_with(['pf', 'data', '-t', 'study', '-i', 'study'])
expected = [str(self.tempdir.path + '/fake_tmp_files/folder/afile.txt'), str(self.tempdir.path + '/fake_directory/fake_file2.txt')]
self.assertEqual(expected,actual)
self.assertEqual('This path was returned by pf, but does not actually exist: fake_path\n', message)
def test_get_lanes_returns_nodata(self):
with mock.patch('archive_project.GetLanes.check_output', return_value=b'') as co:
with mock.patch('os.path.exists', return_value=True) as pe:
actual, message = get_lanes("fake_study")
self.assertEqual([],actual)
pe.assert_not_called()
self.assertEqual('Unknown study or no data associated with study: fake_study\n',message)
class TestConceptTrainer(TestCase):
def setUp(self):
ce = dsconcept.model.ConceptExtractor()
fe = dsconcept.model.FeatureExtractor()
self.d = TempDirectory()
data = b'{"abstract":["Astronauts are very cool."], "concept": ["ASTRONAUTS", "COOL THINGS"]}\n {"abstract":["NASA is going to Mars."], "concept":["NASA", "MARS"]}'
self.d.write("test.json", data)
self.corpus_path = f"{self.d.path}/test.json"
s = 100
self.X = csc_matrix(
np.random.randint(2, size=s * 2).reshape(int(s), 2))
self.y = np.random.randint(2, size=s)
paramgrid = {
"alpha": [0.01, 0.001, 0.0001],
"class_weight": [{
1: 10,
0: 1
}, {
1: 5,
0: 1
}, {
1: 20,
0: 1
}],
"max_iter": [1],
"loss": ["log"],
} # requires loss function with predict_proba
clf = GridSearchCV(SGDClassifier(), paramgrid,
scoring="f1") # requires GridSearchCV
self.ct = dsconcept.train.ConceptTrainer(ce, clf)
def test_create_concept_classifier(self):
out_dir = Path(f"{self.d.path}/models")
out_dir.mkdir()
X_train, X_test, y_train, y_test = train_test_split(self.X,
self.y,
test_size=0.5,
random_state=42)
self.ct.create_concept_classifier("test_concept", X_train, X_test,
y_train, y_test, out_dir)
clf = joblib.load(out_dir / "test_concept.pkl")
LOG.info(clf)
def test_train_all(
self): # This test is super naive. Does not check behaviour.
self.ct.train_all(self.X, Path(f"{self.d.path}/models"), 5)
test_inds = np.load((Path(f"{self.d.path}") / "test_inds.npy"))
train_inds = np.load((Path(f"{self.d.path}") / "train_inds.npy"))
LOG.info(f"test_inds: {test_inds}")
LOG.info(f"train_inds: {train_inds}")
@given(arrays(dtype=np.float_, shape=1))
def test_get_dispersed_subset(self, array):
subset = dsconcept.train.get_dispersed_subset(array, 5)
self.assertLessEqual(len(subset), 5)
LOG.info(subset)
def tearDown(self):
self.d.cleanup()
class TestS3Storage(unittest.TestCase):
"""S3Storage.
Testing storage on S3 solution.
"""
def setUp(self):
"""Initialise fixtures."""
self._dir = TempDirectory()
self.env = EnvironmentVarGuard()
self.env.set('ARCHIVE_S3_URL', 'http://*****:*****@mock.patch.object(boto3, 'client')
def test_init_s3storage(self, mock_boto):
"""Initialise S3 storage."""
S3Storage('archive', 'lega')
mock_boto.assert_called()
@mock.patch.object(boto3, 'client')
def test_init_location(self, mock_boto):
"""Initialise S3 storage."""
storage = S3Storage('archive', 'lega')
result = storage.location('file_id')
self.assertEqual('file_id', result)
mock_boto.assert_called()
@mock.patch.object(boto3, 'client')
def test_upload(self, mock_boto):
"""Test copy to S3, should call boto3 client."""
path = self._dir.write('test.file', 'data1'.encode('utf-8'))
storage = S3Storage('archive', 'lega')
storage.copy(path, 'lega')
mock_boto.assert_called_with('s3',
aws_access_key_id='test',
aws_secret_access_key='test',
endpoint_url='http://*****:*****@mock.patch.object(boto3, 'client')
def test_open(self, mock_boto):
"""Test open , should call S3FileReader."""
path = self._dir.write('test.file', 'data1'.encode('utf-8'))
storage = S3Storage('archive', 'lega')
with storage.open(path) as resource:
self.assertEqual(S3FileReader, type(resource))
def temp_directory_with_files(
cls,
files: Union[List[str], Tuple[str,
...]] = ALL_FILES) -> TempDirectory:
"""Create a temp directory with files."""
temp_dict = TempDirectory()
for file_ in files:
temp_dict.write(file_, b"")
return temp_dict
class TestFeatureExtractor(TestCase):
def setUp(self):
self.fe = FeatureExtractor()
self.d = TempDirectory()
data = b'{"abstract":"Astronauts are very cool.", "concept": ["ASTRONAUTS", "COOL THINGS"]} \n {"abstract":"NASA is going to Mars.", "concept":["NASA", "MARS"]}'
self.d.write("test.json", data)
self.corpus_path = f"{self.d.path}/test.json"
@given(features())
def test_features(self, d):
self.fe.features = d
self.assertEqual(len(self.fe.features), len(d))
def test_from_corpus_to_jsonlines(self):
self.fe.from_corpus_to_jsonlines(
self.corpus_path,
f"{self.d.path}/features.jsonl",
"abstract",
)
def test_from_jsonlines(self):
data = b'{"astronaut":"NOUN", "space": "NOUN", "NASA": "ENT"}\n{"Mars": "PROPN", "dog": "NOUN"}'
features_out = "features.jsonl"
self.d.write(features_out, data)
self.fe.from_jsonlines(f"{self.d.path}/{features_out}")
self.assertSetEqual(self.fe.term_types, {"NOUN", "PROPN", "ENT"})
def test_to_jsonlines(self):
self.fe.features = [
{
"space": "NOUN",
"Mars": "PROPN"
},
{
"Anita": "PROPN",
"Adams": "PROPN"
},
]
out_features = "features.jsonl"
self.fe.to_jsonlines(f"{self.d.path}/{out_features}")
@given(features(), weights())
def test_weight_terms(self, d, w):
self.fe.features = d
self.fe.weight_terms(w)
@given(features(), weights())
def test_limit_features(self, d, w):
self.fe.features = d
weighted_features = self.fe.weight_terms(
w) # Test method contingent upon another test. Bad?
self.fe.limit_features(weighted_features,
feature_min=1,
feature_max=0.90)
def tearDown(self):
self.d.cleanup()
def test_evaluate_read_same(self):
dir = TempDirectory()
dir.write('foo', b'content')
d = TestContainer('parsed',FileBlock('foo','content','read'))
d.evaluate_with(Files('td'),globs={'td':dir})
compare([C(FileResult,
passed=True,
expected=None,
actual=None)],
[r.evaluated for r in d])
def test_evaluate_read_difference(self):
dir = TempDirectory()
dir.write('foo', b'actual')
d = TestContainer('parsed',FileBlock('foo','expected','read'))
d.evaluate_with(Files('td'),globs={'td':dir})
compare([C(FileResult,
passed=False,
path='foo',
expected='expected',
actual='actual')],
[r.evaluated for r in d])
class TestImageValidator(unittest.TestCase):
def setUp(self):
"""
Set up a folder structure containing one timepoint (0)
one channel (1) and two images in channel subfolder
"""
self.tempdir = TempDirectory()
self.temp_path = self.tempdir.path
self.tempdir.makedir('timepoint_0')
self.tempdir.makedir('timepoint_0/channel_1')
# Write images as bytes
im = np.zeros((15, 12), dtype=np.uint16)
res, im_encoded = cv2.imencode('.png', im)
im_encoded = im_encoded.tostring()
self.tempdir.write('timepoint_0/channel_1/im_0.png', im_encoded)
self.tempdir.write('timepoint_0/channel_1/im_1.png', im_encoded)
self.tempdir.listdir(recursive=True)
def tearDown(self):
"""
Tear down temporary folder and file structure
"""
TempDirectory.cleanup_all()
nose.tools.assert_equal(os.path.isdir(self.temp_path), False)
def test_image_validator(self):
"""
Test image validator on temporary folder structure
"""
meta_name = 'image_volumes_info.csv'
validator = image_validator.ImageValidator(
input_dir=self.tempdir.path,
meta_name=meta_name,
verbose=10,
)
validator.folder_validator()
# Check written metadata
metadata = pd.read_csv(os.path.join(self.tempdir.path, meta_name))
# Metadata should have 8 fields + 1 index and we have two files
nose.tools.assert_equal(metadata.shape, (2, 9))
# Metadata should contain the following column names
expected_names = [
'Unnamed: 0', 'timepoint', 'channel_num', 'sample_num',
'slice_num', 'fname', 'size_x_microns', 'size_y_microns',
'size_z_microns'
]
nose.tools.assert_equal(list(metadata), expected_names)
# Both files should have timepoint 0
nose.tools.assert_equal(np.all(metadata['timepoint'] == 0), True)
# Both file should have channel 1
nose.tools.assert_equal(np.all(metadata['channel_num'] == 1), True)
# The two image numbers should be sorted by index
nose.tools.assert_equal(metadata['sample_num'][0], 0)
nose.tools.assert_equal(metadata['sample_num'][1], 1)
class TestFixtureLoad:
def setUp(self):
self.d = TempDirectory()
fixtureload.set_source_dir(self.d.path)
self.d.write(
'test.json',
json.dumps({
'description': {
'samples': {
'fixture0': {
'a': 'b'
},
'fixture1': {
'c': 'd'
}
}
}
}))
self.d.write('test_corrupted.json', 'corrupted json')
def tearDown(self):
self.d.cleanup()
def test_can_set_fixture_source_directory(self):
fixtureload.set_source_dir('/tmp')
assert fixtureload.fixture_source_dir == '/tmp'
def test_can_load_fixture(self):
fixture = fixtureload.load('test/description/fixture0')
compare(fixture, {'a': 'b'})
fixture = fixtureload.load('test/description/fixture1')
compare(fixture, {'c': 'd'})
def test_load_fixture_with_not_existed_file_should_raise(self):
with ShouldRaise(IOError):
fixtureload.load('not_existed/description/fixture0')
def test_load_fixture_with_corrupted_file_should_raise(self):
with ShouldRaise(ValueError):
fixtureload.load('test_corrupted/description/fixture0')
def test_parse_fixture_path(self):
path = fixtureload.parse_fixture_path('test/desc/fixture0')
compare(
path, {
'source_file': 'test.json',
'fixture_desc': 'desc',
'fixture': 'fixture0'
})
def test_parse_fixture_path_if_path_is_not_valid(self):
with ShouldRaise(
ValueError('Fixture Path is not valid (testfixture0)')):
fixtureload.parse_fixture_path('testfixture0')
def test_inifile_discovery_should_ignore_invalid_files_without_raising_exception(self):
root_dir = TempDirectory()
self.tmpdirs.append(root_dir)
cfg_file = root_dir.write(('some', 'strange', 'config.cfg'), '&ˆ%$#$%ˆ&*()(*&ˆ'.encode('utf8'))
root_dir.write(('some', 'config.ini'), '$#%ˆ&*((*&ˆ%'.encode('utf8'))
discovery = ConfigurationDiscovery(
os.path.realpath(os.path.dirname(cfg_file)), filetypes=(IniFileConfigurationLoader, ))
self.assertEqual(discovery.config_files, [])
def test_evaluate_read_difference(self):
dir = TempDirectory()
dir.write('foo', b'actual')
d = TestContainer('parsed',FileBlock('foo','expected','read'))
d.evaluate_with(Files('td'),globs={'td':dir})
compare([C(FileResult,
passed=False,
path='foo',
expected='expected',
actual='actual')],
[r.evaluated for r in d])
def get_td_zipped_twice(self):
'''Zip the same filenames twice
with different content
'''
td = TempDirectory()
td.write('file.1', b'1')
td.write('file.2', b'2')
gzip_logs(td.path)
td.write('file.1', b'3')
td.write('file.2', b'4')
gzip_logs(td.path)
return td
class TestFixtureLoad:
def setUp(self):
self.d = TempDirectory()
fixtureload.set_source_dir(self.d.path)
self.d.write('test.json',
json.dumps(
{
'description':
{
'samples': {
'fixture0': {'a': 'b'},
'fixture1': {'c': 'd'}
}
}
}
))
self.d.write('test_corrupted.json', 'corrupted json')
def tearDown(self):
self.d.cleanup()
def test_can_set_fixture_source_directory(self):
fixtureload.set_source_dir('/tmp')
assert fixtureload.fixture_source_dir == '/tmp'
def test_can_load_fixture(self):
fixture = fixtureload.load('test/description/fixture0')
compare(fixture, {'a': 'b'})
fixture = fixtureload.load('test/description/fixture1')
compare(fixture, {'c': 'd'})
def test_load_fixture_with_not_existed_file_should_raise(self):
with ShouldRaise(IOError):
fixtureload.load('not_existed/description/fixture0')
def test_load_fixture_with_corrupted_file_should_raise(self):
with ShouldRaise(ValueError):
fixtureload.load('test_corrupted/description/fixture0')
def test_parse_fixture_path(self):
path = fixtureload.parse_fixture_path('test/desc/fixture0')
compare(
path,
{
'source_file': 'test.json',
'fixture_desc': 'desc',
'fixture': 'fixture0'
}
)
def test_parse_fixture_path_if_path_is_not_valid(self):
with ShouldRaise(ValueError('Fixture Path is not valid (testfixture0)')):
fixtureload.parse_fixture_path('testfixture0')
def test_inifile_discovery_should_ignore_invalid_files_without_raising_exception(
self):
root_dir = TempDirectory()
self.tmpdirs.append(root_dir)
cfg_file = root_dir.write(('some', 'strange', 'config.cfg'),
'&ˆ%$#$%ˆ&*()(*&ˆ'.encode('utf8'))
root_dir.write(('some', 'config.ini'), '$#%ˆ&*((*&ˆ%'.encode('utf8'))
discovery = ConfigurationDiscovery(
os.path.realpath(os.path.dirname(cfg_file)),
filetypes=(IniFileConfigurationLoader, ))
self.assertEqual(discovery.config_files, [])
class HomeDirTest(unittest.TestCase):
def setUp(self):
self.temp_dir = TempDirectory(create=True)
self.home = PathHomeDir(self.temp_dir.path)
def tearDown(self):
self.temp_dir.cleanup()
def test_read(self):
self.temp_dir.write("filename", "contents")
self.assertEquals(self.home.read("filename"), "contents")
def test_write(self):
self.temp_dir.write("existing_file", "existing_contents")
self.home.write("new_file", "contents")
self.home.write("existing_file", "new_contents")
self.assertEquals(self.temp_dir.read("existing_file"),
"new_contents")
self.assertEquals(self.temp_dir.read("new_file"), "contents")
def test_config_file(self):
with collect_outputs() as outputs:
self.home.write_config_file("new config")
self.temp_dir.check(".cosmosrc")
self.assertEquals(self.home.read_config_file(), "new config")
self.assertIn("Settings saved", outputs.stdout.getvalue())
file_mode = os.stat(self.temp_dir.getpath(".cosmosrc")).st_mode
self.assertEquals(file_mode, stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR)
def test_override_config_file(self):
with collect_outputs():
other_config = self.temp_dir.write("path/other", "config")
self.assertEquals(
self.home.read_config_file(filename_override=other_config),
"config")
def test_warn_on_unprotected_config_file(self):
with collect_outputs() as outputs:
self.home.write_config_file("new config")
config_path = self.temp_dir.getpath(".cosmosrc")
os.chmod(config_path, 0777)
self.home.read_config_file()
assertFunc = (self.assertNotIn if os.name=='nt' else self.assertIn)
assertFunc("WARNING", outputs.stderr.getvalue())
def test_last_cluster(self):
self.home.write_last_cluster("0000000")
self.temp_dir.check(".cosmoslast")
self.assertEquals(self.home.read_last_cluster(), "0000000")
class HomeDirTest(unittest.TestCase):
def setUp(self):
self.temp_dir = TempDirectory(create=True)
self.home = PathHomeDir(self.temp_dir.path)
def tearDown(self):
self.temp_dir.cleanup()
def test_read(self):
self.temp_dir.write("filename", "contents")
self.assertEquals(self.home.read("filename"), "contents")
def test_write(self):
self.temp_dir.write("existing_file", "existing_contents")
self.home.write("new_file", "contents")
self.home.write("existing_file", "new_contents")
self.assertEquals(self.temp_dir.read("existing_file"), "new_contents")
self.assertEquals(self.temp_dir.read("new_file"), "contents")
def test_config_file(self):
with collect_outputs() as outputs:
self.home.write_config_file("new config")
self.temp_dir.check(".cosmosrc")
self.assertEquals(self.home.read_config_file(), "new config")
self.assertIn("Settings saved", outputs.stdout.getvalue())
file_mode = os.stat(self.temp_dir.getpath(".cosmosrc")).st_mode
self.assertEquals(file_mode,
stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR)
def test_override_config_file(self):
with collect_outputs():
other_config = self.temp_dir.write("path/other", "config")
self.assertEquals(
self.home.read_config_file(filename_override=other_config),
"config")
def test_warn_on_unprotected_config_file(self):
with collect_outputs() as outputs:
self.home.write_config_file("new config")
config_path = self.temp_dir.getpath(".cosmosrc")
os.chmod(config_path, 0777)
self.home.read_config_file()
assertFunc = (self.assertNotIn
if os.name == 'nt' else self.assertIn)
assertFunc("WARNING", outputs.stderr.getvalue())
def test_last_cluster(self):
self.home.write_last_cluster("0000000")
self.temp_dir.check(".cosmoslast")
self.assertEquals(self.home.read_last_cluster(), "0000000")
def test_replace_file_contents(self):
"""ファイルの文字列を書き換える関数のテスト。"""
# ファイルやディレクトリのテストでは、まず以下インスタンスを生成する。
temp_dir = TempDirectory()
# writeメソッドで、ファイル作成できる(以下のように、存在しないディレクトリも再帰的に作ってくれる)。
# 一点注意すべきなのが、デフォルトだとbyte文字列しか書き込みできないこと。
# 通常の文字列を書き込みたい場合、エンコーディングの指定が必須。
# http://testfixtures.readthedocs.io/en/latest/api.html
temp_file = temp_dir.write('foo/bar/sample.txt',
'I love cat.\nMy cat\'s name is mike.\n',
encoding='utf-8')
# テストケース実施
replace_file_contents(temp_dir.path + '/foo/bar', 'sample.txt', 'cat',
'dog')
# ファイルの結果を確認する。
self.assertEqual(
'I love dog.\nMy dog\'s name is mike.\n',
# readメソッド呼び出し時にも、エンコーディングの指定が要るので注意(省略した場合はバイト文字列)。
temp_dir.read('foo/bar/sample.txt', encoding='utf-8'))
# 以下を呼ぶことで、ディレクトリ以下の内容を再帰的にcleanしてくれる。
temp_dir.cleanup()
class TestPathSource(TestCase):
def setUp(self):
self.dir = TempDirectory()
self.addCleanup(self.dir.cleanup)
def test_abc(self):
self.assertTrue(issubclass(Plugin, Source))
def test_schema_ok(self):
p1 = self.dir.write('foo', b'f')
p2 = self.dir.write('bar', b'b')
compare(
dict(type='paths', values=[p1, p2], repo='config'),
Plugin.schema(
dict(type='paths', values=[p1, p2], repo='config')
))
def test_schema_wrong_type(self):
text = "not a valid value for dictionary value @ data['type']"
with ShouldFailSchemaWith(text):
Plugin.schema(dict(type='bar', values=['/']))
def test_schema_extra_keys(self):
with ShouldFailSchemaWith("extra keys not allowed @ data['foo']"):
Plugin.schema(dict(type='paths', foo='bar'))
def test_name_supplied(self):
text = "not a valid value for dictionary value @ data['name']"
with ShouldFailSchemaWith(text):
Plugin.schema(dict(type='paths', name='foo'))
def test_no_paths(self):
text = "length of value must be at least 1 for dictionary value " \
"@ data['values']"
with ShouldFailSchemaWith(text):
Plugin.schema(dict(type='paths', values=[]))
def test_path_not_string(self):
text = "invalid list value @ data['values'][0]"
with ShouldFailSchemaWith(text):
Plugin.schema(dict(type='paths', values=[1]))
def test_path_not_starting_with_slash(self):
text = "invalid list value @ data['values'][0]"
with ShouldFailSchemaWith(text):
Plugin.schema(dict(type='paths', values=['foo']))
def test_path_not_there(self):
text = "invalid list value @ data['values'][0]"
with ShouldFailSchemaWith(text):
Plugin.schema(dict(type='paths', values=[self.dir.getpath('bad')]))
def test_interface(self):
plugin = Plugin('source', name=None, repo='config',
values=['/foo/bar'])
compare(plugin.type, 'source')
compare(plugin.name, None)
compare(plugin.repo, 'config')
compare(plugin.source_paths, ['/foo/bar'])
class Test_SnapshotArchive_Repository(TestCase):
def setUp(self):
store = MemoryCommitStorage()
self.repo = BBRepository(store)
self.tempdir = TempDirectory()
self.setup_archive_a_snapshot()
def setup_archive_a_snapshot(self):
archive_name = 'somearchive.tgz'
self.archive_contents = '123'
self.archive_path = self.tempdir.write(archive_name,
self.archive_contents)
self.tag = generate_tag()
self.first_WAL = '01234'
self.last_WAL = '45678'
commit_snapshot_to_repository(self.repo, self.archive_path, self.tag,
self.first_WAL, self.last_WAL)
def tearDown(self):
self.tempdir.cleanup()
def test_can_retrieve_snapshot_contents_with_tag(self):
commit = [i for i in self.repo][-1]
restore_path = self.tempdir.getpath('restorearchive.tgz')
commit.get_contents_to_filename(restore_path)
self.assertEqual(self.archive_contents,
open(restore_path, 'rb').read())
def test_get_first_WAL_file_for_archived_snapshot_with_tag(self):
self.assertEqual(self.first_WAL, get_first_WAL(self.repo, self.tag))
def test_get_last_WAL_file_for_archived_snapshot_with_tag(self):
self.assertEqual(self.last_WAL, get_last_WAL(self.repo, self.tag))
def test_cleanup(self):
d = TempDirectory()
p = d.path
assert os.path.exists(p) is True
p = d.write('something', b'stuff')
d.cleanup()
assert os.path.exists(p) is False
class BaseTestCase(unittest.TestCase):
"""Checkout unittest."""
def setUp(self):
self.d1 = TempDirectory()
self.first_file_name = b'foo.py'
self.parse_file_1_name = b'bar.py'
self.d1.write(self.first_file_name, self.parse_file_1_name)
self.first_file_path = self.d1.getpath(self.first_file_name)
self.d1.write(self.parse_file_1_name, b'Some nonsense')
self.parse_file_1_path = self.d1.getpath(self.parse_file_1_name)
def tearDown(self):
TempDirectory.cleanup_all()
def test_testlist(self):
import stat
work_directory = TempDirectory()
work_directory.makedir("test")
work_directory.write(["test", "test_dummy.c"], b"")
build_directory = TempDirectory()
testbin_path = build_directory.write(["test_dummy"], b"")
st = os.stat(testbin_path)
os.chmod(testbin_path, st.st_mode | stat.S_IEXEC)
w = Watcher(work_directory.path, build_directory.path)
w.poll()
exefile = testbin_path + watcher.EXE_SUFFIX
testlist = [(g.source(), g.executable()) for g in w.testlist()]
assert testlist == [(os.path.join("test", "test_dummy.c"), exefile)]
class DirHandlerContext(BaseContext):
def setUp(self):
self.dir = TempDirectory()
self.cleanups.extend([self.dir.cleanup,self.removeAtExit])
# This one is so that we can check what log handlers
# get added.
self.handlers = []
self.r.replace('checker.logger.handlers',self.handlers)
def run_with_config(self,config):
self.dir.write('checker.txt',config)
main(('-C '+self.dir.path).split())
def removeAtExit(self):
# make sure we haven't registered any atexit funcs
atexit._exithandlers[:] = []
def test_testlist(self):
import stat
work_directory = TempDirectory()
work_directory.makedir('test')
work_directory.write(['test', 'test_dummy.c'], b'')
build_directory = TempDirectory()
testbin_path = build_directory.write(['test_dummy'], b'')
st = os.stat(testbin_path)
os.chmod(testbin_path, st.st_mode | stat.S_IEXEC)
w = Watcher(work_directory.path, build_directory.path)
w.poll()
exefile = testbin_path + watcher.EXE_SUFFIX
testlist = [(g.source(), g.executable()) for g in w.testlist()]
assert testlist == [(os.path.join('test', 'test_dummy.c'), exefile)]
class DirHandlerContext(BaseContext):
def setUp(self):
self.dir = TempDirectory()
self.cleanups.extend([self.dir.cleanup, self.removeAtExit])
# This one is so that we can check what log handlers
# get added.
self.handlers = []
self.r.replace('checker.check.logger.handlers', self.handlers)
def run_with_config(self, config):
self.dir.write('checker.txt', config)
main(('-C ' + self.dir.path).split())
def removeAtExit(self):
# make sure we haven't registered any atexit funcs
# (which will be done by MailingLogger!)
atexit._exithandlers[:] = []
def test_statistics():
dir = TempDirectory()
original_dir = os.getcwd()
with open('tests/notebooks/a.ipynb', 'rb') as input:
dir.write('a.ipynb', input.read())
dir.write('__main__.py', b'')
res = {
'cells': 25,
'code': 12,
'ename': u'KeyError',
'evalue': u"'engines'",
'executed': 12,
'markdown': 13
}
os.chdir(dir.path)
assert (notebook.statistics('a.ipynb') == res)
os.chdir(original_dir)
dir.cleanup()
def test_clean_build(self):
cmake_source_directory = TempDirectory()
source_path = cmake_source_directory.path
cmake_build_directory = TempDirectory()
build_path = cmake_build_directory.path
builder = create_builder(source_path, build_path)
raised = None
try:
builder()
except IOError as e:
raised = str(e)
assert raised == "[Errno 22] No CMakeLists.txt detected in {}".format(
source_path) # noqa
cmake_source_directory.write("CMakeLists.txt", b"project(test)")
builder()
assert exists(join(build_path, "CMakeFiles"))
def test_poll(self):
work_directory = TempDirectory()
work_directory.write("a.h", b"")
work_directory.write("a.c", b"")
work_directory.write("a.cc", b"")
work_directory.write("CMakeLists.txt", b"")
work_directory.write("blah.txt", b"")
w = Watcher(work_directory.path, None)
watchstate = w.poll()
assert watcher.has_changes(watchstate)
watchstate = w.poll()
assert not watcher.has_changes(watchstate)
work_directory.write("b.c", b"")
watchstate = w.poll()
assert watcher.has_changes(watchstate)
def setUp(self):
self.app = Flask(__name__)
self.app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///:memory:'
self.app.config['USER_LIST'] = ['Test Admin']
db.init_app(self.app)
d = TempDirectory()
d.makedir('dataset')
d.write('dataset/files/testfile.jpg', 'abc')
d.write('dataset/testfile2.jpg', 'abc')
self.directory = d
self.app.config['DATASET_ROOT'] = d.path
with self.app.app_context(), TempDirectory() as d:
db.create_all()
lib.models.buildDB()
dataset = lib.models.Datasets()
dataset.name = 'Test'
dataset.path = "dataset"
db.session.add(dataset)
db.session.commit()
class ActionHandlerTest(unittest.TestCase):
def setUp(self):
self.event = Mock()
self.event.body = {}
self.event.headers = {}
self.context = Mock()
self.temp_dir = TempDirectory()
os.environ['appname'] = 'appname'
os.environ['secrets'] = self.temp_dir.path
def tearDown(self):
self.temp_dir.cleanup()
def test_secret(self):
self.temp_dir.write('test-secret', b'top-secret-info')
secret = base.secret('test-secret')
self.assertEqual('top-secret-info', secret)
def test_poll(self):
work_directory = TempDirectory()
work_directory.write('a.h', b'')
work_directory.write('a.c', b'')
work_directory.write('a.cc', b'')
work_directory.write('CMakeLists.txt', b'')
work_directory.write('blah.txt', b'')
w = Watcher(work_directory.path, None)
watchstate = w.poll()
assert watcher.has_changes(watchstate)
watchstate = w.poll()
assert not watcher.has_changes(watchstate)
work_directory.write('b.c', b'')
watchstate = w.poll()
assert watcher.has_changes(watchstate)
def setUp(self):
self.app = Flask(__name__)
self.app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///:memory:'
self.app.config['USER_LIST'] = ['Test Admin']
db.init_app(self.app)
d = TempDirectory()
d.makedir('dataset')
d.write('dataset/files/testfile.jpg', 'abc')
d.write('dataset/testfile2.jpg', 'abc')
self.directory = d
self.app.config['DATASET_ROOT'] = d.path
with self.app.app_context(), TempDirectory() as d:
db.create_all()
lib.models.buildDB()
dataset = lib.models.Datasets()
dataset.name = 'Test'
dataset.path = "dataset"
db.session.add(dataset)
db.session.commit()
class GitHelper(object):
repo = 'local/'
def setUp(self):
self.dir = TempDirectory()
self.addCleanup(self.dir.cleanup)
def git(self, command, repo=None):
repo_path = self.dir.getpath(repo or self.repo)
try:
return check_output(['git'] + command.split(),
cwd=repo_path,
stderr=STDOUT)
except CalledProcessError as e:
self.fail(e.output)
def git_rev_parse(self, label, repo=None):
return self.git('rev-parse --verify -q --short ' + label, repo).strip()
def check_tags(self, expected, repo=None):
actual = {}
for tag in self.git('tag', repo).split():
actual[tag] = self.git_rev_parse(tag, repo)
compare(expected, actual=actual)
def make_repo_with_content(self, repo):
if not os.path.exists(self.dir.getpath(repo)):
self.dir.makedir(repo)
self.git('init', repo)
self.dir.write(repo + 'a', 'some content')
self.dir.write(repo + 'b', 'other content')
self.dir.write(repo + 'c', 'more content')
self.git('add .', repo)
self.git('commit -m initial', repo)
class TestFileStorage(unittest.TestCase):
"""FileStorage.
Testing storage on disk.
"""
def setUp(self):
"""Initialise fixtures."""
self._dir = TempDirectory()
self.outputdir = self._dir.makedir('output')
self.env = EnvironmentVarGuard()
self.env.set('ARCHIVE_LOCATION', self.outputdir + '/%s/')
self._store = FileStorage('archive', 'lega')
def tearDown(self):
"""Remove setup variables."""
self.env.unset('ARCHIVE_LOCATION')
self._dir.cleanup_all()
def test_location(self):
"""Test file location."""
result = self._store.location('12')
self.assertEqual(
os.path.join('/', '000', '000', '000', '000', '000', '000', '12'),
result)
def test_copy(self):
"""Test copy file."""
path = self._dir.write('output/lega/test.file',
'data1'.encode('utf-8'))
path1 = self._dir.write('output/lega/test1.file', ''.encode('utf-8'))
result = self._store.copy(open(path, 'rb'), path1)
self.assertEqual(os.stat(path).st_size, result)
def test_open(self):
"""Test open file."""
path = self._dir.write('output/lega/test.file',
'data1'.encode('utf-8'))
print(path)
with self._store.open('test.file') as resource:
self.assertEqual(BufferedReader, type(resource))
class GitHelper(object):
repo = 'local/'
def setUp(self):
self.dir = TempDirectory()
self.addCleanup(self.dir.cleanup)
def git(self, command, repo=None):
repo_path = self.dir.getpath(repo or self.repo)
try:
return check_output(['git'] + command.split(), cwd=repo_path, stderr=STDOUT)
except CalledProcessError as e:
self.fail(e.output)
def git_rev_parse(self, label, repo=None):
return self.git('rev-parse --verify -q --short '+label, repo).strip()
def check_tags(self, expected, repo=None):
actual = {}
for tag in self.git('tag', repo).split():
actual[tag] = self.git_rev_parse(tag, repo)
compare(expected, actual=actual)
def make_repo_with_content(self, repo):
if not os.path.exists(self.dir.getpath(repo)):
self.dir.makedir(repo)
self.git('init', repo)
self.dir.write(repo + 'a', 'some content')
self.dir.write(repo + 'b', 'other content')
self.dir.write(repo + 'c', 'more content')
self.git('add .', repo)
self.git('commit -m initial', repo)
class TestFileDownload(unittest.TestCase):
def setUp(self):
self.tempdir = TempDirectory()
self.tempdir.write('1/Accession1.fastq.gz', b'the text')
self.tempdir.write('2/Accession1_1.fastq.gz', b'the text')
self.tempdir.write('2/Accession1_2.fastq.gz',b'the text')
self.tempdir_path = self.tempdir.path
print('temp',self.tempdir_path)
self.under_test1 = Preparation.new_instance(Spreadsheet.new_instance("MyStudy", [
RawRead(forward_read='Accession1', reverse_read='T', sample_name='SAMPLE1',
taxon_id='1280', library_name='LIB1', sample_accession=None),
RawRead(forward_read='Accession2', reverse_read='T', sample_name='SAMPLE2',
taxon_id='1280', library_name='LIB2', sample_accession=None)]), self.tempdir_path, 0, 0)
self.under_test2 = Preparation.new_instance(Spreadsheet.new_instance("MyStudy", [
RawRead(forward_read='Accession1', reverse_read='T', sample_name='SAMPLE1',
taxon_id='1280', library_name='LIB1', sample_accession=None),
RawRead(forward_read='Accession2', reverse_read='T', sample_name='SAMPLE2',
taxon_id='1280', library_name='LIB2', sample_accession=None)]), self.tempdir_path, 1, 0)
self.under_test3 = Preparation.new_instance(Spreadsheet.new_instance("MyStudy", [
RawRead(forward_read='Accession1', reverse_read='T', sample_name='SAMPLE1',
taxon_id='1280', library_name='LIB1', sample_accession=None),
RawRead(forward_read='Accession2', reverse_read='T', sample_name='SAMPLE2',
taxon_id='1280', library_name='LIB2', sample_accession=None)]), self.tempdir_path, 2, 0)
def tearDown(self):
self.tempdir.cleanup()
pass
def test_ENA_download_calls_create_commands_correctly(self):
connections=1
with patch("importer.writer.Preparation.create_dataframe_from_list", return_value = 'df') as mock_create_dataframe_from_list:
with patch("importer.writer.Preparation.check_if_file_downloaded", return_value =False) as mock_check_if_file_downloaded:
with patch("importer.writer.create_commands", return_value='df') as mock_create_commands:
with patch("importer.writer.submit_commands") as mock_submit_commands:
self.under_test1.download_files_from_ena(connections)
mock_create_dataframe_from_list.assert_called_once_with(['Accession1','Accession2'])
mock_create_commands.assert_called_once_with('df', 1, self.tempdir_path+'/0')
mock_submit_commands.assert_called_once_with('df')
def test_create_dataframe_from_list(self):
reads = ['Accession1','Accession2']
actual=self.under_test1.create_dataframe_from_list(reads)
expected = pd.DataFrame(([read, 'import_%s' % read] for read in reads),
columns=('Read accession', 'Job_name'))
pd.testing.assert_frame_equal(actual,expected)
def test_check_if_file_downloaded_no_files(self):
actual=self.under_test1.check_if_file_downloaded('Accession1')
self.assertEqual(actual,False)
def test_check_if_file_downloaded_single_ended_exists(self):
actual=self.under_test2.check_if_file_downloaded('Accession1')
self.assertEqual(actual,True)
def test_check_if_file_downloaded_double_ended_exists(self):
actual=self.under_test3.check_if_file_downloaded('Accession1')
self.assertEqual(actual,True)
def test_default_watcher_gets_all_files(self):
work_directory = TempDirectory()
work_directory.write("a.h", b"")
work_directory.write("a.c", b"")
work_directory.write("a.cc", b"")
work_directory.write("CMakeLists.txt", b"")
work_directory.write("blah.txt", b"")
work_directory.makedir(".git")
work_directory.makedir(".hg")
wd_len = len(work_directory.path) + 1
w = Watcher(work_directory.path, None)
watchstate = w.poll()
filelist = [f[wd_len:] for f in watchstate.inserts]
filelist.sort()
assert filelist == ["CMakeLists.txt", "a.c", "a.cc", "a.h", "blah.txt"]
class RealParserWalkerTestCase(BaseTestCase):
"""Test when first_file and parse files are not in the same directory."""
def setUp(self):
super(RealParserWalkerTestCase, self).setUp()
self.d2 = TempDirectory()
self.dirname = self.d2.getpath('')
self.parse_file_2_name = b'baz.py'
self.d2.write(self.parse_file_1_name, self.parse_file_2_name)
self.d2.write(self.parse_file_2_name, b'Some nonsense')
self.parse_file_1_path = self.d2.getpath(self.parse_file_1_name)
self.parse_file_2_path = self.d2.getpath(self.parse_file_2_name)
self.pw = ParserWalker(self.first_file_path, self.dirname)
def test_dif(self):
"""Test second condition. Buy one get one free fruit tea."""
expected = self.pw.parse_files()
self.assertItemsEqual(list(expected), [
self.first_file_path, self.parse_file_1_path,
self.parse_file_2_path
])
class TestRawConverter(unittest.TestCase):
def setUp(self):
self.tmp = TempDirectory()
foo = self.tmp.write('foo.csv', b"1,2,3\n4,5,6\n7,8,9")
bar = self.tmp.write('bar.csv', b"9,8,7\n6,5,4\n3,2,1")
self.rawleft = self.tmp.write('P100_E4_left.dat',
b'\x05\x07\x3d\x08\xdd\x07\xf1\x06\x98\x06\x3a\x07')
self.rawright = self.tmp.write('P100_E4_right.dat',
b'\xe2\x06\x6b\x08\xe8\x07\x39\x07\xb1\x06\x48\x07')
self.foo = shared.UninitializedSensor._make([shared.Orientation(1), foo])
self.bar = shared.UninitializedSensor._make([shared.Orientation(2), bar])
def tearDown(self):
TempDirectory.cleanup_all()
def test_sensor_setup(self):
conv = RawConverter([], [self.foo, self.bar])
conv.processDatFiles()
self.assertIsInstance(conv.initSensors[
shared.Orientation(1).name], Sensor)
self.assertIsInstance(conv.initSensors[
shared.Orientation(2).name], Sensor)
self.assertEquals(conv.initSensors[
shared.Orientation(1).name].orientation, shared.Orientation(1))
self.assertEquals(conv.initSensors[
shared.Orientation(2).name].orientation, shared.Orientation(2))
def test_invalid_filesToConvert_parameter(self):
conv = RawConverter(
[self.rawleft, self.rawright], [self.foo, self.bar])
self.assertRaises(TypeError, conv.processDatFiles)
def test_dat_file_processing(self):
first = shared.Sourcefile._make(
[os.path.dirname(self.rawleft), 'P100_E4_left.dat', '01', '01', shared.Orientation(1)])
second = shared.Sourcefile._make(
[os.path.dirname(self.rawright), 'P100_E4_right.dat', '01', '01', shared.Orientation(2)])
conv = RawConverter(
[(first, second)], [self.foo, self.bar])
conv.processDatFiles()
class test_next_output_stem(unittest.TestCase):
"""tests for next_output_stem"""
#next_output_stem(prefix, path=None, start=1, inc=1, zfill=3,
# overwrite=False)
def setUp(self):
self.tempdir = TempDirectory()
self.tempdir.write('a_004', b'some text a4')
self.tempdir.write('a_005', b'some text a5')
self.tempdir.write('b_002.txt', b'some text b2')
self.tempdir.write('b_008.out', b'some text b8')
self.tempdir.write(('c_010', 'por'), b'some text c5por')
def tearDown(self):
self.tempdir.cleanup()
# @with_setup(setup=self.setup, teardown=self.teardown)
def test_file(self):
assert_equal(next_output_stem(prefix='a_', path=self.tempdir.path),
'a_006')
def test_file2(self):
assert_equal(next_output_stem(prefix='b_', path=self.tempdir.path),
'b_009')
def test_directory(self):
assert_equal(next_output_stem(prefix='c_', path=self.tempdir.path),
'c_011')
def test_file_overwrite(self):
assert_equal(
next_output_stem(prefix='a_',
path=self.tempdir.path,
overwrite=True), 'a_005')
def test_inc(self):
assert_equal(
next_output_stem(prefix='a_', path=self.tempdir.path, inc=3),
'a_008')
def test_zfill(self):
assert_equal(
next_output_stem(prefix='a_', path=self.tempdir.path, zfill=5),
'a_00006')
def test_does_not_exist(self):
assert_equal(next_output_stem(prefix='g_', path=self.tempdir.path),
'g_001')
def test_does_not_exist(self):
assert_equal(
next_output_stem(prefix='g_', path=self.tempdir.path, start=4),
'g_004')
class TestConfig(unittest.TestCase):
def setUp(self):
self.tmp_dir = TempDirectory()
def tearDown(self):
self.tmp_dir.cleanup()
def test_load_config_for_path_none(self):
self.assertEqual(load_config(None), {})
def test_load_config_for_invalid_path(self):
self.assertRaises(ConfigurationError, load_config, '/some/path')
def test_load_config_for_valid_path(self):
path = self._write_config_file('.shaker.yml', b'')
self.assertEqual(load_config(path), None)
def test_load_config_with_sources(self):
path = self._write_config_file('.shaker.yml', b'sources:')
self.assertEqual(load_config(path), { 'sources': None })
def _write_config_file(self, filename, filecontent):
self.tmp_dir.write(filename, filecontent)
return os.path.join(self.tmp_dir.path, filename)
def setUp(self):
data.app.config['TESTING'] = True
data.app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///:memory:'
data.app.config['USER_LIST'] = ['Test Admin']
db.init_app(data.app)
d = TempDirectory()
d.makedir('dataset')
d.write('dataset/files/testfile.jpg', 'abc')
d.write('dataset/testfile2.jpg', 'abc')
self.directory = d
data.app.config['DATASET_ROOT'] = d.path
with data.app.app_context(), TempDirectory() as d:
db.create_all()
lib.models.buildDB()
license = lib.models.License.query.first()
provider = lib.models.Provider.query.first()
dataset = lib.models.Datasets()
dataset.name = 'Test'
dataset.license = [license]
dataset.provider = [provider]
dataset.path = "dataset"
db.session.add(dataset)
db.session.commit()
self.app = data.app.test_client()
class IniFileConfigurationLoaderTestCase(BaseTestCase):
def setUp(self):
super(IniFileConfigurationLoaderTestCase, self).setUp()
self.tmp_dir = TempDirectory()
def tearDown(self):
super(IniFileConfigurationLoaderTestCase, self).tearDown()
self.tmp_dir.cleanup_all()
def test_skip_invalid_ini_file(self):
from prettyconf.loaders import IniFileConfigurationLoader
test_file = self.tmp_dir.write('some/strange/config.cfg', '*&ˆ%$#$%ˆ&*('.encode('utf8'))
with self.assertRaises(InvalidConfigurationFile):
IniFileConfigurationLoader(test_file)
class TranslationLoaderTest(TestCase):
u"""
Tests ``TranslationLoader`` template loader. Checks that the loader loads original template
only if there is no translated template for the active language. If there is a translated
template for the active languate, the loader loads this translated template. Also tests that an
exception is raised if there is no original nor translated template.
"""
def setUp(self):
self.tempdir = TempDirectory()
self.settings_override = override_settings(
LANGUAGES=((u'de', u'Deutsch'), (u'en', u'English'), (u'fr', u'Francais')),
TEMPLATE_LOADERS=((u'poleno.utils.template.TranslationLoader', u'django.template.loaders.filesystem.Loader'),),
TEMPLATE_DIRS=(self.tempdir.path,),
)
self.settings_override.enable()
self.tempdir.write(u'first.html', u'(first.html)\n')
self.tempdir.write(u'first.en.html', u'(first.en.html)\n')
self.tempdir.write(u'second.de.html', u'(second.de.html)\n')
def tearDown(self):
self.settings_override.disable()
self.tempdir.cleanup()
def test_translated_template_has_priority(self):
# Existing: first.html, first.en.html
with translation(u'en'):
rendered = squeeze(render_to_string(u'first.html'))
self.assertEqual(rendered, u'(first.en.html)')
def test_with_only_translated_template(self):
# Existing: second.de.html
# Missing: second.html
with translation(u'de'):
rendered = squeeze(render_to_string(u'second.html'))
self.assertEqual(rendered, u'(second.de.html)')
def test_with_only_untranslated_template(self):
# Existing: first.html
# Missing: first.de.html
with translation(u'de'):
rendered = squeeze(render_to_string(u'first.html'))
self.assertEqual(rendered, u'(first.html)')
def test_missing_template_raises_exception(self):
# Missing: second.html, second.en.html
with self.assertRaises(TemplateDoesNotExist):
render_to_string(u'second.html')
class TestSensor(unittest.TestCase):
def setUp(self):
self.tmp = TempDirectory()
self.calibration = self.tmp.write('calibration.csv',
b"1.1,2,3\n4,5.5,6\n7,8,9.9")
def tearDown(self):
TempDirectory.cleanup_all()
def test_missing_sensor_calibration_values(self):
calibration = self.tmp.write('calibration.csv',
b"1.1,9")
self.assertRaises(TypeError, Sensor,
shared.Orientation.left, calibration)
def test_invalid_sensor_calibration_values(self):
calibration = self.tmp.write('calibration.csv',
b"1.1,2,3\n4,foobar,6\n7,8,9.9")
self.assertRaises(ValueError, Sensor,
shared.Orientation.left, calibration)
def test_sensor_calibration_file(self):
calibration = self.tmp.write('calibration.csv',
b"1.1,2,3\n4,5.5,6\n7,8,9.9")
sens = Sensor(shared.Orientation.left, calibration)
self.assertEquals(sens.calibration[1], 2)
self.assertEquals(sens.calibration[4], 5.5)
self.assertEquals(sens.calibration[8], 9.9)
def test_normalizing_return(self):
sens = Sensor(shared.Orientation(1).name, self.calibration)
foo = shared.Sensorsegment._make([1,2,3,4,5,6])
bar = sens._normalizeRawSegment(foo)
self.assertIsInstance(foo, type(foo))
def test_normalizing(self):
# eGaiT_database/P100_E4_left.dat
# A917.csv
calibrationLeft = self.tmp.write('left.csv',
b"2367.1,2274.9,2271.2\n1871.8,1795.5,1753.1\n1785.8,1684.4,1855.4")
# A6DF.csv
calibrationRight = self.tmp.write('right.csv',
b"2403.5,2254.8,2266.1\n1899.7,1769.3,1773.2\n1835.2, 1709.6,1860.5")
# first 12 bytes of P100_E4_left.dat
rawRight = shared.Sensorsegment._make([1762, 2155, 2024, 1849, 1713, 1864])
# first 12 bytes of P100_E4_right.dat
rawLeft = shared.Sensorsegment._make([1797, 2109, 2013, 1777, 1688, 1850])
# expected left end results
resL = shared.Sensorsegment._make([-1.3020391681808998,0.30788485607008736,0.003281219841729923,-3.2222629073599247,1.318198462101761,-1.9772976931527246])
# expected right end results
resR = shared.Sensorsegment._make([-1.5466454942437473,0.5888774459320278,0.01765063907486269,5.053094104723528,1.2449652142072833,1.2815818381545223])
sensL = Sensor(shared.Orientation.left, calibrationLeft)
sensR = Sensor(shared.Orientation.right, calibrationRight)
bar = sensL._normalizeRawSegment(rawLeft)
foo = sensR._normalizeRawSegment(rawRight)
self.assertEqual(resL, bar)
self.assertEqual(resR, foo)
def test_default_watcher_gets_all_files(self):
work_directory = TempDirectory()
work_directory.write('a.h', b'')
work_directory.write('a.c', b'')
work_directory.write('a.cc', b'')
work_directory.write('CMakeLists.txt', b'')
work_directory.write('blah.txt', b'')
work_directory.makedir('.git')
work_directory.makedir('.hg')
wd_len = len(work_directory.path) + 1
w = Watcher(work_directory.path, None)
watchstate = w.poll()
filelist = [f[wd_len:] for f in watchstate.inserts]
filelist.sort()
assert filelist == [
'CMakeLists.txt',
'a.c',
'a.cc',
'a.h',
'blah.txt'
]
class test_next_output_stem(unittest.TestCase):
"""tests for next_output_stem"""
#next_output_stem(prefix, path=None, start=1, inc=1, zfill=3,
# overwrite=False)
def setUp(self):
self.tempdir = TempDirectory()
self.tempdir.write('a_004', b'some text a4')
self.tempdir.write('a_005', b'some text a5')
self.tempdir.write('b_002.txt', b'some text b2')
self.tempdir.write('b_008.out', b'some text b8')
self.tempdir.write(('c_010', 'por'), b'some text c5por')
def tearDown(self):
self.tempdir.cleanup()
# @with_setup(setup=self.setup, teardown=self.teardown)
def test_file(self):
assert_equal(next_output_stem(prefix='a_', path=self.tempdir.path),
'a_006')
def test_file2(self):
assert_equal(next_output_stem(prefix='b_', path=self.tempdir.path),
'b_009')
def test_directory(self):
assert_equal(next_output_stem(prefix='c_', path=self.tempdir.path),
'c_011')
def test_file_overwrite(self):
assert_equal(next_output_stem(prefix='a_', path=self.tempdir.path,
overwrite=True),
'a_005')
def test_inc(self):
assert_equal(next_output_stem(prefix='a_', path=self.tempdir.path,
inc=3),
'a_008')
def test_zfill(self):
assert_equal(next_output_stem(prefix='a_', path=self.tempdir.path,
zfill=5),
'a_00006')
def test_does_not_exist(self):
assert_equal(next_output_stem(prefix='g_', path=self.tempdir.path),
'g_001')
def test_does_not_exist(self):
assert_equal(next_output_stem(prefix='g_', path=self.tempdir.path,
start=4),
'g_004')
class Test_Logging_setup(TestCase):
def setUp(self):
self.default_config = bbpgsql.configuration.config()
self.td = TempDirectory()
self.logdir = self.td.makedir('log')
self.logfilepath = os.path.join(self.logdir, 'bbpgsql')
self.config_path = self.td.write('bbpgsql', config_text)
self.full_config = bbpgsql.configuration.config([self.config_path])
self.full_config.set('Logging', 'logfile', self.logfilepath)
self.config_path = self.td.write('bad_level', config_bad_values)
self.bad_config = bbpgsql.configuration.config([self.config_path])
self.config_path = self.td.write('section_only', config_only_section)
self.section_only = bbpgsql.configuration.config([self.config_path])
def tearDown(self):
self.td.cleanup()
def test_default_config_lacks_logging(self):
self.assertFalse(self.default_config.has_section('Logger'))
def test_section_only_does_no_logging(self):
log_config = bbpgsql.configuration.set_up_logging(self.section_only)
# assert only null handler
self.assertEqual(1, len(log_config['handlers']))
@patch('logging.config.dictConfig')
def test_logger_sets_up_default_config(self, mock_dictConfig):
mock_dictConfig.return_value = None
bbpgsql.configuration.set_up_logging(self.default_config)
self.assertTrue(mock_dictConfig.called)
expected = ((default_log_config, ), {})
self.assertEqual(expected, mock_dictConfig.call_args)
def test_logger_sets_log_level_from_configfile(self):
log_config = bbpgsql.configuration.set_up_logging(self.full_config)
self.assertEqual(DEBUG,
log_config['loggers']['bbpgsql']['level'])
def test_logger_rejects_bad_level_name(self):
self.assertRaises(Exception, bbpgsql.configuration.set_up_logging,
(self.bad_config))
def test_logger_gets_filename_from_config(self):
handlers, formatters = \
bbpgsql.configuration.set_up_logger_file_handler(self.full_config)
self.assertEqual(handlers,
{'file': {
'class': 'logging.handlers.TimedRotatingFileHandler',
'formatter': 'file_formatter',
'filename': self.logfilepath,
'when': 'd',
'interval': 1,
'backupCount': 14
}
}
)
self.assertEqual(formatters,
{
'file_formatter': {
'format': "%(asctime)s - %(levelname)s - %(message)s",
}
}
)
def test_logger_does_nothing_if_section_is_missing(self):
handlers, formatters = \
bbpgsql.configuration.set_up_logger_file_handler(
self.default_config)
self.assertEqual({}, handlers)
self.assertEqual({}, formatters)
def test_logger_gets_hostname_port_from_config(self):
handlers, formatters = \
bbpgsql.configuration.set_up_logger_syslog_handler(
self.full_config)
self.assertEqual(handlers['syslog']['address'],
('localhost', 514))
def test_logger_does_nothing_if_syslog_host_not_defined(self):
conf = SafeConfigParser()
conf.add_section('Logging')
conf.set('Logging', 'logport', '514')
handlers, formatters = \
bbpgsql.configuration.set_up_logger_syslog_handler(conf)
self.assertEqual({}, handlers)
self.assertEqual({}, formatters)
def test_logger_does_nothing_if_syslog_port_not_defined(self):
conf = SafeConfigParser()
conf.add_section('Logging')
conf.set('Logging', 'loghost', 'localhost')
handlers, formatters = \
bbpgsql.configuration.set_up_logger_syslog_handler(conf)
self.assertEqual({}, handlers)
self.assertEqual({}, formatters)
def test_logger_uses_UDP_by_default(self):
conf = SafeConfigParser()
conf.add_section('Logging')
conf.set('Logging', 'loghost', 'localhost')
conf.set('Logging', 'logport', '514')
handlers, formatters = \
bbpgsql.configuration.set_up_logger_syslog_handler(conf)
syslog_handler = handlers[handlers.keys()[0]]
self.assertEqual(socket.SOCK_DGRAM, syslog_handler['socktype'])
def test_logger_uses_TCP_when_tcpon_is_false(self):
conf = SafeConfigParser()
conf.add_section('Logging')
conf.set('Logging', 'loghost', 'localhost')
conf.set('Logging', 'logport', '514')
conf.set('Logging', 'logtcp', 'off')
handlers, formatters = \
bbpgsql.configuration.set_up_logger_syslog_handler(conf)
syslog_handler = handlers[handlers.keys()[0]]
self.assertEqual(socket.SOCK_DGRAM, syslog_handler['socktype'])
def test_logger_uses_TCP_when_tcpon_is_true(self):
conf = SafeConfigParser()
conf.add_section('Logging')
conf.set('Logging', 'loghost', 'localhost')
conf.set('Logging', 'logport', '514')
conf.set('Logging', 'logtcp', 'on')
handlers, formatters = \
bbpgsql.configuration.set_up_logger_syslog_handler(conf)
syslog_handler = handlers[handlers.keys()[0]]
self.assertEqual(socket.SOCK_STREAM, syslog_handler['socktype'])
def test_logger_uses_facility_user_by_default(self):
conf = SafeConfigParser()
conf.add_section('Logging')
conf.set('Logging', 'loghost', 'localhost')
conf.set('Logging', 'logport', '514')
handlers, formatters = \
bbpgsql.configuration.set_up_logger_syslog_handler(conf)
syslog_handler = handlers[handlers.keys()[0]]
self.assertEqual(logging.handlers.SysLogHandler.LOG_USER,
syslog_handler['facility'])
def test_logger_uses_facility_user_when_specified(self):
conf = SafeConfigParser()
conf.add_section('Logging')
conf.set('Logging', 'loghost', 'localhost')
conf.set('Logging', 'logport', '514')
conf.set('Logging', 'logfacility', 'user')
handlers, formatters = \
bbpgsql.configuration.set_up_logger_syslog_handler(conf)
syslog_handler = handlers[handlers.keys()[0]]
self.assertEqual(logging.handlers.SysLogHandler.LOG_USER,
syslog_handler['facility'])
def test_logger_uses_facility_local0_when_specified(self):
conf = SafeConfigParser()
conf.add_section('Logging')
conf.set('Logging', 'loghost', 'localhost')
conf.set('Logging', 'logport', '514')
conf.set('Logging', 'logfacility', 'local0')
handlers, formatters = \
bbpgsql.configuration.set_up_logger_syslog_handler(conf)
syslog_handler = handlers[handlers.keys()[0]]
self.assertEqual(logging.handlers.SysLogHandler.LOG_LOCAL0,
syslog_handler['facility'])
def test_logger_raises_exception_on_invalid_facility(self):
conf = SafeConfigParser()
conf.add_section('Logging')
conf.set('Logging', 'loghost', 'localhost')
conf.set('Logging', 'logport', '514')
conf.set('Logging', 'logfacility', 'quidgybo')
self.assertRaises(Exception,
bbpgsql.configuration.set_up_logger_syslog_handler, conf)
@patch('bbpgsql.configuration.set_up_logger_file_handler')
@patch('bbpgsql.configuration.set_up_logger_syslog_handler')
def test_set_up_logging_calls_handler_setups(
self,
mock_set_up_logger_syslog_handler,
mock_set_up_logger_file_handler,
):
mock_set_up_logger_syslog_handler.return_value = ({}, {})
mock_set_up_logger_file_handler.return_value = ({}, {})
log_config = bbpgsql.configuration.set_up_logging(self.full_config)
from pprint import pprint
pprint(log_config)
self.assertEqual(1, mock_set_up_logger_file_handler.call_count)
self.assertEqual(1, mock_set_up_logger_syslog_handler.call_count)
@patch('bbpgsql.configuration.set_up_logger_file_handler')
@patch('bbpgsql.configuration.set_up_logger_syslog_handler')
def test_set_up_logging_does_not_setup_handlers_if_no_logging_section(
self,
mock_set_up_logger_syslog_handler,
mock_set_up_logger_file_handler,
):
bbpgsql.configuration.set_up_logging(self.default_config)
self.assertEqual(0, mock_set_up_logger_file_handler.call_count)
self.assertEqual(0, mock_set_up_logger_syslog_handler.call_count)
class Tests(ContextTest):
context = OutputtingContext
def setUp(self):
ContextTest.setUp(self)
self.jenkins = TempDirectory()
def tearDown(self):
self.jenkins.cleanup()
ContextTest.tearDown(self)
def test_simple(self):
self.jenkins.write('nodeMonitors.xml', 'nodeMonitors')
self.jenkins.write('another.xml', 'another')
self.jenkins.write('jobs/test-multi/config.xml', 'multi-config')
self.jenkins.write('jobs/test-another/config.xml', 'single-config')
self.jenkins.write('jobs/test-another/workspace/junk.xml', 'junk')
compare(check(self.c.dir.path, self.jenkins.path),'')
self.c.dir.check_all(
self.jenkins.path.split(os.sep)[1:],
'another.xml',
'jobs/',
'jobs/test-another/',
'jobs/test-another/config.xml',
'jobs/test-multi/',
'jobs/test-multi/config.xml',
'nodeMonitors.xml'
)
def test_overwrite(self):
path = self.jenkins.write('config.xml', 'new')
path = path.split(os.sep)[1:]
self.c.dir.write(path, 'old')
compare(check(self.c.dir.path, self.jenkins.path),'')
compare(self.c.dir.read(path), 'new')
class SendFileResponseTest(TestCase):
u"""
Tests ``send_file_response()`` function. Checks that regular files are sent correctly, but
sending non-regular or non-existent files raises an exception. Also checks that if the request
has ``HTTP_IF_MODIFIED_SINCE`` header, the file is sent only if it was changes since then.
Finally checks if ``Last-Modified``, ``Content-Disposition`` and ``Content-Length`` headers are
set correctly.
"""
def file_view(request):
path = request.GET[u'path']
name = request.GET[u'name']
content_type = request.GET[u'content-type']
return send_file_response(request, path, name, content_type)
urls = patterns(u'',
url(r'^file/$', file_view),
)
def setUp(self):
self.tempdir = TempDirectory()
def tearDown(self):
self.tempdir.cleanup()
def _create_file(self, filename=u'myfile.tmp', content=u'Some text.'):
self.tempdir.write(filename, content)
return self.tempdir.getpath(filename)
def _request_file(self, path, name=u'filename.bin', content_type=u'text/plain', **kwargs):
params = urlencode({u'path': path, u'name': name, u'content-type': content_type})
return self.client.get(u'/file/?%s' % params, **kwargs)
def _check_response(self, response, klass, status_code):
self.assertIs(type(response), klass)
self.assertEqual(response.status_code, status_code)
def _check_content(self, response, path):
with open(path, 'rb') as f:
content = f.read()
self.assertEqual(u''.join(response.streaming_content), content)
def test_regular_file(self):
path = self._create_file()
response = self._request_file(path)
self._check_response(response, FileResponse, 200)
self._check_content(response, path)
def test_directory_raises_exception(self):
with self.assertRaisesMessage(OSError, u'Not a regular file: /'):
response = self._request_file(u'/')
def test_nonexistent_file_raises_exception(self):
with self.assertRaisesMessage(OSError, u"[Errno 2] No such file or directory: '/nonexistent.txt'"):
response = self._request_file(u'/nonexistent.txt')
def test_random_file(self):
content = random_string(random.randrange(1000, 2000))
path = self._create_file(content=content)
response = self._request_file(path)
self._check_response(response, FileResponse, 200)
self._check_content(response, path)
def test_if_modified_since_with_modified_file(self):
u"""
Checks that if the request has ``HTTP_IF_MODIFIED_SINCE`` header and the file was indeed
modified since then, the file is sent.
"""
modified_timestamp = 1413500000
if_modified_since_timestamp = modified_timestamp + 1000000
path = self._create_file()
os.utime(path, (modified_timestamp, modified_timestamp))
response = self._request_file(path, HTTP_IF_MODIFIED_SINCE=http_date(if_modified_since_timestamp))
self._check_response(response, HttpResponseNotModified, 304)
def test_if_modified_since_with_unmodified_file(self):
u"""
Checks that if the request has ``HTTP_IF_MODIFIED_SINCE`` header and the file was NOT
modified since then, the file is not sent.
"""
modified_timestamp = 1413500000
if_modified_since_timestamp = modified_timestamp - 1000000
path = self._create_file()
os.utime(path, (modified_timestamp, modified_timestamp))
response = self._request_file(path, HTTP_IF_MODIFIED_SINCE=http_date(if_modified_since_timestamp))
self._check_response(response, FileResponse, 200)
self._check_content(response, path)
def test_last_modified_response_header(self):
modified_timestamp = 1413500000
path = self._create_file()
os.utime(path, (modified_timestamp, modified_timestamp))
response = self._request_file(path)
self.assertEqual(response[u'Last-Modified'], u'Thu, 16 Oct 2014 22:53:20 GMT')
def test_content_length_header(self):
path = self._create_file(content=u'1234567890')
response = self._request_file(path)
self.assertEqual(response[u'Content-Length'], u'10')
def test_content_length_header_for_random_file(self):
content = random_string(random.randrange(1000, 2000))
path = self._create_file(content=content)
response = self._request_file(path)
self.assertEqual(response[u'Content-Length'], str(len(content)))
def test_content_disposition_header(self):
path = self._create_file()
response = self._request_file(path, u'thefile.txt')
self.assertEqual(response[u'Content-Disposition'], u"attachment; filename*=UTF-8''thefile.txt")
def test_content_disposition_header_with_space(self):
path = self._create_file()
response = self._request_file(path, u'the file.txt')
self.assertEqual(response[u'Content-Disposition'], u"attachment; filename*=UTF-8''the%20file.txt")
def test_content_disposition_header_with_diacritic(self):
path = self._create_file()
response = self._request_file(path, u'ľťéŠÝÄÚ.txt')
self.assertEqual(response[u'Content-Disposition'], u"attachment; filename*=UTF-8''%C4%BE%C5%A5%C3%A9%C5%A0%C3%9D%C3%84%C3%9A.txt")
def test_content_disposition_header_with_random_unicode_junk(self):
path = self._create_file()
name = random_string(20, chars=u'BａｃòԉíρｓûϻᏧｏｌｒѕìｔãｍｅéӽѵ߀ɭｐèлｕｉｎ.Iüà,ɦëǥｈƅɢïêｇԁSùúâɑｆäｂƃｄｋϳɰյƙｙáFХ-åɋｗ')
response = self._request_file(path, name)
self.assertEqual(response[u'Content-Disposition'], u"attachment; filename*=UTF-8''%s" % urlquote(name))
class Skeleton_S3CommitStorage_Against_Mock(TestCase):
__test__ = False
def setUp(self):
self.tempdir = TempDirectory()
self.setup_mock_defaults()
self.store = S3CommitStorage(self.mock_bucket, self.bucket_prefix)
def setup_mock_defaults(self):
self.mock_bucket = Mock(spec=Bucket)
self.set_bucket_list([])
def tearDown(self):
self.tempdir.cleanup()
def prefix_key(self, key):
return ''.join([self.bucket_prefix or '', key])
def set_bucket_list(self, keynames):
prefixed_keynames = [self.prefix_key(key) for key in keynames]
key_objs = [Key(None, key) for key in prefixed_keynames]
self.mock_bucket.list.return_value = key_objs
def test_get_tags_calls_bucket_list__empty(self):
self.assertEqual([], self.store.get_tags())
self.mock_bucket.list.assert_called_with(prefix=self.prefix_key(''))
def test_get_tags_calls_bucket_list_not_empty(self):
self.set_bucket_list(['tag1_msg1', 'tag2_msg2'])
self.store.get_tags()
self.mock_bucket.list.assert_called_with(prefix=self.prefix_key(''))
def test_get_tags_parses_keys_properly(self):
self.set_bucket_list(['tag1_msg1', 'tag2_msg2'])
self.assertEqual(['tag1', 'tag2'], self.store.get_tags())
def test_get_message_for_tag_calls_bucket_list(self):
self.set_bucket_list(['tag1_msg1'])
self.store.get_message_for_tag('tag1')
prefix = self.prefix_key(''.join(['tag1', '_']))
self.mock_bucket.list.assert_called_with(prefix=prefix)
def test_get_message_for_tag_parses_keyname_properly(self):
self.set_bucket_list(['tag1_msg1'])
self.assertEqual('msg1', self.store.get_message_for_tag('tag1'))
def test_add_commit_calls_new_key_with_expected_format(self):
filename1 = self.tempdir.write('file1', 'some file contents')
self.store.add_commit('tag1', open(filename1, 'rb'), 'some_message')
expected_key_name = self.prefix_key(''.join(['tag1',
'_', 'some_message']))
self.mock_bucket.new_key.assert_called_with(expected_key_name)
def test_add_commit_calls_set_contents_from_filename(self):
commit_contents = 'some file contents'
filename1 = self.tempdir.write('file1', commit_contents)
fp1 = open(filename1, 'rb')
self.store.add_commit('tag1', fp1, 'some_message')
new_key_mock = self.mock_bucket.new_key.return_value
new_key_mock.set_contents_from_file.assert_called_once()
mock_last_called_kwargs = new_key_mock.\
set_contents_from_file.call_args[1]
mock_actual_headers = mock_last_called_kwargs['headers']
lower_headers = {}
for k in mock_actual_headers:
lower_headers[k.lower()] = mock_actual_headers[k]
expected_headers = {
'Content-Type': 'application/octet-stream',
'Content-Encoding': 'gzip',
'x-amz-meta-fingerprint': md5(commit_contents).hexdigest()}
for header in expected_headers:
self.assertIn(header.lower(), lower_headers.keys())
self.assertEqual(expected_headers[header],
lower_headers[header.lower()])
def test_delete_commit_calls_get_key(self):
self.set_bucket_list(['tag1_msg1'])
self.store.delete_commit('tag1')
self.mock_bucket.get_key.assert_called_with(
self.prefix_key('tag1_msg1'))
self.mock_bucket.get_key.return_value.delete.assert_called_with()
def test_get_commit_contents_calls_get_contents_to_filename(self):
self.set_bucket_list(['tag1_msg1'])
target_file = self.tempdir.getpath('restored_file')
self.store.get_commit_contents_to_filename('tag1', target_file)
get_key = self.mock_bucket.get_key.return_value
get_key.get_contents_to_filename.assert_called_once()
def test_if_no_content_encoding_restores_to_target_file(self):
self.set_bucket_list(['tag1_msg1'])
target_file = self.tempdir.getpath('restored_file')
self.store.get_commit_contents_to_filename('tag1', target_file)
get_key = self.mock_bucket.get_key.return_value
get_key.get_contents_to_filename.assert_called_with(target_file)
def test_if_content_encoding_gzip_calls_gunzip_key_to_filename(self):
self.set_bucket_list(['tag1_msg1'])
target_file = self.tempdir.getpath('restored_file')
self.store._gunzip_key_to_filename = Mock()
# set content-encoding on key
get_key = self.mock_bucket.get_key.return_value
get_key.content_encoding = 'gzip'
self.store.get_commit_contents_to_filename('tag1', target_file)
get_key.get_contents_to_filename.assert_called_once()
self.store._gunzip_key_to_filename.assert_called_with(get_key,
target_file)
def test_get_commit_contents_raises_Exception_if_file_exists(self):
self.set_bucket_list(['tag1_msg1'])
file1 = self.tempdir.write('file1', 'some file contents')
def will_raise_Exception():
self.store.get_commit_contents_to_filename('tag1', file1)
self.assertRaises(FileAlreadyExistsError, will_raise_Exception)
def test_dictionary_interface_returns_a_commit_object(self):
self.set_bucket_list(['tag1_msg1'])
commit = self.store['tag1']
self.assertEqual('tag1', commit.tag)
self.assertEqual('msg1', commit.message)
def test_dictionary_interface_uses_custom_header_to_get_fingerprint(self):
self.set_bucket_list(['tag1_msg1'])
self.mock_bucket.get_key.return_value.get_metadata.return_value = '123'
commit = self.store['tag1']
self.assertEqual('tag1', commit.tag)
self.assertEqual('msg1', commit.message)
self.assertEqual('123', commit.fingerprint)
def test_dictionary_interface_raises_Exception_if_unknown_tag(self):
def will_raise_UnknownTagError():
self.store['tag1']
self.assertRaises(UnknownTagError, will_raise_UnknownTagError)
def test_contains_interface_calls_bucket_list_with_prefix(self):
self.set_bucket_list(['tag1_msg1'])
'tag1' in self.store
self.mock_bucket.list.assert_called_with(
prefix=self.prefix_key('tag1_'))
def test_contains_interface_returns_true_for_tags_in_bucket(self):
self.set_bucket_list(['tag1_msg1'])
self.assertTrue('tag1' in self.store)
def test_contains_interface_returns_false_for_tags_not_in_bucket(self):
self.assertFalse('tag2' in self.store)
def test_raises_Exception_if_prefix_has_leading_slash(self):
def will_raise_Exception():
S3CommitStorage(self.mock_bucket, '/illegal_prefix')
self.assertRaises(Exception, will_raise_Exception)
class test_GenericInputFileArgParser(unittest.TestCase):
"""tests GenericInputFileArgParser"""
def setUp(self):
self.tempdir = TempDirectory()
self.tempdir.write('a1.py', "a1", 'utf-8')
self.tempdir.write('a2.py', "a2", 'utf-8')
self.tempdir.write('b1.txt', "b1", 'utf-8')
self.tempdir.write('b2.txt', "b2", 'utf-8')
def tearDown(self):
self.tempdir.cleanup()
def _abc_fobj(self, fobj):
"""print file contents to out.zebra"""
with open(os.path.join(self.tempdir.path, 'out.zebra'), 'a') as f:
f.write(fobj.read()+'\n')
return
def _abc_path(self, path):
"""print file basename out.zebra"""
with open(os.path.join(self.tempdir.path, 'out.zebra'), 'a') as f:
f.write(os.path.basename(path)+'\n')
return
# def dog(self):
# with open(os.path.join(self.tempdir.path, 'out.zebra'), 'a') as f:
# f.write('dog\n')
def test_directory_with_path(self):
a = GenericInputFileArgParser(self._abc_path, False)
args = '-d {0} -p'.format(self.tempdir.path).split()
print(args)
a.main(argv=args)
assert_equal(self.tempdir.read(('out.zebra'), 'utf-8').splitlines(),
textwrap.dedent("""\
a1.py
a2.py""").splitlines())
def test_directory_with_fobj(self):
a = GenericInputFileArgParser(self._abc_fobj, True)
args = '-d {0} -p'.format(self.tempdir.path).split()
# print(args)
a.main(argv=args)
assert_equal(self.tempdir.read(('out.zebra'), 'utf-8').splitlines(),
textwrap.dedent("""\
a1
a2""").splitlines())
def test_pattern_with_path(self):
a = GenericInputFileArgParser(self._abc_path, False)
args = '-d {0} -p *.txt'.format(self.tempdir.path).split()
# print(args)
a.main(argv=args)
assert_equal(self.tempdir.read(('out.zebra'), 'utf-8').splitlines(),
textwrap.dedent("""\
b1.txt
b2.txt""").splitlines())
def test_pattern_with_fobj(self):
a = GenericInputFileArgParser(self._abc_fobj, True)
args = '-d {0} -p *.txt'.format(self.tempdir.path).split()
# print(args)
a.main(argv=args)
assert_equal(self.tempdir.read(('out.zebra'), 'utf-8').splitlines(),
textwrap.dedent("""\
b1
b2""").splitlines())
def test_filename_with_fobj(self):
a = GenericInputFileArgParser(self._abc_fobj, True)
args = '-f {0} {1}'.format(
os.path.join(self.tempdir.path, 'a1.py'),
os.path.join(self.tempdir.path, 'b1.txt')
).split()
# print(args)
a.main(argv=args)
assert_equal(self.tempdir.read(('out.zebra'), 'utf-8').splitlines(),
textwrap.dedent("""\
a1
b1""").splitlines())
def test_filename_with_path(self):
a = GenericInputFileArgParser(self._abc_path, False)
args = '-f {0} {1}'.format(
os.path.join(self.tempdir.path, 'a1.py'),
os.path.join(self.tempdir.path, 'b1.txt')
).split()
# print(args)
a.main(argv=args)
assert_equal(self.tempdir.read(('out.zebra'), 'utf-8').splitlines(),
textwrap.dedent("""\
a1.py
b1.txt""").splitlines())
def test_default_directory(self):
a = GenericInputFileArgParser(self._abc_path, False)
args = '-d -p'.format(self.tempdir.path).split()
with working_directory(self.tempdir.path):
a.main(argv=args)
assert_equal(self.tempdir.read(('out.zebra'), 'utf-8').splitlines(),
textwrap.dedent("""\
a1.py
a2.py""").splitlines())
def test_methods_with_path(self):
a = GenericInputFileArgParser(HelperForGenericInputFileArgParser,
False, [('dog',[],{})])
args = '-f {0} {1}'.format(
os.path.join(self.tempdir.path, 'a1.py'),
os.path.join(self.tempdir.path, 'b1.txt')
).split()
a.main(argv=args)
assert_equal(self.tempdir.read(('out.zebra'), 'utf-8').splitlines(),
textwrap.dedent("""\
a1.py
dog
b1.txt
dog
""").splitlines())
class test_InputFileLoaderCheckerSaver(unittest.TestCase):
"""tests for InputFileLoaderCheckerSaver"""
def setUp(self):
self.dir = os.path.abspath(os.curdir)
self.tempdir = TempDirectory()
self.tempdir.write('inp1.py', 'a=4\nb=6', 'utf-8')
self.tempdir.write('out0001.py', 'a=4\nb=6', 'utf-8')
self.tempdir.write(('what', 'out0001.py'), 'a=4\nb=6', 'utf-8')
# self.tempdir.write('a_005', b'some text a5')
# self.tempdir.write('b_002.txt', b'some text b2')
# self.tempdir.write('b_008.out', b'some text b8')
# self.tempdir.write(('c_010', 'por'), b'some text c5por')
os.chdir(self.tempdir.path)
def tearDown(self):
os.chdir(self.dir)
self.tempdir.cleanup()
def test_init_from_str(self):
a = InputFileLoaderCheckerSaver()
a._attributes = "a b".split()
a._initialize_attributes()
a.__init__('a=4\nb=6')
assert_equal(a.a, 4)
assert_equal(a.b, 6)
assert_equal(a._input_text, 'a=4\nb=6')
def test_init_from_fileobj(self):
a = InputFileLoaderCheckerSaver()
a._attributes = "a b".split()
a._initialize_attributes()
with open(os.path.join(self.tempdir.path, 'inp1.py'), 'r') as f:
a.__init__(f)
assert_equal(a.a, 4)
assert_equal(a.b, 6)
assert_equal(a._input_text, 'a=4\nb=6')
def test_attribute_defaults(self):
a = InputFileLoaderCheckerSaver()
a._input_text = 'b=6'
a._attributes = "a b".split()
a._attribute_defaults = {'a': 24}
a._initialize_attributes()
a._transfer_attributes_from_inputfile()
assert_equal(a.a, 24)
assert_equal(a.b, 6)
assert_equal(a._input_text, 'b=6')
def test_check_attributes_that_should_be_lists(self):
a = InputFileLoaderCheckerSaver()
a.a=4
a.b=6
a._attributes_that_should_be_lists = ['b']
a.check_input_attributes()
assert_equal(a.a, 4)
assert_equal(a.b, [6])
def test_check_zero_or_all(self):
a = InputFileLoaderCheckerSaver()
a.a=4
a.b=6
a.c=None
a._zero_or_all = ['a b c'.split()]
assert_raises(ValueError, a.check_input_attributes)
def test_check_at_least_one(self):
a = InputFileLoaderCheckerSaver()
a.c=None
a._at_least_one = ['c'.split()]
assert_raises(ValueError, a.check_input_attributes)
def test_check_one_implies_others(self):
a = InputFileLoaderCheckerSaver()
a.a = 4
a.c=None
a._one_implies_others = ['a c'.split()]
assert_raises(ValueError, a.check_input_attributes)
def test_check_attributes_to_force_same_len(self):
a = InputFileLoaderCheckerSaver()
a.a = [4,5]
a.c=None
a._attributes_to_force_same_len = ['a c'.split()]
a.check_input_attributes()
assert_equal(a.c, [None, None])
def test_check_attributes_that_should_have_same_x_limits(self):
a = InputFileLoaderCheckerSaver()
a.a = PolyLine([0,1], [2,5])
a.c = PolyLine([0,7], [5,6])
a._attributes_that_should_have_same_x_limits = ['a c'.split()]
assert_raises(ValueError, a.check_input_attributes)
def test_check_attributes_that_should_have_same_len_pairs(self):
a = InputFileLoaderCheckerSaver()
a.a = [2, 3]
a.c = [3]
a._attributes_that_should_have_same_len_pairs = ['a c'.split()]
assert_raises(ValueError, a.check_input_attributes)
def test_determine_output_stem_defaults(self):
a = InputFileLoaderCheckerSaver()
a._determine_output_stem()
assert_equal(a._file_stem, '.\\out0002\\out0002')
def test_determine_output_stem_overwrite(self):
a = InputFileLoaderCheckerSaver()
a.overwrite = True
a._determine_output_stem()
assert_equal(a._file_stem, '.\\out0001\\out0001')
def test_determine_output_stem_create_directory(self):
a = InputFileLoaderCheckerSaver()
a.create_directory = False
a._determine_output_stem()
assert_equal(a._file_stem, '.\\out0002')
def test_determine_output_stem_prefix(self):
a = InputFileLoaderCheckerSaver()
a.prefix = 'hello_'
a._determine_output_stem()
assert_equal(a._file_stem, '.\\hello_0001\\hello_0001')
def test_determine_output_stem_directory(self):
a = InputFileLoaderCheckerSaver()
a.directory = os.path.join(self.tempdir.path, 'what')
a._determine_output_stem()
assert_equal(a._file_stem, os.path.join(self.tempdir.path,'what', 'out0002', 'out0002'))
def test_save_data_parsed(self):
a = InputFileLoaderCheckerSaver()
a._attributes = "a b ".split()
a.save_data_to_file=True
# a._initialize_attributes()
a.a=4
a.b=6
a._save_data()
# print(os.listdir(self.tempdir.path))
# print(os.listdir(os.path.join(self.tempdir.path,'out0002')))
assert_equal(self.tempdir.read(
('out0002','out0002_input_parsed.py'), 'utf-8').strip().splitlines(),
'a = 4\nb = 6'.splitlines())
def test_save_data_input_text(self):
a = InputFileLoaderCheckerSaver()
a._input_text= "hello"
a.save_data_to_file=True
a._save_data()
assert_equal(self.tempdir.read(
('out0002','out0002_input_original.py'), 'utf-8').strip().splitlines(),
'hello'.splitlines())
def test_save_data_input_ext(self):
a = InputFileLoaderCheckerSaver()
a._input_text= "hello"
a.input_ext= '.txt'
a.save_data_to_file=True
a._save_data()
ok_(os.path.isfile(os.path.join(
self.tempdir.path, 'out0002','out0002_input_original.txt')))
def test_save_data_grid_data_dicts(self):
a = InputFileLoaderCheckerSaver()
a._grid_data_dicts= {'data': np.arange(6).reshape(3,2)}
a.save_data_to_file=True
a._save_data()
# print(os.listdir(os.path.join(self.tempdir.path,'out0002')))
ok_(os.path.isfile(os.path.join(
self.tempdir.path, 'out0002','out0002.csv')))
def test_save_data_grid_data_dicts_data_ext(self):
a = InputFileLoaderCheckerSaver()
a._grid_data_dicts= {'data': np.arange(6).reshape(3,2)}
a.save_data_to_file=True
a.data_ext = ".txt"
a._save_data()
# print(os.listdir(os.path.join(self.tempdir.path,'out0002')))
ok_(os.path.isfile(os.path.join(
self.tempdir.path, 'out0002','out0002.txt')))
def test_save_figures(self):
a = InputFileLoaderCheckerSaver()
a.save_figures_to_file=True
fig = matplotlib.pyplot.figure()
ax = fig.add_subplot('111')
ax.plot(4,5)
fig.set_label('sing')
a._figures=[fig]
a._save_figures()
a._figures=None
matplotlib.pyplot.clf()
# print(os.listdir(os.path.join(self.tempdir.path,'out0002')))
ok_(os.path.isfile(os.path.join(
self.tempdir.path, 'out0002','out0002_sing.eps')))
def test_save_figures_figure_ext(self):
a = InputFileLoaderCheckerSaver()
a.save_figures_to_file=True
a.figure_ext='.pdf'
fig = matplotlib.pyplot.figure()
ax = fig.add_subplot('111')
ax.plot(4,5)
fig.set_label('sing')
a._figures=[fig]
a._save_figures()
a._figures=None
matplotlib.pyplot.clf()
# print(os.listdir(os.path.join(self.tempdir.path,'out0002')))
ok_(os.path.isfile(os.path.join(
self.tempdir.path, 'out0002','out0002_sing.pdf')))
class Test_archivepgsql_BasicCommandLineOperation(TestCase):
ARCHIVEPGSQL_PATH = os.path.join('bbpgsql', 'cmdline_scripts')
CONFIG_FILE = 'config.ini'
exe_script = 'archivewal'
def setUp(self):
self.setup_environment()
self.setup_config()
self.cmd = [self.exe_script, '--dry-run', '--config', self.config_path]
def setup_environment(self):
self.env = deepcopy(os.environ)
self.env['PATH'] = ''.join([
self.env['PATH'],
':',
self.ARCHIVEPGSQL_PATH])
self.tempdir = TempDirectory()
def setup_config(self):
self.storage_path = self.tempdir.makedir('repo')
self.config_path = self.tempdir.getpath(self.CONFIG_FILE)
self.log_file = self.tempdir.getpath('bbpgsql.log')
self.config_dict = {
'WAL': {
'driver': 'filesystem',
'path': self.storage_path,
},
'General': {
'pgsql_data_directory': self.tempdir.path,
},
'Logging': {
'logfile': self.log_file,
'level': 'DEBUG',
},
}
write_config_to_filename(self.config_dict, self.config_path)
#print '----'
#print open(self.config_path, 'rb').read()
#print '----'
self.pg_xlog_path = 'pg_xlog'
self.tempdir.makedir(self.pg_xlog_path)
self.wal_basename = '00001'
self.wal_filename = os.path.join(self.pg_xlog_path, self.wal_basename)
self.tempdir.write(self.wal_filename, '')
print 'TEMPDIR', self.tempdir.listdir(recursive=True)
def tearDown(self):
self.tempdir.cleanup()
def test_archivewal_returns_error_with_if_less_than_one_argument(self):
proc = Popen(self.cmd, env=self.env, stdout=PIPE, stderr=STDOUT)
proc.wait()
print(proc.stdout.read())
self.assertNotEqual(0, proc.returncode)
def test_archivewal_logs_error_with_if_less_than_one_argument(self):
proc = Popen(self.cmd, env=self.env, stdout=PIPE, stderr=STDOUT)
proc.wait()
self.assertNotEqual(0, proc.returncode)
print proc.stdout.read()
log_output = open(self.log_file, 'rb').read()
print 'log_output:'
print log_output
assert 'ERROR' in log_output
def test_archivewal_success_with_file(self):
self.cmd.append(self.wal_filename)
proc = Popen(self.cmd, env=self.env, stdout=PIPE, stderr=STDOUT)
proc.wait()
print proc.stdout.read()
self.assertEqual(0, proc.returncode)
def test_archivewal_actually_archives_file(self):
self.cmd.append(self.wal_filename)
proc = Popen(self.cmd, env=self.env, stdout=PIPE, stderr=STDOUT)
proc.wait()
print proc.stdout.read()
self.assertEqual(0, proc.returncode)
archives = os.listdir(self.storage_path)
print archives
self.assertTrue(archives[0].startswith(''.join([
self.wal_basename, '_'])))
class RenderMailTest(TestCase):
u"""
Tests ``render_mail()`` function. Checks that message subject and body alternatives are
rendered correctly, that missing templates raise an exception, and that all other arguments are
passed directly to the message constructor.
"""
def setUp(self):
self.tempdir = TempDirectory()
self.settings_override = override_settings(
TEMPLATE_LOADERS=(u'django.template.loaders.filesystem.Loader',),
TEMPLATE_DIRS=(self.tempdir.path,),
)
self.settings_override.enable()
self.tempdir.write(u'first_subject.txt', u'\n\n\t\t\r\r\t\tSubject with\t\t lots \t \n\n\t\r\r\n of whitespace\n\n \n\n \t')
self.tempdir.write(u'first_message.txt', u' \n\n \t\n\tFirst message text with leading and trailing whitespace \n \n\n \n ')
self.tempdir.write(u'first_message.html', u' \n\n \t <p>\nFirst message HTML with leading and trailing whitespace\n</p>\n\n\n\n ')
self.tempdir.write(u'second_subject.txt', u'Second subject\n')
self.tempdir.write(u'second_message.txt', u'Second message with only text\n')
self.tempdir.write(u'third_subject.txt', u'Third subject\n')
self.tempdir.write(u'third_message.html', u'<p>Third message with only HTML</p>\n')
self.tempdir.write(u'fourth_subject.txt', u'Fourth message with no body\n')
self.tempdir.write(u'fifth_message.txt', u'Fifth message with no subject\n')
self.tempdir.write(u'sixth_subject.txt', u'Subject with dictionary: {{ variable }}\n')
self.tempdir.write(u'sixth_message.txt', u'Text message with dictionary: {{ variable }}\n')
self.tempdir.write(u'sixth_message.html', u'<p>HTML message with dictionary: {{ variable }}</p>\n')
def tearDown(self):
self.settings_override.disable()
self.tempdir.cleanup()
def _get_alternatives(self, msg):
alternatives = [(u'text/' + msg.content_subtype, msg.body)]
for content, mimetype in getattr(msg, u'alternatives', []):
alternatives.append((mimetype, content))
return alternatives
def test_message_with_text_and_html(self):
# existing: first_subject.txt, first_message.txt, first_message.html
# missing: --
msg = render_mail(u'first')
alternatives = self._get_alternatives(msg)
self.assertEqual(alternatives, [
(u'text/plain', u'First message text with leading and trailing whitespace'),
(u'text/html', u'<p>\nFirst message HTML with leading and trailing whitespace\n</p>'),
])
def test_message_with_text_only(self):
# existing: second_subject.txt, second_message.txt
# missing: second_message.html
msg = render_mail(u'second')
alternatives = self._get_alternatives(msg)
self.assertEqual(alternatives, [
(u'text/plain', u'Second message with only text'),
])
def test_message_with_html_only(self):
# existing: third_subject.txt, third_message.html
# missing: third_message.txt
msg = render_mail(u'third')
alternatives = self._get_alternatives(msg)
self.assertEqual(alternatives, [
(u'text/html', u'<p>Third message with only HTML</p>'),
])
def test_message_with_missing_templates(self):
# existing: fourth_subject.txt
# missing: fourth_message.txt, fourth_message.html
with self.assertRaisesMessage(TemplateDoesNotExist, u'fourth_message.txt'):
msg = render_mail(u'fourth')
def test_message_with_missing_subject_template(self):
# existing: fifth_message.txt
# missing: fifth_subject.txt, fifth_message.html
with self.assertRaisesMessage(TemplateDoesNotExist, u'fifth_subject.txt'):
msg = render_mail(u'fifth')
def test_subject_squeezed(self):
u"""
Checks that even if the subject template contains leading, trailing or consecutive
whitespace, tabs or linebreaks, the rendered subject is normalized.
"""
# Ensure "first_subject.txt" contains leading/trailing/consecutive whitespace
rendered = render_to_string(u'first_subject.txt')
self.assertNotRegexpMatches(rendered, r'^(\S+ )*\S+$')
msg = render_mail(u'first')
self.assertRegexpMatches(msg.subject, r'^(\S+ )*\S+$')
def test_body_stripped(self):
u"""
Checks that even if the message template contains leading or trailing whitespace, the
rendered message is stripped.
"""
# Ensure both "first_message.txt" and "first_message.html" contain leading/trailing whitespace
rendered_txt = render_to_string(u'first_message.txt')
rendered_html = render_to_string(u'first_message.html')
self.assertNotEqual(rendered_txt, rendered_txt.strip())
self.assertNotEqual(rendered_html, rendered_html.strip())
msg = render_mail(u'first')
alternatives = self._get_alternatives(msg)
self.assertEqual(alternatives[0][1], alternatives[0][1].strip())
self.assertEqual(alternatives[1][1], alternatives[1][1].strip())
def test_message_arguments(self):
u"""
Tests arguments passed directly to the message constructor. Checks for ``from_email``,
``to``, ``cc``, ``bcc``, ``headers`` and ``attachments`` arguments. Tests them with all
three message forms: text only, html only and with both text and html parts.
"""
kwargs = {
u'from_email': u'*****@*****.**',
u'to': [u'*****@*****.**', u'*****@*****.**'],
u'cc': [u'John Shmith <*****@*****.**>'],
u'bcc': [u'*****@*****.**'],
u'headers': {'Reply-To': '*****@*****.**'},
u'attachments': [(u'filename', u'content', u'application/octet-stream')],
}
for template in [u'first', u'second', u'third']:
msg = render_mail(template, **kwargs)
self.assertEqual(msg.from_email, kwargs[u'from_email'])
self.assertEqual(msg.to, kwargs[u'to'])
self.assertEqual(msg.cc, kwargs[u'cc'])
self.assertEqual(msg.bcc, kwargs[u'bcc'])
self.assertEqual(msg.extra_headers, kwargs[u'headers'])
self.assertEqual(msg.attachments, kwargs[u'attachments'])
def test_render_mail_with_dictionary(self):
u"""
Passes a dictionary to ``render_mail()`` function and checks if templates using it are
rendered corectly.
"""
msg = render_mail(u'sixth', dictionary={u'variable': 47})
alternatives = self._get_alternatives(msg)
self.assertEqual(msg.subject, u'[example.com] Subject with dictionary: 47')
self.assertEqual(alternatives, [
(u'text/plain', u'Text message with dictionary: 47'),
(u'text/html', u'<p>HTML message with dictionary: 47</p>'),
])
