def setUp(self):
self.view = CSVJinjaView(
env_options={'loader': FunctionLoader(lambda x: x)})
self.data = [
['1', 'hi', 'yes', '2017-07-19', '3.5'],
['2', 'bye', 'no', '2017-07-18', '3.6'],
['3', 'heh', 'y', '2017-07-20', '3.7'],
]
self.model = CSVModel(self.data)
def test_from_file(self):
with tempfile.NamedTemporaryFile(mode='w') as f:
writer = csv.writer(f)
writer.writerows(self.data)
f.flush()
file_model = CSVModel.from_file(f.name)
self.assertCSVModelsAreEqual(file_model, self.model)
def __init__(self,
experiment_root_path,
experiment_tag='experiment',
log_to_file=True,
sub_experiment_dirs=True):
"""Initialize an ExperimentLogger.
Parameters
----------
experiment_root_path : :obj:`str`
The root directory in which to save experiment files.
experiment_tag : :obj:`str`
The tag to use when prefixing new experiments
log_to_file : bool, optional
Default: True
If True will log all logging statements to a log file
sub_experiment_dirs : bool, optional
Defautl: True
If True will make sub directories corresponding to generated experiment name
"""
self.experiment_root_path = experiment_root_path
# open the master record
self.master_record_filepath = os.path.join(
self.experiment_root_path,
ExperimentLogger._MASTER_RECORD_FILENAME)
self.master_record = CSVModel.get_or_create(
self.master_record_filepath, self.experiment_meta_headers)
# add new experiment to the master record
self.id = ExperimentLogger.gen_experiment_ref(experiment_tag)
self._master_record_uid = self.master_record.insert(
self.experiment_meta_data)
# make experiment output dir
if sub_experiment_dirs:
self.experiment_path = os.path.join(self.experiment_root_path,
self.id)
if not os.path.exists(self.experiment_path):
os.makedirs(self.experiment_path)
else:
self.experiment_path = self.experiment_root_path
if log_to_file:
# redirect logging statements to a file
if not sub_experiment_dirs:
self.log_path = os.path.join(self.experiment_root_path, 'logs')
else:
self.log_path = self.experiment_path
if not os.path.exists(self.log_path):
os.makedirs(self.log_path)
experiment_log = os.path.join(self.log_path, '%s.log' % (self.id))
formatter = logging.Formatter(
'%(asctime)s %(levelname)s: %(message)s')
hdlr = logging.FileHandler(experiment_log)
hdlr.setFormatter(formatter)
logging.getLogger().addHandler(hdlr)
# internal dir struct
self._dirs = {}
def test_from_file_with_types(self):
types = [str, float, int, bool, str, cast_to_bool]
expected_results = [
['Hello', 3.6, 1, False, '99', True],
['Bye', 9.0, 0, True, '9.5', False],
['s', 3.14, 55, True, 'false', True],
['eee', 4.0, 88, True, 'yes', False],
]
with tempfile.NamedTemporaryFile(mode='w') as f:
writer = csv.writer(f)
writer.writerows(self.data)
f.flush()
file_model = CSVModel.from_file(f.name, types=types)
self.assertCSVModelsAreEqual(file_model, expected_results)
class TestViewFilters(unittest.TestCase):
def setUp(self):
self.view = CSVJinjaView(
env_options={'loader': FunctionLoader(lambda x: x)})
self.data = [
['1', 'hi', 'yes', '2017-07-19', '3.5'],
['2', 'bye', 'no', '2017-07-18', '3.6'],
['3', 'heh', 'y', '2017-07-20', '3.7'],
]
self.model = CSVModel(self.data)
def test_bool(self):
data = ['yes', 'no', 'True', 'y', 'false', 'N', 'TrUE']
expected = ['True', 'False', 'True', 'True', 'False', 'False', 'True']
self.assertEqual(
len(data),
len(expected),
msg='# of test cases should match # of expected outcomes')
for test_data, val in zip(data, expected):
template = '{{ ' + repr(test_data) + ' | bool }}'
self.assertEqual(val,
self.view.render_jinja_template(template, None))
def test_date(self):
today = datetime.now()
data = ['2017-07-02', '02/29/2008', '10:15:45.31 AM', '05/11/96']
expected = [
datetime(2017, 7, 2),
datetime(2008, 2, 29),
datetime(today.year, today.month, today.day, 10, 15, 45, 310000),
datetime(1996, 5, 11)
]
self.assertEqual(
len(data),
len(expected),
msg='# of test cases should match # of expected outcomes')
for test_data, val in zip(data, expected):
template = '{{ ' + repr(test_data) + ' | date }}'
self.assertEqual(str(val),
self.view.render_jinja_template(template, None))
def test_date_from_timestamp(self):
data = ['1000', '0', '1', '1931516412']
for test_data in data:
template = '{{ ' + str(test_data) + ' | int | date }}'
expected = str(datetime.fromtimestamp(int(test_data)))
self.assertEqual(expected,
self.view.render_jinja_template(template, None))
def test_dateformat(self):
date = '2017-07-18'
formats = ['%d/%m/%y', '%Y-%d-%m', '%y']
expected = ['18/07/17', '2017-18-07', '17']
self.assertEqual(
len(formats),
len(expected),
msg='# of test cases should match # of expected outcomes')
for fmt, val in zip(formats, expected):
template = '{{ ' + repr(date) + ' | date | dateformat(' + repr(
fmt) + ') }}'
self.assertEqual(val,
self.view.render_jinja_template(template, None))
def test_cast(self):
template = '{{ rows | cast(["date", None, "int", "date", "bool"]) }}'
expected = str(
self.model.cast([cast_to_date, str, int, cast_to_date, bool]))
self.assertEqual(expected,
self.view.render_jinja_template(template, self.model))
def test_castrange(self):
template = '{{ rows | castrange(["float", None, "int"], 0, 3) }}'
expected = str(self.model.cast_range([float, str, int], 0, 3))
self.assertEqual(expected,
self.view.render_jinja_template(template, self.model))
def test_rowrange(self):
template = '{{ rows | rowrange(1) }}'
expected = str(self.model.row_slice(1))
self.assertEqual(expected,
self.view.render_jinja_template(template, self.model))
def test_columnrange(self):
template = '{{ rows | columnrange(2, 3) }}'
expected = str(self.model.col_slice(2, 3))
self.assertEqual(expected,
self.view.render_jinja_template(template, self.model))
def setUp(self):
self.data = [['Hello', '3.6', '1', '', '99', 'True'],
['Bye', '9', '0', 'eh', '9.5', 'false'],
['s', '3.14', '55', 's', 'false', 'yes'],
['eee', '4', '88', 'f', 'yes', 'NO']]
self.model = CSVModel(self.data)
class TestCSVModel(unittest.TestCase):
def setUp(self):
self.data = [['Hello', '3.6', '1', '', '99', 'True'],
['Bye', '9', '0', 'eh', '9.5', 'false'],
['s', '3.14', '55', 's', 'false', 'yes'],
['eee', '4', '88', 'f', 'yes', 'NO']]
self.model = CSVModel(self.data)
def assertCSVModelsAreEqual(self, m1, m2, msg=None):
self.assertEqual(getValueTypeList(m1), getValueTypeList(m2), msg=msg)
def test_autocast(self):
expected_results = [['Hello', 3.6, 1, '', '99', True],
['Bye', 9.0, 0, 'eh', '9.5', False],
['s', 3.14, 55, 's', 'false', True],
['eee', 4.0, 88, 'f', 'yes', False]]
self.assertCSVModelsAreEqual(expected_results, self.model)
def test_from_file(self):
with tempfile.NamedTemporaryFile(mode='w') as f:
writer = csv.writer(f)
writer.writerows(self.data)
f.flush()
file_model = CSVModel.from_file(f.name)
self.assertCSVModelsAreEqual(file_model, self.model)
def test_from_file_with_types(self):
types = [str, float, int, bool, str, cast_to_bool]
expected_results = [
['Hello', 3.6, 1, False, '99', True],
['Bye', 9.0, 0, True, '9.5', False],
['s', 3.14, 55, True, 'false', True],
['eee', 4.0, 88, True, 'yes', False],
]
with tempfile.NamedTemporaryFile(mode='w') as f:
writer = csv.writer(f)
writer.writerows(self.data)
f.flush()
file_model = CSVModel.from_file(f.name, types=types)
self.assertCSVModelsAreEqual(file_model, expected_results)
def test_cast(self):
filters = [str, bool, float, len, lambda x: str(x)[0], int]
expected_results = [['Hello', True, 1.0, 0, '9', 1],
['Bye', True, 0.0, 2, '9', 0],
['s', True, 55.0, 1, 'f', 1],
['eee', True, 88.0, 1, 'y', 0]]
model = self.model.cast(filters)
self.assertCSVModelsAreEqual(expected_results, model)
def test_cast_range(self):
tests = [{
'filters': [int, float, len],
'start':
1,
'stop':
4,
'expected': [['Hello', 3, 1.0, 0, '99', True],
['Bye', 9, 0.0, 2, '9.5', False],
['s', 3, 55.0, 1, 'false', True],
['eee', 4, 88.0, 1, 'yes', False]]
}, {
'filters': [str] * 6,
'start':
None,
'stop':
None,
'expected': [['Hello', '3.6', '1', '', '99', 'True'],
['Bye', '9.0', '0', 'eh', '9.5', 'False'],
['s', '3.14', '55', 's', 'false', 'True'],
['eee', '4.0', '88', 'f', 'yes', 'False']]
}, {
'filters': [str] * 6,
'start':
0,
'stop':
None,
'expected': [['Hello', '3.6', '1', '', '99', 'True'],
['Bye', '9.0', '0', 'eh', '9.5', 'False'],
['s', '3.14', '55', 's', 'false', 'True'],
['eee', '4.0', '88', 'f', 'yes', 'False']]
}, {
'filters': [lambda x: x[1], int],
'start':
4,
'stop':
None,
'expected': [['Hello', 3.6, 1, '', '9', 1],
['Bye', 9.0, 0, 'eh', '.', 0],
['s', 3.14, 55, 's', 'a', 1],
['eee', 4.0, 88, 'f', 'e', 0]]
}, {
'filters': [lambda x: str(x)[-1], int],
'start':
None,
'stop':
2,
'expected': [['o', 3, 1, '', '99', True],
['e', 9, 0, 'eh', '9.5', False],
['s', 3, 55, 's', 'false', True],
['e', 4, 88, 'f', 'yes', False]]
}, {
'filters': [bool],
'start':
2,
'stop':
3,
'expected': [['Hello', 3.6, True, '', '99', True],
['Bye', 9.0, False, 'eh', '9.5', False],
['s', 3.14, True, 's', 'false', True],
['eee', 4.0, True, 'f', 'yes', False]]
}]
for idx, test in enumerate(tests):
casted_model = self.model.cast_range(test['filters'],
test['start'], test['stop'])
errMsg = 'test case #{}'.format(idx)
self.assertCSVModelsAreEqual(casted_model,
test['expected'],
msg=errMsg)
def test_row_slice(self):
expected_results = [['Bye', 9.0, 0, 'eh', '9.5', False],
['s', 3.14, 55, 's', 'false', True]]
self.assertCSVModelsAreEqual(self.model.row_slice(1, 3),
expected_results)
def test_col_slice(self):
expected_results = [[1, ''], [0, 'eh'], [55, 's'], [88, 'f']]
self.assertCSVModelsAreEqual(self.model.col_slice(2, 4),
expected_results)
def test_str(self):
expected = "('Hello', 3.6, 1, '', '99', True)\n" \
"('Bye', 9.0, 0, 'eh', '9.5', False)\n" \
"('s', 3.14, 55, 's', 'false', True)\n" \
"('eee', 4.0, 88, 'f', 'yes', False)"
self.assertEqual(expected, str(self.model))