def test_create_twofiles_with_formats(self): file_path_json = dm.outputs.testfile(extension='json').__fspath__() self.assertEqual(os.path.split(file_path_json)[1], 'testfile.json') file_path = dm.outputs.testfile.__fspath__() self.assertEqual(os.path.split(file_path)[1], 'testfile') datasets = [ f for f in DataSet.select().where( DataSet.name == 'testfile').order_by(DataSet.id) ] self.assertEqual(len(datasets), 2) dataset1 = datasets[0] self.assertEqual(dataset1.name, "testfile") self.assertEqual(dataset1.project, "") self.assertEqual(dataset1.branch.name, "master") self.assertEqual( os.path.split(dataset1.get_fact('localpath'))[1], 'testfile.json') self.assertTrue( dataset1.get_fact('calling_filename').endswith( 'outputer_tests.py')) self.assertIsNotNone(dataset1.get_fact('metaargfilename')) dataset2 = datasets[1] self.assertEqual(dataset2.name, "testfile") self.assertEqual(dataset2.project, "") self.assertEqual(dataset2.branch.name, "master") self.assertEqual( os.path.split(dataset2.get_fact('localpath'))[1], 'testfile') self.assertTrue( dataset2.get_fact('calling_filename').endswith( 'outputer_tests.py')) self.assertIsNone( dataset2.get_fact('metaargfilename') ) # is None because we didn't declare metaargs or format
def list_datasets(): """ List datasets that exist locally """ datasets = DataSet.select().order_by(DataSet.is_default.desc()) dataset_grouping = {} for dataset in datasets: ds_key = dataset.full_name project = dataset.project if project not in dataset_grouping: dataset_grouping[project] = {} if ds_key not in dataset_grouping[project]: dataset_grouping[project][ds_key] = [] dataset_grouping[project][ds_key].append(dataset) print(_datasets_to_string(dataset_grouping))
def test_is_default_on_three_files(self): # Set is_default on separate files dm.outputs.f1(meta={'a': 1}).__fspath__() dm.outputs.f2(meta={'a': 1}).__fspath__() dm.outputs.f1(meta={'a': 2}).__fspath__() datasets = list(DataSet.select()) self.assertFalse(datasets[0].is_default) self.assertTrue(datasets[1].is_default) self.assertTrue(datasets[2].is_default) # Getting dm1 should return second copy. dm1_output = dm.inputs.f1 self.assertEqual('{"a": 2}', dm1_output._dataset.get_metaargs_str())
def test_create_file_with_timefield_extra_slash(self): file_path = dm.outputs.testfile(timepath='2019/11/03/').__fspath__() file_path = file_path.replace('\\', '/') self.assertTrue(file_path.endswith('testfile/2019/11/03/testfile')) self.state_check() dataset = DataSet.get() self.assertEqual(dataset.name, "testfile") self.assertEqual(dataset.project, "") self.assertEqual(dataset.branch.name, "master") self.assertEqual( dataset.get_fact('localpath').replace('\\', '/')[-20:], '/2019/11/03/testfile') self.assertTrue( dataset.get_fact('calling_filename').endswith('outputer_tests.py')) self.assertIsNone(dataset.get_fact('metaargfilename'))
def test_create_file_with_extension(self): file_path = dm.outputs.testfile(extension='json').__fspath__() self.assertEqual(os.path.split(file_path)[1], 'testfile.json') self.state_check() dataset = DataSet.get() self.assertEqual(dataset.name, "testfile") self.assertEqual(dataset.project, "") self.assertEqual(dataset.branch.name, "master") self.assertEqual( os.path.split(dataset.get_fact('localpath'))[1], 'testfile.json') self.assertTrue( dataset.get_fact('calling_filename').endswith('outputer_tests.py')) self.assertIsNotNone(dataset.get_fact('metaargfilename')) self.assertEqual(dataset.get_fact(DataSetFactKeys.FileExtension), 'json')
def test_save_to_different_branch(self): original_branch = Branch.get() f1 = dm.outputs.file.__fspath__() set_branch("test") f2 = dm.outputs.file.__fspath__() self.assertNotEqual(f1, f2) datasets = [f for f in DataSet.select().order_by(DataSet.id)] ds1 = datasets[0] ds2 = datasets[1] self.assertEqual(ds1.branch, original_branch) self.assertEqual(ds2.branch.name, dm.settings.active_branch) # Both are default in their branch self.assertTrue(ds1.is_default) self.assertTrue(ds2.is_default)
def test_create_file(self): file_path = dm.outputs.testfile.__fspath__() self.assertEqual(os.path.split(file_path)[1], 'testfile') self.state_check() dataset = DataSet.get() self.assertEqual(dataset.name, "testfile") self.assertEqual(dataset.project, "") self.assertEqual(dataset.branch.name, "master") self.assertEqual( os.path.split(dataset.get_fact('localpath'))[1], 'testfile') self.assertTrue( dataset.get_fact('calling_filename').endswith('outputer_tests.py')) self.assertIsNone( dataset.get_fact('metaargfilename') ) # is None because we didn't declare metaargs or format self.assertEqual(0, dataset.last_server_version)
def state_check(self): """ Utility to check that the model state is correct """ self.assertEqual(DataSet.select().count(), 1)