def test_misconfigured_pipeline(self):
with self.assertRaises(RuntimeError):
pl.Pipeline('test', 'Test', log_status=False) \
.run()
with self.assertRaises(RuntimeError):
pl.Pipeline('test', 'Test', log_status=False) \
.extract(pl.CSVExtractor, None).run()
with self.assertRaises(RuntimeError):
pl.Pipeline('test', 'Test', log_status=False) \
.extract(pl.CSVExtractor, None).run()
with self.assertRaises(RuntimeError):
pl.Pipeline('test', 'Test', log_status=False) \
.extract(pl.CSVExtractor, None).schema(pl.BaseSchema).run()
with self.assertRaises(RuntimeError):
pl.Pipeline('test', 'Test', log_status=False) \
.schema(pl.BaseSchema).load(TestLoader).run()
def setUp(self):
self.pipeline = pl.Pipeline('test',
'Test',
settings_file=os.path.join(
HERE,
'../mock/first_test_settings.json'),
log_status=False)
def setUp(self):
self.pipeline = pl.Pipeline(
'test', 'Test',
log_status=False
)
settings_file = os.path.join(HERE, '../mock/first_test_settings.json')
with open(settings_file) as f:
self.ckan_config = json.load(f)['loader']['ckan']
pass
def test_checksum_duplicate_prevention(self):
pipeline = pl.Pipeline(
'fatal_od_pipeline', 'Fatal OD Pipeline',
settings_file=self.settings_file,
log_status=True, conn=self.conn
) \
.connect(pl.FileConnector, os.path.join(HERE, '../mock/simple_mock.csv')) \
.extract(pl.CSVExtractor, firstline_headers=True) \
.schema(TestSchema) \
.load(self.Loader)
pipeline.run()
status = self.cur.execute('select * from status').fetchall()
self.assertEquals(len(status), 1)
with self.assertRaises(pl.DuplicateFileException):
pipeline.run()
status = self.cur.execute('select * from status').fetchall()
self.assertEquals(len(status), 1)
def test_no_config(self):
with self.assertRaises(pl.InvalidConfigException):
pl.Pipeline('test',
'Test',
settings_file=os.path.join(HERE, 'NOT-A-VALID-PATH'))
def test_invalid_config(self):
with self.assertRaises(pl.InvalidConfigException):
pl.Pipeline('test',
'Test',
settings_file=os.path.join(HERE,
'first_test_settings.json'))
with self.runner.isolated_filesystem():
result = self.runner.invoke(create_db, ['--db', './status.db'])
self.assertEquals(result.exit_code, 0)
def test_create_from_default_path(self):
with self.runner.isolated_filesystem():
result = self.runner.invoke(create_db, ['--db', ''])
self.assertEquals(result.exit_code, 0)
test_pipeline = pl.Pipeline(
'test', 'Test',
settings_file=SETTINGS_FILE,
log_status=False
).connect(TestConnector, None) \
.extract(TestExtractor) \
.schema(pl.BaseSchema) \
.load(TestLoader)
not_working = pl.Pipeline('nope', 'It does not work')
class Junk(object):
pass
junk = Junk()
class TestRunJobScript(TestCase):