def test_missing_url(self):
dataset, url = import_fixture("file:///dev/null")
source = tasks.extract_url(dataset, url)
assert source is None, source
for run in db.session.query(Run).all():
assert run.status == Run.STATUS_FAILED, run
def test_missing_url(self):
dataset, url = import_fixture('file:///dev/null')
source = tasks.extract_url(dataset, url)
assert source is None, source
for run in db.session.query(Run).all():
assert run.status == Run.STATUS_FAILED, run
def test_extract_missing_url(self):
url = csvimport_fixture_path('../data', 'xcra.csv')
source = tasks.extract_url(self.ds, url)
assert source is None, source
run = db.session.query(Run).first()
package = data_manager.package(self.ds.name)
messages = list(logger.load(package, run.id))
assert len(messages) > 2, messages
def load_from_url(dataset_name, url):
with flask_app.app_context():
dataset = Dataset.by_name(dataset_name)
if dataset is None:
log.error("Dataset not found: %s", dataset_name)
return
source = tasks.extract_url(dataset, url)
if source is not None:
load_from_source.delay(dataset_name, source.name)
def load_from_url(dataset_name, url):
with flask_app.app_context():
dataset = Dataset.by_name(dataset_name)
if dataset is None:
log.error("Dataset not found: %s", dataset_name)
return
source = tasks.extract_url(dataset, url)
if source is not None:
load_from_source.delay(dataset_name, source.name)
def _test_import(self, name, lines=None):
dataset, url = import_fixture(name)
data = urllib.urlopen(url)
if lines is None:
lines = self.count_lines_in_stream(data) - 1 # -1 for header row
source = tasks.extract_url(dataset, url)
tasks.transform_source(dataset, source.name)
tasks.load(dataset, source_name=source.name)
for run in db.session.query(Run).all():
assert run.status == Run.STATUS_COMPLETE, run
# check correct number of entries
dataset = db.session.query(Dataset).first()
entries = list(dataset.fact_table.entries())
assert len(entries) == lines, len(entries)
def _test_import(self, name, lines=None):
dataset, url = import_fixture(name)
data = urllib.urlopen(url)
if lines is None:
lines = self.count_lines_in_stream(data) - 1 # -1 for header row
source = tasks.extract_url(dataset, url)
tasks.transform_source(dataset, source.name)
tasks.load(dataset, source_name=source.name)
for run in db.session.query(Run).all():
assert run.status == Run.STATUS_COMPLETE, run
# check correct number of entries
dataset = db.session.query(Dataset).first()
entries = list(dataset.fact_table.entries())
assert len(entries) == lines, len(entries)
def test_extract_url(self):
source = tasks.extract_url(self.ds, self.cra_url)
assert 'cra.csv' == source.name, source.name
def load_from_url(dataset_name, url):
with flask_app.app_context():
dataset = Dataset.by_name(dataset_name)
source = tasks.extract_url(dataset, url)
if source is not None:
load_from_source.delay(dataset_name, source.name)