def test_urlopen_lines(self, urlopen_mock):
urlopen_mock.return_value = DATA_FP
lines = [line for line in util.urlopen_lines("http://none")]
h.assert_equal(lines,
["line one\n", "line two\n", "line three"])
def command(self):
super(CSVImportCommand, self).command()
self._check_args_length(1)
def json_of_url(url):
return json.load(urllib2.urlopen(url))
csv_data_url = self.args.pop(0)
have_model = self.options.model or (self.options.mapping and self.options.metadata)
if not have_model:
print("You must provide --model OR (--mapping AND --metadata)!",
file=sys.stderr)
return 1
if self.options.model:
model = json_of_url(self.options.model)
else:
model = {}
from openspending.ui.lib.mappingimporter import MappingImporter
mi = MappingImporter()
model["mapping"] = mi.import_from_url(self.options.mapping)
model["dataset"] = json_of_url(self.options.metadata)
csv = util.urlopen_lines(csv_data_url)
importer = CSVImporter(csv, model, csv_data_url)
try:
importer.run(**self.get_args())
return 0
except ImporterError as e:
log.error(e)
return 1
def __init__(self, package,
model_url=None, resource_uuid=None):
if not isinstance(package, Package):
package = Package(package)
if resource_uuid:
data = package.get_resource(resource_uuid)
else:
data = package.openspending_resource('data')
if not model_url:
# Use magic CKAN tags
model = package.openspending_resource('model')
model_url = model['url']
model_fp = util.urlopen(model_url)
try:
model = json.load(model_fp)
except Exception as e:
raise ImporterError("Error encountered while parsing JSON model. "
"http://jsonlint.com might help! Error was: %s"
% e)
csv = util.urlopen_lines(data["url"])
super(CKANImporter, self).__init__(csv, model, data["url"])
def test_urlopen_lines(urlopen_mock):
urlopen_mock.return_value = StringIO("line one\nline two\r\nline three")
lines = [line for line in util.urlopen_lines("http://none")]
h.assert_equal(lines,
["line one\n", "line two\n", "line three"])
def csv_import(resource_url, model_url, **kwargs):
import urllib
from openspending.lib import json
from openspending.etl import util
from openspending.etl.importer import CSVImporter
model = json.load(urllib.urlopen(model_url))
csv = util.urlopen_lines(resource_url)
importer = CSVImporter(csv, model, resource_url)
importer.run(**kwargs)
def csvimport(csv_data_url, args):
def json_of_url(url):
return json.load(urllib2.urlopen(url))
if args.model:
model = json_of_url(args.model)
else:
print("You must provide --model!",
file=sys.stderr)
return 1
csv = util.urlopen_lines(csv_data_url)
importer = CSVImporter(csv, model, csv_data_url)
importer.run(**vars(args))
return 0
def __init__(self, package,
model_url=None, mapping_url=None, resource_uuid=None):
if not isinstance(package, ckan.Package):
package = ckan.Package(package)
if resource_uuid:
data = package.get_resource(resource_uuid)
else:
data = package.openspending_resource('data')
explicit = (model_url and not mapping_url) or (mapping_url and not model_url)
if not explicit:
# Use magic CKAN tags
model = package.openspending_resource('model')
mapping = package.openspending_resource('model:mapping')
if model:
model_url = model['url']
elif mapping:
mapping_url = mapping['url']
# Model given
if model_url and data:
model = json.load(util.urlopen(model_url))
# Mapping given, need to extract metadata from CKAN
elif mapping_url and data:
model = {}
model['mapping'] = MappingImporter().import_from_url(mapping_url)
model['dataset'] = package.metadata_for_resource(data)
csv = util.urlopen_lines(data["url"])
super(CKANImporter, self).__init__(csv, model, data["url"])