def test_iterator_with_schema(self, csv_tmpfile):
csv_contents = (
'superhero,awesomeness\n'
'Captain America,8\n'
'Iron Man,10\n'
'Ant Man,5\n'
'SuperMan,7\n'
).encode('utf-8')
csv_tmpfile.write(csv_contents)
csv_tmpfile.flush()
resource = TabularResource(
{'path': csv_tmpfile.name,
'schema': {
'fields':[
{
'name': 'superhero',
'type': 'string'
},
{
'name': 'awesomeness',
'type': 'integer'
}
]
}
})
data = [row for row in resource.iter()]
assert data == [
{'superhero': 'Captain America', 'awesomeness': 8},
{'superhero': 'Iron Man', 'awesomeness': 10},
{'superhero': 'Ant Man', 'awesomeness': 5},
{'superhero': 'SuperMan', 'awesomeness': 7},
]
def test_iterator_with_schema(self, csv_tmpfile):
csv_contents = (
'superhero,awesomeness\n'
'Captain America,8\n'
'Iron Man,10\n'
'Ant Man,5\n'
'SuperMan,7\n'
).encode('utf-8')
csv_tmpfile.write(csv_contents)
csv_tmpfile.flush()
resource = TabularResource(
{'path': csv_tmpfile.name,
'schema': {
'fields':[
{
'name': 'superhero',
'type': 'string'
},
{
'name': 'awesomeness',
'type': 'integer'
}
]
}
})
data = [row for row in resource.iter()]
assert data == [
{'superhero': 'Captain America', 'awesomeness': 8},
{'superhero': 'Iron Man', 'awesomeness': 10},
{'superhero': 'Ant Man', 'awesomeness': 5},
{'superhero': 'SuperMan', 'awesomeness': 7},
]
def test_iterator_with_local_non_tabular_data(self):
with tempfile.NamedTemporaryFile(suffix='.txt') as tmpfile:
tmpfile.write('foo'.encode('utf-8'))
tmpfile.flush()
resource = TabularResource({'path': tmpfile.name})
with pytest.raises(ValueError):
[row for row in resource.iter()]
def test_iterator_with_inline_data(self):
data = [
{'country': 'China', 'value': '中国'},
{'country': 'Brazil', 'value': 'Brasil'},
]
resource = TabularResource({'data': data})
assert [row for row in resource.iter()] == data
def test_iterator_with_inline_data(self):
data = [
{'country': 'China', 'value': '中国'},
{'country': 'Brazil', 'value': 'Brasil'},
]
resource = TabularResource({'data': data})
assert [row for row in resource.iter()] == data
def test_iterator_with_inline_data(self):
contents = (
'first line\n'
'second line\n'
)
resource = datapackage.Resource.load({'data': contents})
data = [row for row in resource.iter()]
assert data == [b'first line\n', b'second line\n']
def test_iterator_with_local_data(self, txt_tmpfile):
contents = ('first line\n' 'second line\n')
txt_tmpfile.write(contents.encode('utf-8'))
txt_tmpfile.flush()
resource = datapackage.Resource.load({'path': txt_tmpfile.name})
data = [row for row in resource.iter()]
assert data == [b'first line\n', b'second line\n']
def test_iterator_with_inline_data(self):
contents = (
'first line\n'
'second line\n'
)
resource = datapackage.Resource.load({'data': contents})
data = [row for row in resource.iter()]
assert data == [b'first line\n', b'second line\n']
def test_iterator_with_remote_non_tabular_data(self):
httpretty.HTTPretty.allow_net_connect = False
resource_dict = {
'url': 'http://someplace.com/data.txt',
}
httpretty.register_uri(httpretty.GET, resource_dict['url'], body='foo')
resource = TabularResource(resource_dict)
with pytest.raises(ValueError):
[row for row in resource.iter()]
def test_iterator_with_remote_non_tabular_data(self):
httpretty.HTTPretty.allow_net_connect = False
resource_dict = {
'url': 'http://someplace.com/data.txt',
}
httpretty.register_uri(httpretty.GET, resource_dict['url'],
body='foo')
resource = TabularResource(resource_dict)
with pytest.raises(ValueError):
[row for row in resource.iter()]
def test_iterator_with_local_data(self, txt_tmpfile):
contents = (
'first line\n'
'second line\n'
)
txt_tmpfile.write(contents.encode('utf-8'))
txt_tmpfile.flush()
resource = datapackage.Resource.load({'path': txt_tmpfile.name})
data = [row for row in resource.iter()]
assert data == [b'first line\n', b'second line\n']
def test_raises_with_wrong_encoding(self, csv_tmpfile):
csv_contents = (
'country,value\n'
'China,中国\n'
'Brazil,Brasil\n'
).encode('utf-8')
csv_tmpfile.write(csv_contents)
csv_tmpfile.flush()
resource = TabularResource({'path': csv_tmpfile.name, 'encoding': 'utf-16'})
with pytest.raises(ValueError):
data = [row for row in resource.iter()]
def test_raises_with_wrong_encoding(self, csv_tmpfile):
csv_contents = (
'country,value\n'
'China,中国\n'
'Brazil,Brasil\n'
).encode('utf-8')
csv_tmpfile.write(csv_contents)
csv_tmpfile.flush()
resource = TabularResource({'path': csv_tmpfile.name, 'encoding': 'utf-16'})
with pytest.raises(ValueError):
data = [row for row in resource.iter()]
def test_iterator_with_local_data(self):
contents = (
'first line\n'
'second line\n'
)
with tempfile.NamedTemporaryFile(suffix='.txt') as tmpfile:
tmpfile.write(contents.encode('utf-8'))
tmpfile.flush()
resource = datapackage.Resource.load({'path': tmpfile.name})
data = [row for row in resource.iter()]
assert data == [b'first line\n', b'second line\n']
def test_iterator_with_inline_data(self):
data = (
'['
'{"country": "China", "value": "中国"},'
'{"country": "Brazil", "value": "Brasil"}'
']'
)
resource = TabularResource({'data': data})
assert [row for row in resource.iter()] == [
{'country': 'China', 'value': '中国'},
{'country': 'Brazil', 'value': 'Brasil'},
]
def test_iterator_with_remote_data(self):
httpretty.HTTPretty.allow_net_connect = False
contents = ('first line\n' 'second line\n')
resource_dict = {
'url': 'http://someplace.com/data.txt',
}
httpretty.register_uri(httpretty.GET,
resource_dict['url'],
body=contents)
resource = datapackage.Resource.load(resource_dict)
data = [row for row in resource.iter()]
assert data == [b'first line\n', b'second line\n']
def test_iterator_with_local_data(self, csv_tmpfile):
csv_contents = (
'country,value\n'
'China,中国\n'
'Brazil,Brasil\n'
).encode('utf-8')
csv_tmpfile.write(csv_contents)
csv_tmpfile.flush()
resource = TabularResource({'path': csv_tmpfile.name})
data = [row for row in resource.iter()]
assert data == [
{'country': 'China', 'value': '中国'},
{'country': 'Brazil', 'value': 'Brasil'},
]
def test_iterator_with_local_data(self, csv_tmpfile):
csv_contents = (
'country,value\n'
'China,中国\n'
'Brazil,Brasil\n'
).encode('utf-8')
csv_tmpfile.write(csv_contents)
csv_tmpfile.flush()
resource = TabularResource({'path': csv_tmpfile.name, 'encoding': 'utf-8'})
data = [row for row in resource.iter()]
assert data == [
{'country': 'China', 'value': '中国'},
{'country': 'Brazil', 'value': 'Brasil'},
]
def test_iterator_with_remote_data(self):
httpretty.HTTPretty.allow_net_connect = False
contents = (
'first line\n'
'second line\n'
)
resource_dict = {
'url': 'http://someplace.com/data.txt',
}
httpretty.register_uri(httpretty.GET, resource_dict['url'],
body=contents)
resource = datapackage.Resource.load(resource_dict)
data = [row for row in resource.iter()]
assert data == [b'first line\n', b'second line\n']
def test_iterator_with_local_data(self):
csv_contents = (
'country,value\n'
'China,中国\n'
'Brazil,Brasil\n'
).encode('utf-8')
with tempfile.NamedTemporaryFile(suffix='.csv') as tmpfile:
tmpfile.write(csv_contents)
tmpfile.flush()
resource = TabularResource({'path': tmpfile.name})
data = [row for row in resource.iter()]
assert data == [
{'country': 'China', 'value': '中国'},
{'country': 'Brazil', 'value': 'Brasil'},
]
def test_iterator_with_remote_data(self):
httpretty.HTTPretty.allow_net_connect = False
csv_contents = (
'country,value\n'
'China,中国\n'
'Brazil,Brasil\n'
).encode('utf-8')
resource_dict = {
'url': 'http://someplace.com/data.csv',
}
httpretty.register_uri(httpretty.GET, resource_dict['url'],
body=csv_contents)
resource = TabularResource(resource_dict)
assert [row for row in resource.iter()] == [
{'country': 'China', 'value': '中国'},
{'country': 'Brazil', 'value': 'Brasil'},
]
def test_iterator_with_remote_data(self):
httpretty.HTTPretty.allow_net_connect = False
csv_contents = (
'country,value\n'
'China,中国\n'
'Brazil,Brasil\n'
).encode('utf-8')
resource_dict = {
'url': 'http://someplace.com/data.csv',
}
httpretty.register_uri(httpretty.GET, resource_dict['url'],
body=csv_contents)
resource = TabularResource(resource_dict)
assert [row for row in resource.iter()] == [
{'country': 'China', 'value': '中国'},
{'country': 'Brazil', 'value': 'Brasil'},
]
def test_iterator_with_inline_numerical_data(self):
contents = 51
resource = datapackage.Resource.load({'data': contents})
assert [row for row in resource.iter()] == [51]
def test_iterator_raises_resourceerror_if_file_doesnt_exist(self):
resource = TabularResource({'path': 'inexistent-file.csv'})
with pytest.raises(datapackage.exceptions.ResourceError):
[row for row in resource.iter()]
def test_iterator_raises_if_file_doesnt_exist(self):
resource = datapackage.Resource.load({'path': 'inexistent-file.txt'})
with pytest.raises(IOError):
[row for row in resource.iter()]
def test_iterator_raises_valueerror_if_theres_no_data(self):
resource = datapackage.Resource.load({})
with pytest.raises(ValueError):
[row for row in resource.iter()]
def test_iterator_raises_if_file_doesnt_exist(self):
resource = datapackage.Resource.load({'path': 'inexistent-file.txt'})
with pytest.raises(IOError):
[row for row in resource.iter()]
def test_iterator_raises_resourceerror_if_url_doesnt_exist(self):
url = 'http://someplace.com/inexistent-file.txt'
httpretty.register_uri(httpretty.GET, url, status=404)
resource = datapackage.Resource.load({'url': url})
with pytest.raises(IOError):
[row for row in resource.iter()]
def test_iterator_raises_resourceerror_if_url_doesnt_exist(self):
url = 'http://someplace.com/inexistent-file.txt'
httpretty.register_uri(httpretty.GET, url, status=404)
resource = datapackage.Resource.load({'url': url})
with pytest.raises(IOError):
[row for row in resource.iter()]
def test_iterator_raises_valueerror_if_theres_no_data(self):
resource = datapackage.Resource.load({})
with pytest.raises(ValueError):
[row for row in resource.iter()]
def test_iterator_with_inline_non_tabular_data(self):
resource = TabularResource({'data': 'foo'})
with pytest.raises(ValueError):
[row for row in resource.iter()]
def test_iterator_with_inline_numerical_data(self):
contents = 51
resource = datapackage.Resource.load({'data': contents})
assert [row for row in resource.iter()] == [51]
def test_iterator_raises_if_file_doesnt_exist(self):
resource = TabularResource({'path': 'inexistent-file.csv'})
with pytest.raises(IOError):
[row for row in resource.iter()]
def test_iterator_raises_if_url_doesnt_exist(self):
url = 'http://someplace.com/inexistent-file.csv'
httpretty.register_uri(httpretty.GET, url, status=404)
resource = TabularResource({'url': url})
with pytest.raises(ValueError):
[row for row in resource.iter()]
def test_iterator_raises_if_file_doesnt_exist(self):
resource = TabularResource({'path': 'inexistent-file.csv'})
with pytest.raises(IOError):
[row for row in resource.iter()]
def test_iterator_with_local_non_tabular_data(self, txt_tmpfile):
txt_tmpfile.write('foo'.encode('utf-8'))
txt_tmpfile.flush()
resource = TabularResource({'path': txt_tmpfile.name})
with pytest.raises(ValueError):
[row for row in resource.iter()]
def test_iterator_with_inline_non_tabular_data(self):
resource = TabularResource({'data': 'foo'})
with pytest.raises(ValueError):
[row for row in resource.iter()]
def test_iterator_raises_valueerror_if_theres_no_data(self):
resource = TabularResource({})
with pytest.raises(ValueError):
[row for row in resource.iter()]
def test_iterator_with_local_non_tabular_data(self, txt_tmpfile):
txt_tmpfile.write('foo'.encode('utf-8'))
txt_tmpfile.flush()
resource = TabularResource({'path': txt_tmpfile.name})
with pytest.raises(ValueError):
[row for row in resource.iter()]
def test_iterator_raises_if_url_doesnt_exist(self):
url = 'http://someplace.com/inexistent-file.csv'
httpretty.register_uri(httpretty.GET, url, status=404)
resource = TabularResource({'url': url})
with pytest.raises(IOError):
[row for row in resource.iter()]
def test_iterator_raises_valueerror_if_theres_no_data(self):
resource = TabularResource({})
with pytest.raises(ValueError):
[row for row in resource.iter()]