def test_csv_file(self):
data = {'url': 'http://0.0.0.0:50001/static/simple.csv',
'format': 'csv',
'id': 'uuid1'}
context = {'site_url': 'http://%s' % self.host,
'site_user_apikey': self.api_key,
'apikey': self.api_key}
resource_id = self.make_resource_id()
data['id'] = resource_id
tasks.datastorer_upload(json.dumps(context), json.dumps(data))
response = requests.get(
'http://%s/api/action/datastore_search?resource_id=%s' % (self.host, resource_id),
headers={"content-type": "application/json"})
result = json.loads(response.content)
value = result['result']['records'][0][u'temperature']
assert int(value) == 1, value
assert result['result']['total'] == 6, (result['result']['total'], resource_id)
assert result['result']['fields'] == [{u'type': u'int4', u'id': u'_id'},
{u'type': u'timestamp', u'id': u'date'},
{u'type': u'numeric', u'id': u'temperature'},
{u'type': u'text', u'id': u'place'}], result['result']['fields']
def test_long_file(self):
data = {'url': 'http://0.0.0.0:50001/static/long.csv', 'format': 'csv'}
context = {
'site_url': 'http://%s' % self.host,
'apikey': self.api_key,
'site_user_apikey': self.api_key,
}
resource_id = self.make_resource_id()
data['id'] = resource_id
tasks.datastorer_upload(json.dumps(context), json.dumps(data))
response = requests.get(
'http://%s/api/action/datastore_search?resource_id=%s' %
(self.host, resource_id),
headers={"content-type": "application/json"})
result = json.loads(response.content)
assert result['result']['total'] == 4000, (result['result']['total'],
resource_id)
assert len(result['result']['records']) == 100
def test_messier_file(self):
data = {
'url': 'http://0.0.0.0:50001/static/3ffdcd42-5c63-4089-84dd-c23876259973.csv',
'format': 'csv'}
context = {'site_url': 'http://%s' % self.host,
'apikey': self.api_key,
'site_user_apikey': self.api_key,
}
resource_id = self.make_resource_id()
data['id'] = resource_id
tasks.datastorer_upload(json.dumps(context), json.dumps(data))
response = requests.get(
'http://%s/api/action/datastore_search?resource_id=%s' % (self.host, resource_id),
headers={"content-type": "application/json"})
result = json.loads(response.content)
value = result['result']['records'][0][u'Transaction Number']
assert int(value) == 136980, value
assert result['result']['total'] == 564, (result['result']['total'], resource_id)
assert len(result['result']['records']) == 100
assert result['result']['fields'] == [{u'type': u'int4', u'id': u'_id'},
{u'type': u'text', u'id': u'Body Name'},
{u'type': u'timestamp', u'id': u'Date'},
{u'type': u'numeric', u'id': u'Transaction Number'},
{u'type': u'numeric', u'id': u'Amount'},
{u'type': u'text', u'id': u'Supplier'},
{u'type': u'text', u'id': u'Expense Area'}], result['result']['fields']
def test_tsv_file_with_incorrect_mimetype(self):
'''Not all servers are well-behaved, and provide the wrong mime type.
Force the test server to provide the wrong Content-Type by changing
the filename to have a .txt extension. However, the owner of the
resource knows it's a tsv file, and can set the format directly.
'''
data = {'url': 'http://0.0.0.0:50001/static/tsv_as_txt.txt',
'format': 'tsv',
'id': 'uuid4'}
context = {'site_url': 'http://%s' % self.host,
'site_user_apikey': self.api_key,
'apikey': self.api_key}
resource_id = self.make_resource_id()
data['id'] = resource_id
tasks.datastorer_upload(json.dumps(context), json.dumps(data))
response = requests.get(
'http://%s/api/action/datastore_search?resource_id=%s' % (self.host, resource_id),
headers={"content-type": "application/json"})
result = json.loads(response.content)
assert result['result']['total'] == 6, (result['result']['total'], resource_id)
assert result['result']['fields'] == [{u'type': u'int4', u'id': u'_id'},
{u'type': u'timestamp', u'id': u'date'},
{u'type': u'numeric', u'id': u'temperature'},
{u'type': u'text', u'id': u'place'}], result['fields']
def test_csv_file(self):
data = {'url': 'http://0.0.0.0:50001/static/simple.csv',
'format': 'csv',
'id': 'uuid1'}
context = {'site_url': 'http://0.0.0.0:8088',
'apikey': 'moo',
'site_user_apikey': 'moo',
'username': '******'}
resource_id = self.make_resource_id()
data['id'] = resource_id
tasks.datastorer_upload(json.dumps(context), json.dumps(data))
import time; time.sleep(0.5)
response = requests.get(
'http://0.0.0.0:8088/api/data/%s/_search?q=*' % resource_id,
)
response = json.loads(response.content)
assert len(response['hits']['hits']) == 6, len(response['hits']['hits'])
def test_bus_stops(self):
data = {
'url': 'http://0.0.0.0:50001/static/bus-stops.csv',
'format': 'csv'}
context = {'site_url': 'http://%s' % self.host,
'apikey': self.api_key,
'site_user_apikey': self.api_key,
}
resource_id = self.make_resource_id()
data['id'] = resource_id
tasks.datastorer_upload(json.dumps(context), json.dumps(data))
response = requests.get(
'http://%s/api/action/datastore_search?resource_id=%s' % (self.host, resource_id),
headers={"content-type": "application/json"})
result = json.loads(response.content)
def test_another_file(self):
data = {
'url': 'http://0.0.0.0:50001/static/october_2011.csv',
'format': 'csv'}
context = {'site_url': 'http://%s' % self.host,
'apikey': self.api_key,
'site_user_apikey': self.api_key,
}
resource_id = self.make_resource_id()
data['id'] = resource_id
tasks.datastorer_upload(json.dumps(context), json.dumps(data))
response = requests.get(
'http://%s/api/action/datastore_search?resource_id=%s' % (self.host, resource_id),
headers={"content-type": "application/json"})
result = json.loads(response.content)
value = result['result']['records'][0][u'Supplier Name']
assert value == 'ALBANY OFFICE FURNITURE SOLUTIONS', value
assert result['result']['total'] == 230, (result['result']['total'], resource_id)
assert len(result['result']['records']) == 100
value = result['result']['records'][65][u'Supplier Name']
assert_equal(value, 'CAREWATCH (NEWCASTLE UPON TYNE)')
assert_equal(result['result']['fields'], [{u'type': u'int4', u'id': u'_id'},
{u'type': u'text', u'id': u'Directorate'},
{u'type': u'text', u'id': u'Service Area'},
{u'type': u'text', u'id': u'Expenditure Category'},
{u'type': u'timestamp', u'id': u'Payment Date'},
{u'type': u'text', u'id': u'Supplier Name'},
{u'type': u'numeric', u'id': u'Internal Ref'},
{u'type': u'text', u'id': u'Capital/ Revenue'},
{u'type': u'text', u'id': u'Cost Centre'},
{u'type': u'text', u'id': u'Cost Centre Description'},
{u'type': u'float8', u'id': u'Grand Total'}])
def test_messier_file(self):
data = {'url': 'http://0.0.0.0:50001/static/3ffdcd42-5c63-4089-84dd-c23876259973',
'format': 'csv'}
context = {'site_url': 'http://0.0.0.0:8088',
'apikey': 'moo',
'site_user_apikey': 'moo',
'username': '******'}
resource_id = self.make_resource_id()
data['id'] = resource_id
tasks.datastorer_upload(json.dumps(context), json.dumps(data))
response = requests.get(
'http://0.0.0.0:8088/api/data/%s/_search?q=*' % resource_id,
)
response = json.loads(response.content)
assert len(response['hits']['hits']) == 10, len(response['hits']['hits'])
def test_long_file(self):
data = {
'url': 'http://0.0.0.0:50001/static/long.csv',
'format': 'csv'}
context = {'site_url': 'http://%s' % self.host,
'apikey': self.api_key,
'site_user_apikey': self.api_key,
}
resource_id = self.make_resource_id()
data['id'] = resource_id
tasks.datastorer_upload(json.dumps(context), json.dumps(data))
response = requests.get(
'http://%s/api/action/datastore_search?resource_id=%s' % (self.host, resource_id),
headers={"content-type": "application/json"})
result = json.loads(response.content)
assert result['result']['total'] == 4000, (result['result']['total'], resource_id)
assert len(result['result']['records']) == 100
