class TestCase(unittest.TestCase):
def create_rev(self):
p = Socrata(auth)
(ok, r) = self.view.revisions.create_update_revision()
assert ok
self.rev = r
return r
def create_input_schema(self, rev=None, filename='simple.csv'):
if not rev:
rev = self.create_rev()
(ok, source) = rev.create_upload('foo.csv')
assert ok
with open('test/fixtures/%s' % filename, 'rb') as f:
(ok, source) = source.csv(f)
assert ok, source
return source.get_latest_input_schema()
def create_output_schema(self, input_schema=None):
if not input_schema:
input_schema = self.create_input_schema()
(ok, output_schema) = input_schema.transform({
'output_columns': [{
"field_name": "b",
"display_name": "b, but as a number",
"position": 0,
"description": "b but with a bunch of errors",
"transform": {
"transform_expr": "to_number(b)"
}
}]
})
assert ok
return output_schema
def setUp(self):
self.pub = Socrata(auth)
(ok, rev) = self.pub.new({'name': 'test-view'})
assert ok, rev
self.rev = rev
(ok, view) = self.pub.views.lookup(rev.attributes['fourfour'])
assert ok, view
self.view = view
def tearDown(self):
if getattr(self, 'rev', False):
self.rev.discard()
self.view.delete()
class SocrataDataset(object):
def __init__(self, uuid=None, mq_fp=None, auth_param=auth_param):
self.existing_dataset = bool(uuid)
self.uuid = uuid
self.mq_fp = mq_fp
self.auth_param = auth_param
self.socrata = Socrata(Authorization(*auth_param))
self.mq = None
self.metadata = None
if mq_fp:
self.mq = ITSMetadataQuestionnaire(mq_fp)
self.retrieve_metadata()
def generate_dataset_url(self):
return 'https://{}/d/{}'.format(self.auth_param[0], self.uuid)
def retrieve_metadata(self):
if self.uuid:
url = 'https://{}/api/views/metadata/v1/{}'.format(
self.auth_param[0], self.uuid)
r = requests.get(url, auth=self.auth_param[1:])
self.metadata = r.json()
return
def create_dataset(self):
if self.uuid is None:
# NOTE: publishing this way seems to default to creating integrated data set only
# TODO: modify so that we can publish external datasets
# TODO: fix permission - right now it's published as public data set
revision_json = self.mq.generate_dtg_metadata()
revision_json.update(
{'action': {
'type': 'update',
'permission': 'private'
}})
(ok, revision) = self.socrata.new(revision_json)
(ok, job) = revision.apply()
(ok, job) = job.wait_for_finish(progress=lambda job: print(
'Job progress:', job.attributes['status']))
self.uuid = revision.view_id()
print('Dataset has been created at {}'.format(
self.generate_dataset_url()))
else:
print('Dataset already exists at {}'.format(
self.generate_dataset_url()))
return
def update_metadata(self):
if self.uuid:
url = 'https://{}/api/views/metadata/v1/{}'.format(
self.auth_param[0], self.uuid)
response = requests.patch(url,
auth=self.auth_param[1:],
data=json.dumps(
self.mq.generate_dtg_metadata()))
if response.status_code != 200:
print(response)
else:
print('Metadata updated for {}'.format(
self.generate_dataset_url()))