def test_create():
mock_adapter = {}
mock_adapter["prefix"] = PREFIX
adapter = requests_mock.Adapter()
mock_adapter["adapter"] = adapter
client = Socrata(DOMAIN, APPTOKEN, username=USERNAME, password=PASSWORD,
session_adapter=mock_adapter)
response_data = "create_foobar.txt"
setup_mock(adapter, "POST", response_data, 200, dataset_identifier=None)
columns = [
{"fieldName": "foo", "name": "Foo", "dataTypeName": "text"},
{"fieldName": "bar", "name": "Bar", "dataTypeName": "number"}
]
tags = ["foo", "bar"]
response = client.create("Foo Bar", description="test dataset",
columns=columns, tags=tags, row_identifier="bar")
request = adapter.request_history[0]
request_payload = json.loads(request.text) # can't figure out how to use .json
# Test request payload
for dataset_key in ["name", "description", "columns", "tags"]:
assert dataset_key in request_payload
for column_key in ["fieldName", "name", "dataTypeName"]:
assert column_key in request_payload["columns"][0]
# Test response
assert isinstance(response, dict)
assert len(response.get("id")) == 9
client.close()
def CreateETL(view_data, title, description, category, tags):
#preparing data
cols = list(view_data)
columns = [{
"fieldName": cols[0].lower(),
"name": cols[0],
"dataTypeName": "text"
}]
for i in range(1, len(cols)):
x = {
"fieldName": cols[i].lower(),
"name": cols[i],
"dataTypeName": "text"
}
columns.append(x)
tags = tags.split(",")
#Uploadin data
try:
client = Socrata(cfg["web"],
cfg["token"],
username=cfg["email"],
password=cfg["password"])
print(tags)
print(category)
print(description)
print(columns)
print(cols)
NewDataSet = client.create(title,
description=description,
columns=columns,
tags=tags,
category=category)
client.publish(NewDataSet.get('id'))
client.set_permission(NewDataSet.get('id'), "private")
# Convertion to JSON
datajson = view_data.to_json(None, orient='records')
# JSON to list
datajson = json.loads(datajson)
client.replace(NewDataSet.get('id'), datajson)
print('Socrata done')
error = 'OK'
dataset_id = NewDataSet.get('id')
client.close()
except BaseException as e:
#if there is an error, reload login with error message
error = str(e)
print('Error description:')
print(error)
dataset_id = 'NoData'
return error, dataset_id
def CreateMetadata(filename, sheetname, FirstCol, LastCol, tags, title):
#Create Socrata client
client = Socrata(cfg["web"],
cfg["token"],
username=cfg["email"],
password=cfg["password"])
#preparing Excel file
parse_cols = '%s:%s' % (FirstCol, LastCol)
#getting path of the excel file
path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'documents/%s' % filename.replace(" ", "_"))
#Reading excel with Panda
ExcelFile = pd.read_excel(path,
sheetname=sheetname,
parse_cols=parse_cols,
index=False)
#Defining columns automatically
cols = list(ExcelFile)
columns = [{
"fieldName": cols[0].lower(),
"name": cols[0],
"dataTypeName": "text"
}]
for i in range(1, len(cols)):
x = {
"fieldName": cols[i].lower(),
"name": cols[i],
"dataTypeName": "text"
}
columns.append(x)
#create dataset
NewDataSet = client.create(
title,
description="Lista de antenas Wi-fi en el atlántico",
columns=columns,
tags=tags,
category="Ciencia, Tecnología e Innovación")
## Publish dataset NewDataSet.get('id') get the dataset ID
client.publish(NewDataSet.get('id'))
client.set_permission(NewDataSet.get('id'), "public")
# Reemplazar datos
# Conversion a JSON
datajson = ExcelFile.to_json(None, orient='records')
# Conversion a list
datajson = json.loads(datajson)
client.replace(NewDataSet.get('id'), datajson)
print('Socrata done')
client.close()
