def CreateETL(view_data, title, description, category, tags):
#preparing data
cols = list(view_data)
columns = [{
"fieldName": cols[0].lower(),
"name": cols[0],
"dataTypeName": "text"
}]
for i in range(1, len(cols)):
x = {
"fieldName": cols[i].lower(),
"name": cols[i],
"dataTypeName": "text"
}
columns.append(x)
tags = tags.split(",")
#Uploadin data
try:
client = Socrata(cfg["web"],
cfg["token"],
username=cfg["email"],
password=cfg["password"])
print(tags)
print(category)
print(description)
print(columns)
print(cols)
NewDataSet = client.create(title,
description=description,
columns=columns,
tags=tags,
category=category)
client.publish(NewDataSet.get('id'))
client.set_permission(NewDataSet.get('id'), "private")
# Convertion to JSON
datajson = view_data.to_json(None, orient='records')
# JSON to list
datajson = json.loads(datajson)
client.replace(NewDataSet.get('id'), datajson)
print('Socrata done')
error = 'OK'
dataset_id = NewDataSet.get('id')
client.close()
except BaseException as e:
#if there is an error, reload login with error message
error = str(e)
print('Error description:')
print(error)
dataset_id = 'NoData'
return error, dataset_id
def CreateMetadata(filename, sheetname, FirstCol, LastCol, tags, title):
#Create Socrata client
client = Socrata(cfg["web"],
cfg["token"],
username=cfg["email"],
password=cfg["password"])
#preparing Excel file
parse_cols = '%s:%s' % (FirstCol, LastCol)
#getting path of the excel file
path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'documents/%s' % filename.replace(" ", "_"))
#Reading excel with Panda
ExcelFile = pd.read_excel(path,
sheetname=sheetname,
parse_cols=parse_cols,
index=False)
#Defining columns automatically
cols = list(ExcelFile)
columns = [{
"fieldName": cols[0].lower(),
"name": cols[0],
"dataTypeName": "text"
}]
for i in range(1, len(cols)):
x = {
"fieldName": cols[i].lower(),
"name": cols[i],
"dataTypeName": "text"
}
columns.append(x)
#create dataset
NewDataSet = client.create(
title,
description="Lista de antenas Wi-fi en el atlántico",
columns=columns,
tags=tags,
category="Ciencia, Tecnología e Innovación")
## Publish dataset NewDataSet.get('id') get the dataset ID
client.publish(NewDataSet.get('id'))
client.set_permission(NewDataSet.get('id'), "public")
# Reemplazar datos
# Conversion a JSON
datajson = ExcelFile.to_json(None, orient='records')
# Conversion a list
datajson = json.loads(datajson)
client.replace(NewDataSet.get('id'), datajson)
print('Socrata done')
client.close()
def test_publish():
mock_adapter = {}
mock_adapter["prefix"] = PREFIX
adapter = requests_mock.Adapter()
mock_adapter["adapter"] = adapter
client = Socrata(DOMAIN, APPTOKEN, username=USERNAME, password=PASSWORD,
session_adapter=mock_adapter)
response_data = "create_foobar.txt"
setup_publish_mock(adapter, "POST", response_data, 200)
response = client.publish(DATASET_IDENTIFIER)
assert isinstance(response, dict)
assert len(response.get("id")) == 9
client.close()
def test_publish():
mock_adapter = {}
mock_adapter["prefix"] = PREFIX
adapter = requests_mock.Adapter()
mock_adapter["adapter"] = adapter
client = Socrata(DOMAIN, APPTOKEN, username=USERNAME, password=PASSWORD,
session_adapter=mock_adapter)
response_data = "create_foobar.txt"
resource = "/api/views/songs/publication.json" # publish() removes .json
set_up_mock(adapter, "POST", response_data, 200, resource=resource)
response = client.publish("/resource/songs.json") # hard-coded so request uri is matched
assert isinstance(response, dict)
assert len(response.get("id")) == 9
client.close()