def test_create(): mock_adapter = {} mock_adapter["prefix"] = PREFIX adapter = requests_mock.Adapter() mock_adapter["adapter"] = adapter client = Socrata(DOMAIN, APPTOKEN, username=USERNAME, password=PASSWORD, session_adapter=mock_adapter) response_data = "create_foobar.txt" setup_mock(adapter, "POST", response_data, 200, dataset_identifier=None) columns = [ {"fieldName": "foo", "name": "Foo", "dataTypeName": "text"}, {"fieldName": "bar", "name": "Bar", "dataTypeName": "number"} ] tags = ["foo", "bar"] response = client.create("Foo Bar", description="test dataset", columns=columns, tags=tags, row_identifier="bar") request = adapter.request_history[0] request_payload = json.loads(request.text) # can't figure out how to use .json # Test request payload for dataset_key in ["name", "description", "columns", "tags"]: assert dataset_key in request_payload for column_key in ["fieldName", "name", "dataTypeName"]: assert column_key in request_payload["columns"][0] # Test response assert isinstance(response, dict) assert len(response.get("id")) == 9 client.close()
def CreateETL(view_data, title, description, category, tags): #preparing data cols = list(view_data) columns = [{ "fieldName": cols[0].lower(), "name": cols[0], "dataTypeName": "text" }] for i in range(1, len(cols)): x = { "fieldName": cols[i].lower(), "name": cols[i], "dataTypeName": "text" } columns.append(x) tags = tags.split(",") #Uploadin data try: client = Socrata(cfg["web"], cfg["token"], username=cfg["email"], password=cfg["password"]) print(tags) print(category) print(description) print(columns) print(cols) NewDataSet = client.create(title, description=description, columns=columns, tags=tags, category=category) client.publish(NewDataSet.get('id')) client.set_permission(NewDataSet.get('id'), "private") # Convertion to JSON datajson = view_data.to_json(None, orient='records') # JSON to list datajson = json.loads(datajson) client.replace(NewDataSet.get('id'), datajson) print('Socrata done') error = 'OK' dataset_id = NewDataSet.get('id') client.close() except BaseException as e: #if there is an error, reload login with error message error = str(e) print('Error description:') print(error) dataset_id = 'NoData' return error, dataset_id
def CreateMetadata(filename, sheetname, FirstCol, LastCol, tags, title): #Create Socrata client client = Socrata(cfg["web"], cfg["token"], username=cfg["email"], password=cfg["password"]) #preparing Excel file parse_cols = '%s:%s' % (FirstCol, LastCol) #getting path of the excel file path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'documents/%s' % filename.replace(" ", "_")) #Reading excel with Panda ExcelFile = pd.read_excel(path, sheetname=sheetname, parse_cols=parse_cols, index=False) #Defining columns automatically cols = list(ExcelFile) columns = [{ "fieldName": cols[0].lower(), "name": cols[0], "dataTypeName": "text" }] for i in range(1, len(cols)): x = { "fieldName": cols[i].lower(), "name": cols[i], "dataTypeName": "text" } columns.append(x) #create dataset NewDataSet = client.create( title, description="Lista de antenas Wi-fi en el atlántico", columns=columns, tags=tags, category="Ciencia, Tecnología e Innovación") ## Publish dataset NewDataSet.get('id') get the dataset ID client.publish(NewDataSet.get('id')) client.set_permission(NewDataSet.get('id'), "public") # Reemplazar datos # Conversion a JSON datajson = ExcelFile.to_json(None, orient='records') # Conversion a list datajson = json.loads(datajson) client.replace(NewDataSet.get('id'), datajson) print('Socrata done') client.close()