def CreateETL(view_data, title, description, category, tags): #preparing data cols = list(view_data) columns = [{ "fieldName": cols[0].lower(), "name": cols[0], "dataTypeName": "text" }] for i in range(1, len(cols)): x = { "fieldName": cols[i].lower(), "name": cols[i], "dataTypeName": "text" } columns.append(x) tags = tags.split(",") #Uploadin data try: client = Socrata(cfg["web"], cfg["token"], username=cfg["email"], password=cfg["password"]) print(tags) print(category) print(description) print(columns) print(cols) NewDataSet = client.create(title, description=description, columns=columns, tags=tags, category=category) client.publish(NewDataSet.get('id')) client.set_permission(NewDataSet.get('id'), "private") # Convertion to JSON datajson = view_data.to_json(None, orient='records') # JSON to list datajson = json.loads(datajson) client.replace(NewDataSet.get('id'), datajson) print('Socrata done') error = 'OK' dataset_id = NewDataSet.get('id') client.close() except BaseException as e: #if there is an error, reload login with error message error = str(e) print('Error description:') print(error) dataset_id = 'NoData' return error, dataset_id
def CreateMetadata(filename, sheetname, FirstCol, LastCol, tags, title): #Create Socrata client client = Socrata(cfg["web"], cfg["token"], username=cfg["email"], password=cfg["password"]) #preparing Excel file parse_cols = '%s:%s' % (FirstCol, LastCol) #getting path of the excel file path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'documents/%s' % filename.replace(" ", "_")) #Reading excel with Panda ExcelFile = pd.read_excel(path, sheetname=sheetname, parse_cols=parse_cols, index=False) #Defining columns automatically cols = list(ExcelFile) columns = [{ "fieldName": cols[0].lower(), "name": cols[0], "dataTypeName": "text" }] for i in range(1, len(cols)): x = { "fieldName": cols[i].lower(), "name": cols[i], "dataTypeName": "text" } columns.append(x) #create dataset NewDataSet = client.create( title, description="Lista de antenas Wi-fi en el atlántico", columns=columns, tags=tags, category="Ciencia, Tecnología e Innovación") ## Publish dataset NewDataSet.get('id') get the dataset ID client.publish(NewDataSet.get('id')) client.set_permission(NewDataSet.get('id'), "public") # Reemplazar datos # Conversion a JSON datajson = ExcelFile.to_json(None, orient='records') # Conversion a list datajson = json.loads(datajson) client.replace(NewDataSet.get('id'), datajson) print('Socrata done') client.close()
def test_publish(): mock_adapter = {} mock_adapter["prefix"] = PREFIX adapter = requests_mock.Adapter() mock_adapter["adapter"] = adapter client = Socrata(DOMAIN, APPTOKEN, username=USERNAME, password=PASSWORD, session_adapter=mock_adapter) response_data = "create_foobar.txt" setup_publish_mock(adapter, "POST", response_data, 200) response = client.publish(DATASET_IDENTIFIER) assert isinstance(response, dict) assert len(response.get("id")) == 9 client.close()
def test_publish(): mock_adapter = {} mock_adapter["prefix"] = PREFIX adapter = requests_mock.Adapter() mock_adapter["adapter"] = adapter client = Socrata(DOMAIN, APPTOKEN, username=USERNAME, password=PASSWORD, session_adapter=mock_adapter) response_data = "create_foobar.txt" resource = "/api/views/songs/publication.json" # publish() removes .json set_up_mock(adapter, "POST", response_data, 200, resource=resource) response = client.publish("/resource/songs.json") # hard-coded so request uri is matched assert isinstance(response, dict) assert len(response.get("id")) == 9 client.close()