def simulate(nr_buses=10):
ac = AmigoCloud(AMIGOCLOUD_TOKEN)
url = 'users/%s/projects/%s/datasets/%s/realtime' % (
AMIGOCLOUD_USER_ID, AMIGOCLOUD_PROJECT_ID,
AMIGOCLOUD_REALTIME_DATASET_ID
)
buses = [{'lat': 37.780061, 'lng': -122.413661} for _ in xrange(nr_buses)]
while True:
for bus_id in xrange(nr_buses):
buses[bus_id]['lat'] += random.uniform(-0.02, 0.02)
buses[bus_id]['lng'] += random.uniform(-0.02, 0.02)
now = time.time()
moovbox_data = create_moovbox_data(
id=bus_id,
latitude=buses[bus_id]['lng'],
longitude=buses[bus_id]['lat'],
fix=0,
time=int(now),
altitude=0,
climb=0,
speed=0,
separation=0,
track=0,
satellites=0
)
ac.post(url, data=moovbox_data, send_as_json=False)
print moovbox_data
time.sleep(1)
def amigocloud_sync_earthquakes(page=1):
change_data = []
for earthquake in get_earthquakes_data(page):
amigo_id, amigo_data = to_amigo_format(earthquake)
change_data.append({'amigo_id': amigo_id, 'new': amigo_data})
change = {
'type': 'DML',
'entity': 'dataset_%s' % DATASET_ID,
'action': 'INSERT',
'data': change_data
}
amigocloud = AmigoCloud(TOKEN, PROJECT_URL)
amigocloud.post('datasets/%s/submit_change' % DATASET_ID,
{'change': json.dumps(change)})
"action": "ADD COLUMN",
"parent": master,
"data": [
{
"new": {
"name": "log_level",
"type": "integer",
"nullable": False,
"default": 2
}
}
]
}
# Add 'log_level' column to the dataset
response = ac.post(dataset['submit_change'],
{'change': json.dumps(add_column)})
response = ac.get(dataset['schema'])
print 'Current schema of the dataset:'
pprint(response)
# Request and store new master state
response = ac.get(dataset['master'])
new_master = response['master']
# Build "ALTER COLUMN schema change:
alter_column = {
"type": "DDL",
"entity": table_name,
"action": "ALTER COLUMN",
"(location, latitude, longitude) = "
"(ST_SetSRID(ST_MakePoint({lon}, {lat}), 4326), {lat}, {lon}) "
"WHERE amigo_id = '{amigo_id}'"
)
while True:
# Fetch the next 100 records
result = ac.get(sql_url, {'query': select_query,
'limit': limit, 'offset': offset})
for row in result['data']:
# Join street_address + city + state
address = [row['street_address'], row['city'], row['state']]
address_str = ' '.join(elem for elem in address if elem)
# Geocode the address (get coordinates)
print '[%s/%s] Geocoding "%s" ...' % (counter, total_rows, address_str)
geo = Geocoder.geocode(address_str)
# Update the row, saving the lat, long and geometry in the respective
# columns (the dataset must already have these columns)
kwargs = {'table_name': table_name, 'amigo_id': row['amigo_id'],
'lat': geo.coordinates[0], 'lon': geo.coordinates[1]}
ac.post(sql_url, {'query': update_query.format(**kwargs)})
counter += 1
if len(result['data']) < limit:
break
offset += limit
if not offset: # i.e. If first request
print 'The schema of the result is:'
pprint(response['columns'])
fetched_rows = len(response['data'])
offset += fetched_rows
rows += response['data']
print 'Fetched', fetched_rows, 'rows'
print 'These are all the rows:'
pprint(rows)
# Update records based on some condition
query = ("UPDATE {table} SET field1 = field1 + 1 "
"WHERE amigo_id = 'abcd'".format(table=table_name))
response = ac.post(sql_url, {'query': query})
print 'Query:', response['query']
print 'This query updated', response['count'], 'row(s)'
# Delete records based on some condition
query = "DELETE FROM {table} WHERE amigo_id = 'abcd'".format(table=table_name)
response = ac.post(sql_url, {'query': query})
print 'Query:', response['query']
print 'This query deleted', response['count'], 'row(s)'
# Insert new record
query = ("INSERT INTO {table} (field1, field2) "
"VALUES (123, 'Hello world!')".format(table=table_name))
response = ac.post(sql_url, {'query': query})
from amigocloud import AmigoCloud
# Use amigocloud version 1.0.5 or higher to login with tokens
# This will raise an AmigoCloudError if the token is invalid or has expired
ac = AmigoCloud(token='<token>')
query = ({
"author": "",
"extra": "",
"layer_name": "0",
"name": "My first baselayer",
"public_tiles": False,
"transparency": False,
"url": "<baselayer URL>",
"zoom_level_max": 20,
"zoom_level_min": 0
})
sql_url='<AmigoCloud baselater API URL>'
response = ac.post(url=sql_url, data=query, content_type="application/json")
print 'Response:', response
# Add new column called "area"
add_column = {
"type": "DDL",
"entity": dataset_data['table_name'],
"action": "ADD COLUMN",
"data": [{
"new": {
"name": "area",
"type": "float",
"nullable": False,
"default": 0.0
}
}]
}
response = amigocloud.post(dataset_data['submit_change'],
{'change': json.dumps(add_column)})
# Wait until async job is done
job_url = 'me/jobs/%s' % response['job']
while True:
response = amigocloud.get(job_url)
if response['status'] not in ('STARTED', 'PENDING'):
break
time.sleep(0.5) # Wait 500ms and retry
print 'Column "area" was added to dataset'
# Fill new column "area" with the area of the geometry using the SQL API
sql_api_url = 'users/%s/projects/%s/sql' % (PROJECT_OWNER, PROJECT_ID)
query = 'UPDATE "%s" SET area = ST_AREA("%s"::geography)' % (
dataset_data['table_name'], dataset_data['geometry_column']
)
# reverted in reversed order)
FROM_STATE = '...'
TO_STATE = '...'
# Get dataset information
dataset_url = (
'/users/{user_id}/projects/{project_id}/datasets/{dataset_id}'.format(
user_id=PROJECT_OWNER, project_id=PROJECT_ID, dataset_id=DATASET_ID
)
)
dataset = ac.get(dataset_url)
# Get list of all the states between FROM_STATE and TO_STATE
result = ac.get(dataset['states'], {'from': FROM_STATE, 'to': TO_STATE})
states = result['states']
# Revert the states in reverted order
for state in reversed(states):
revert_state_url = dataset_url + '/revert/{state}'.format(state=state)
result = ac.post(revert_state_url) # Send revert
job = result['job']
# Request AmigoCloud every second to check if the async job finished
while True:
result = ac.get('me/jobs/{job}'.format(job=job))
if result['status'] not in ('PENDING', 'STARTED'):
break
time.sleep(1) # Sleep 1s
print 'Reverting state "{state}" finished with status: {status}'.format(
state=state, status=result.get('status', None)
)
class BLDSDataset:
dataset_url = '/users/{user_id}/projects/{project_id}/datasets/{dataset_id}'
def __init__(self, project_id, dataset_id, token):
self.project_id = project_id
self.dataset_id = dataset_id
self.ac = AmigoCloud(token=token)
self.dataset = self.ac.get(
self.dataset_url.format(user_id=1,
project_id=project_id,
dataset_id=dataset_id))
self.table_name = self.dataset['table_name']
self.response = self.ac.get(self.dataset['master'])
self.master = self.response['master']
def add_column(self, column_json):
print("Add column: " + column_json["name"])
add_column = {
"type": "DDL",
"entity": self.table_name,
"action": "ADD COLUMN",
"parent": self.master,
"data": [{
"new": column_json
}]
}
response = self.ac.post(self.dataset['submit_change'],
{'change': json.dumps(add_column)})
time.sleep(5) # to prevent Error: TOO MANY REQUESTS
def add_columns(self, columns_json):
print("Add columns")
add_column = {
"type": "DDL",
"entity": self.table_name,
"action": "ADD COLUMN",
"parent": self.master,
"data": columns_json
}
response = self.ac.post(self.dataset['submit_change'],
{'change': json.dumps(add_column)})
"""
create_schema() creates schema for BLDS standard dataset
"""
def create_schema(self):
print("Create schema for " + self.table_name)
columns = [
# amigo_id field should be already created
# {
# "name": "amigo_id",
# "nullable": False,
# "default": "GENERATE_UUID",
# "auto_populate": True,
# "max_length": 32,
# "type": "string"
# },
# Geometry field
{
"name": "wkb_geometry",
"visible": True,
"geometry_type": "POINT",
"nullable": True,
"editable": True,
"alias": "wkb geometry",
"type": "geometry"
},
# Required fields
{
"name": "permit_num",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "PermitNum",
"type": "string"
},
{
"name": "description",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "Description",
"type": "string"
},
{
"name": "applied_date",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "AppliedDate",
"type": "string"
},
{
"name": "issued_date",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "IssuedDate",
"type": "string"
},
{
"name": "completed_date",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "CompletedDate",
"type": "string"
},
{
"name": "status_current",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "StatusCurrent",
"type": "string"
},
{
"name": "original_address1",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "OriginalAddress1",
"type": "string"
},
{
"name": "original_address2",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "OriginalAddress2",
"type": "string"
},
{
"name": "original_city",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "OriginalCity",
"type": "string"
},
{
"name": "original_state",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "OriginalState",
"type": "string"
},
{
"name": "original_zip",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "OriginalZip",
"type": "string"
},
# Recommended fields
{
"name": "jurisdiction",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "Jurisdiction",
"type": "string"
},
{
"name": "permit_class",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "PermitClass",
"type": "string"
},
{
"name": "permit_class_mapped",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "PermitClassMapped",
"type": "string"
},
{
"name":
"status_current_mapped",
"nullable":
True,
"default":
None,
"editable":
True,
"choices": [{
"code": "Appeal",
"value": "Appeal"
}, {
"code": "Application Accepted",
"value": "Application Accepted"
}, {
"code": "Fees/Payment",
"value": "Fees/Payment"
}, {
"code": "In Review",
"value": "In Review"
}, {
"code": "Inspection Phase",
"value": "Inspection Phase"
}, {
"code": "Occupancy",
"value": "Occupancy"
}, {
"code": "Permit Cancelled",
"value": "Permit Cancelled"
}, {
"code": "Permit Finaled",
"value": "Permit Finaled"
}, {
"code": "Permit Finaled with Conditions",
"value": "Permit Finaled with Conditions"
}, {
"code": "Permit Issued",
"value": "Permit Issued"
}],
"visible":
True,
"alias":
"StatusCurrentMapped",
"type":
"string"
},
{
"name": "work_class",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "WorkClass",
"type": "string"
},
{
"name":
"permit_type_mapped",
"nullable":
True,
"default":
None,
"editable":
True,
"choices": [{
"code": "Building",
"value": "Building"
}, {
"code": "Demolition",
"value": "Demolition"
}, {
"code": "Electrical",
"value": "Electrical"
}, {
"code": "Fence",
"value": "Fence"
}, {
"code": "Grading",
"value": "Grading"
}, {
"code": "Mechanical",
"value": "Mechanical"
}, {
"code": "Plumbing",
"value": "Plumbing"
}, {
"code": "Pool/Spa",
"value": "Pool/Spa"
}, {
"code": "Roof",
"value": "Roof"
}],
"visible":
True,
"alias":
"PermitTypeMapped",
"type":
"string"
},
{
"name": "permit_type",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "PermitType",
"type": "string"
},
{
"name":
"work_class_mapped",
"nullable":
True,
"default":
None,
"editable":
True,
"choices": [{
"code": "Existing",
"value": "Existing"
}, {
"code": "New",
"value": "New"
}],
"visible":
True,
"alias":
"WorkClassMapped",
"type":
"string"
},
{
"name": "permit_type_desc",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "PermitTypeDesc",
"type": "string"
},
{
"name": "est_project_cost",
"nullable": True,
"default": None,
"max_value": None,
"min_value": None,
"editable": True,
"visible": True,
"alias": "EstProjectCost",
"type": "float"
},
{
"name": "status_date",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "StatusDate",
"type": "string"
},
{
"name": "contractor_trade",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "ContractorTrade",
"type": "string"
},
{
"name": "latitude",
"nullable": True,
"default": None,
"max_value": None,
"min_value": None,
"editable": True,
"visible": True,
"alias": "Latitude",
"type": "float"
},
{
"name": "longitude",
"nullable": True,
"default": None,
"max_value": None,
"min_value": None,
"editable": True,
"visible": True,
"alias": "Longitude",
"type": "float"
},
{
"name": "pin",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "PIN",
"type": "string"
},
{
"name": "housing_units",
"nullable": True,
"default": None,
"max_value": None,
"min_value": None,
"editable": True,
"visible": True,
"alias": "HousingUnits",
"type": "integer"
},
{
"name": "total_sqft",
"nullable": True,
"default": None,
"max_value": None,
"min_value": None,
"editable": True,
"visible": True,
"alias": "TotalSqFt",
"type": "float"
},
{
"name": "contractor_state_lic",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "ContractorStateLic",
"type": "string"
},
{
"name": "link",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "Link",
"type": "string"
},
{
"name": "contractor_trade_mapped",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "ContractorTradeMapped",
"type": "string"
},
{
"name": "contractor_lic_num",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "ContractorLicNum",
"type": "string"
},
{
"name": "contractor_company_name",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "ContractorCompanyName",
"type": "string"
},
# Optional fields
{
"name": "proposed_use",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "ProposedUse",
"type": "string"
},
{
"name": "added_sqft",
"nullable": True,
"default": None,
"max_value": None,
"min_value": None,
"editable": True,
"visible": True,
"alias": "AddedSqFt",
"type": "float"
},
{
"name": "removed_sqft",
"nullable": True,
"default": None,
"max_value": None,
"min_value": None,
"editable": True,
"visible": True,
"alias": "RemovedSqFt",
"type": "float"
},
{
"name": "master_permit_num",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "MasterPermitNum",
"type": "string"
},
{
"name": "expires_date",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "ExpiresDate",
"type": "string"
},
{
"name": "co_issued_date",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "COIssuedDate",
"type": "string"
},
{
"name": "hold_date",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "HoldDate",
"type": "string"
},
{
"name": "void_date",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "VoidDate",
"type": "string"
},
{
"name": "project_name",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "ProjectName",
"type": "string"
},
{
"name": "project_id",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "ProjectID",
"type": "string"
},
{
"name": "total_finished_sqft",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "TotalFinishedSqFt",
"type": "string"
},
{
"name": "total_unfinished_sqft",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "TotalUnfinishedSqFt",
"type": "string"
},
{
"name": "total_heated_sqft",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "TotalHeatedSqFt",
"type": "string"
},
{
"name": "total_unheated_sqft",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "TotalUnHeatedSqFt",
"type": "string"
},
{
"name": "total_acc_sqft",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "TotalAccSqFt",
"type": "string"
},
{
"name": "total_sprinkled_sqft",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "TotalSprinkledSqFt",
"type": "string"
},
{
"name": "extra_fields",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "ExtraFields",
"type": "string"
},
{
"name": "publisher",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "Publisher",
"type": "string"
},
{
"name": "fee",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "Fee",
"type": "float"
},
{
"name": "contractor_full_name",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "ContractorFullName",
"type": "string"
},
{
"name": "contractor_company_desc",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "ContractorCompanyDesc",
"type": "string"
},
{
"name": "contractor_phone",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "ContractorPhone",
"type": "string"
},
{
"name": "contractor_address1",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "ContractorAddress1",
"type": "string"
},
{
"name": "contractor_address2",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "ContractorAddress2",
"type": "string"
},
{
"name": "contractor_city",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "ContractorCity",
"type": "string"
},
{
"name": "contractor_state",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "ContractorState",
"type": "string"
},
{
"name": "contractor_zip",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "ContractorZip",
"type": "string"
},
{
"name": "contractor_email",
"visible": True,
"default": None,
"nullable": True,
"editable": True,
"alias": "ContractorEmail",
"type": "string"
},
]
final_columns = []
for column in columns:
final_columns.append({"new": column})
self.add_columns(final_columns)
def upload_records(self, records, action):
insert_records = {
"type": "DML",
"entity": self.table_name,
"action": action,
"data": records
}
# print insert_record
response = self.ac.post(
self.dataset['submit_change'],
{'change': unicode(json.dumps(insert_records), errors='ignore')})
print("Upload " + str(len(records)) + " records to " + self.table_name)
time.sleep(10) # to prevent Error: TOO MANY REQUESTS
def get_record_obj(self, record, city):
obj = dict()
for field_name, value in record.iteritems():
blds_filed = city.get_field(field_name)
blds_value = city.get_value(field_name, value, record)
if blds_filed:
obj[blds_filed] = blds_value
u = uuid.uuid4()
new_obj = {"new": obj}
new_obj["amigo_id"] = u.hex
return new_obj
def upload_permits(self, permits, city):
records = []
index = 0
for key, value in permits.iteritems():
for p in value:
records.append(self.get_record_obj(p, city))
index += 1
if index >= 1000:
self.upload_records(records, "INSERT")
records[:] = []
index = 0
# Upload the rest of the records
self.upload_records(records, "INSERT")
def update_records(self, data):
records = []
index = 0
for r in data:
new_obj = {"new": r}
new_obj["amigo_id"] = r['amigo_id']
records.append(new_obj)
index += 1
if index >= 1000:
self.upload_records(records, "UPDATE")
records[:] = []
index = 0
# Upload the rest of the records
self.upload_records(records, "UPDATE")
def query_page(self, query, limit, offset):
sql_url = '/users/{user_id}/projects/{project_id}/sql'.format(
user_id=1, project_id=self.project_id)
rows = []
response = self.ac.get(
sql_url, {
'query': query,
'offset': offset,
'limit': limit,
'dataset_id': self.dataset_id
})
if not offset: # i.e. If first request
print('The schema of the result is:')
print(response['columns'])
fetched_rows = len(response['data'])
offset += fetched_rows
rows += response['data']
return rows
