def query_aot(sensor_hrf, size_per_page=100000, page_limit=1, mins_ago=12*60):
sensors = (
SENSOR_DF.loc[SENSOR_DF['sensor_measure']==sensor_hrf, 'sensor_path']
.unique()
)
if len(sensors) == 0:
return pd.DataFrame()
client = AotClient()
# TODO: combine many sensors in API call
pages = []
for sensor in sensors:
f = F('size', str(size_per_page))
f &= ('sensor', sensor)
f &= ('timestamp', 'ge', time_x_mins_ago(mins_ago))
response = client.list_observations(filters=f)
pages.extend(unpack_response(response, page_limit=page_limit))
obs_df = pd.DataFrame(pages)
obs_df = process_observations(obs_df)
return obs_df
def test_data_property():
client = AotClient()
res = client.list_observations()
assert isinstance(res.data, list)
for record in res.data:
assert isinstance(record, dict)
def test_iter():
client = AotClient()
dataset = client.list_observations(filters=F('size', 1))
i = 0
for page in dataset:
i += 1
if i >= 5:
break
def check_current_timestamp():
chicago_dataset = AotClient().list_observations()
for observation in chicago_dataset:
timestamp = set()
for item in range(len(observation.data)):
#create a set
current_timestamp = observation.data[item]["timestamp"]
timestamp.add(current_timestamp)
return timestamp, observation
def query_aot(sensor_hrf,
size_per_page=100000,
page_limit=1,
mins_ago=12 * 60):
sensor = SENSOR_DF.loc[SENSOR_DF['sensor_measure'] == sensor_hrf,
'sensor_path'].values[0]
client = AotClient()
f = F('size', str(size_per_page))
f &= ('sensor', sensor)
f &= ('timestamp', 'ge', time_x_mins_ago(mins_ago))
response = client.list_observations(filters=f)
pages = unpack_response(response, page_limit=page_limit)
obs_df = pd.DataFrame(pages)
obs_df = process_observations(obs_df)
return obs_df
import pandas as pd
from aot_client import AotClient
import datetime
import pandas as pd
from aot_client import AotClient
from aot_client import F
from bs4 import BeautifulSoup
import requests
client = AotClient()
def get_nodes(): # Returns a DataFrame with information on nodes.
'''
Returns a DataFrame with information on nodes.
Current columns =
['node_id', 'project_id', 'vsn', 'address', 'lat', 'lon', 'description',
'start_timestamp', 'end_timestamp', 'Unnamed: 9']
'''
r = requests.get('https://aot-file-browser.plenar.io/data-sets/chicago-complete')
soup = BeautifulSoup(r.text, 'lxml')
nodes = pd.read_html(str(soup.findAll(class_='table')[2]))[0]
return nodes
def get_sensors(): # Returns a DataFrame with information on sensors.
'''
Returns a DataFrame with information on sensors.
Current columns =
def test_list_projects():
client = AotClient()
res = client.list_projects()
assert isinstance(res, PagedResponse)
def test_list_observations():
client = AotClient()
res = client.list_observations()
assert isinstance(res, PagedResponse)
def test_get_sensor_details():
client = AotClient()
res = client.get_sensor_details('chemsense.co.concentration')
assert isinstance(res, Response)
def test_get_node_details():
client = AotClient()
res = client.get_node_details('004')
assert isinstance(res, Response)
def test_get_project_doesnt_exist():
client = AotClient()
with pytest.raises(HTTPError):
res = client.get_project_details('nowhere')
def test_get_project_details():
client = AotClient()
res = client.get_project_details('chicago')
assert isinstance(res, Response)
if process_is_running('producer.py'):
print('Found another instance, exiting...')
exit()
topic = "obs-stream"
brokers = ['10.0.0.7:9092', '10.0.0.9:9092', '10.0.0.11:9092']
mins_ago = 60 #*6
# Instantiate a Kafka Producer
producer = KafkaProducer(bootstrap_servers=brokers, \
value_serializer=lambda x: \
dumps(x).encode('utf-8'))
# Initialize AoT client
client = AotClient()
# Get previous record timestamp
try:
fh = open("state.txt", "r")
prev_record_timestamp = fh.read()
t = ciso8601.parse_datetime(prev_record_timestamp)
fh.close()
except FileNotFoundError:
t = (datetime.datetime.utcnow() - datetime.timedelta(minutes=mins_ago))
prev_record_timestamp = t.isoformat()[0:19]
# Initialize filter (city- Chicago, 5000 records, timestamp, order by timestamp)
f = F('project', 'chicago')
f &= ('size', '5000')
f &= ('timestamp', 'gt', prev_record_timestamp)
def test_next_link_property():
client = AotClient()
res = client.list_observations(filters=F('page', '2'))
assert isinstance(res.next_link, str)