def populate_from(self, time_from):
''' Populates database with calls API from 'time_from' until now.
time_now is an integer ms time epoch. API calls are made in steps of
1000 minutes. '''
# get current time
time_now = Scraper._time_now()
""" in steps of 1000 minutes, retrieve data from API and insert into
database. Note: typical interval between sensor reading is 1 minute,
API call returns max 1000 rows."""
for input_time in range(time_from, time_now, 60000000):
sensor_reading_after_data, all_sensor_numbers = \
database.smart_building.sensor_reading_after(
timestamp_epoch_millisec=input_time)
self.insert_sensor_readings_after(sensor_reading_after_data)
def _choose_time():
''' Take user input to choose a time in ms time epoch.
Times are equivalent to those at https://currentmillis.com/ '''
# time before the earliest sensor reading in ms format
earliest_time_ms = 1580920305102
# same time in utc format (/1000 as utcfromtimestamp takes input in s)
earliest_time_utc = dt.datetime.utcfromtimestamp(
int(earliest_time_ms / 1000)).isoformat()
# get time now
time_now_ms = Scraper._time_now()
time_now_utc = dt.datetime.utcfromtimestamp(int(time_now_ms /
1000)).isoformat()
chosen_times = input('Choose start and end time to plot in ms epochs '
'in format "[start, end]". or press enter full '
'time range. For example for 1st to 2nd March, '
'enter: [1583020800000, 1583107200000].'
'\nEarliest:\n ms: {}\n UTC: {}'
'\nLatest:\n ms: {}\n UTC: {}\n>>'.format(
earliest_time_ms, earliest_time_utc,
time_now_ms, time_now_utc))
# by default, choose from earilest time to now
if not chosen_times:
time_from_ms = earliest_time_ms
time_from_utc = earliest_time_utc
time_to_ms = time_now_ms
time_to_utc = time_now_utc
# otherwise, evalute the times inputted.
else:
chosen_times = eval(chosen_times)
time_from_ms = chosen_times[0]
time_to_ms = chosen_times[1]
time_from_utc = dt.datetime.utcfromtimestamp(
int(time_from_ms / 1000)).isoformat()
time_to_utc = dt.datetime.utcfromtimestamp(int(time_to_ms /
1000)).isoformat()
# print what the chosen time range.
print('Chosen time range from {} to {}.'.format(
time_from_utc, time_to_utc))
return (time_from_ms, time_to_ms)
def populate_database(self):
''' Calls API and returns readings from the earliest process, then in
steps of 1000 minutes until the current time. Note: runs based on
earliest from API, not from what exists in database.'''
# check when to start collecting data from
earliest_time, sensor_reading_after_data = self.find_earliest_time()
# put current time into variable
time_now = Scraper._time_now()
# insert the first time
self.insert_sensor_readings_after(sensor_reading_after_data)
""" in steps of 1000 minutes, retrieve data from API and insert into
database. Note: typical interval between sensor reading is 1 minute,
API call returns max 1000 rows."""
for input_time in range(earliest_time, time_now, 60000000):
sensor_reading_after_data, all_sensor_numbers = \
self.smart_building.sensor_reading_after(
timestamp_epoch_millisec=input_time)
self.insert_sensor_readings_after(sensor_reading_after_data)
'--from',
dest='_from',
nargs=1,
type=int,
help="Get all data from a certain point")
# Parse the command line arguments
args = parser.parse_args()
try:
# Connect to the database
database = Database()
if args.recent:
print("Getting most recent data from the API")
database.populate_from(Scraper._time_now())
elif args.all:
print("Getting all data from the api")
database.populate_database()
elif args._from:
time_from = args._from[0]
print("Getting all data from time point {}", time_from)
database.populate_from(time_from)
else:
raise Exception(
"No arguments provided! Should not have gotten here.")
#%%
def set_defaults(self):
'''
Sets plotting parameters of DatabasePlotter() class. Sets only those
set to None to default. Takes no input but checks and changes the
parameters of the class.
Parameters
----------
sensor_numbers : LIST of INT, optional
List of INT corresponding to sensor numbers from
DatabasePlotter.sensor_numbers. Default = all.
sensor_names : LIST of STR, optional
List of STR corresponding to sensor names
DatabasePlotter.sensor_names. Default = all.
room_numbers : LIST of STR, optional
List of INT corresponding to room numbers from
DatabasePlotter.room_numbers. Default = all.
room_names : LIST of STR, optional
List of STR corresponding to room names from
DatabasePlotter.room_names. Default = all.
time_from : INT, optional
First time to plot in ms timestamp format.
Default = first available.
time_to : INT, optional
Last time to plot in ms timestamp format.
Default = most recent available.
parameters : LIST of STR, optional
Choice of parameters from DatabasePlotter.param_list.
Default = all.
overlay : INT, optional
0 = seperate plots, 1 = overlay on same plot. Default = 1.
aggregate : INT, optional
0 = individual sensors, 1 = aggregate sensors in same room.
Default = 0.
seperate : INT, optional
0 = 1 sensors from different rooms on same plot, 1 = sensors
from different rooms are plotted seperately. Only relevant if
overlay = 1 and aggregate = 0.
Default = 1.
Returns
-------
No returns - sets plot parmaters as attributes of DatabasePlotter()
class.
'''
# get sensor numbers and names. Only requires uses sensors number
# as if there is input, others are there from early in
# plot_from_database()
self.sensor_numbers, self.sensor_names, self.room_numbers, \
self.room_names = \
self.get_names_and_numbers(sensors=self.sensor_numbers)
if self.sensor_numbers == None:
self.sensor_numbers = self.all_sensor_numbers
if self.sensor_names == None:
self.sensor_names = self.all_sensor_names
if self.room_numbers == None:
self.room_numbers = self.all_room_numbers
if self.room_names == None:
self.room_names = self.all_room_names
if self.time_from == None:
# time_from = Scraper._time_now() - 86400000 # previous 24 hours
# time_from = Scraper._time_now() - 604800000 # previous week
self.time_from = 1580920305102 # from first sensor reading
if self.time_to == None:
self.time_to = Scraper._time_now()
if self.parameters == None:
self.parameters = self.param_list
if self.overlay == None and len(self.sensor_numbers) > 1:
self.overlay = 1
elif self.overlay == None:
self.overlay = 0
if self.aggregate == None:
self.aggregate = 0
if self.seperate == None:
self.seperate = 1
return
