def check_for_remigration(to_do_list):
"""
Make sure we're not queueing up a device we've already migrated
"""
tbapi_old = TbApi(birdhouse_utils.make_mothership_url(old_server_ip),
thingsboard_username, thingsboard_password)
tbapi_new = TbApi(birdhouse_utils.make_mothership_url(new_server_ip),
thingsboard_username, thingsboard_password)
old_client = create_client(old_server_ip)
new_client = create_client(new_server_ip)
print(
"Conducting remigration checks (to make sure device hasn't already been migrated to new server)"
)
for num in to_do_list:
print(".", end="")
name = birdhouse_utils.make_device_name(num)
if not device_has_data(tbapi_old, old_client, name):
print(
f"\nThere is no data on the old server for device {name}. Nothing to migrate."
)
exit()
if device_has_been_migrated(tbapi_old, old_client, tbapi_new,
new_client, name):
print(f"\nDevice {name} has already been migrated.")
exit()
print()
def main():
tbapi = TbApi(motherShipUrl, username, password)
# Lookup missing fields, such as zip, lat, and lon
birdhouse_utils.update_customer_data(cust_info)
if cust_info["lat"] is None or cust_info["lon"] is None:
print("Must have valid lat/lon to continue!")
exit(1)
name = birdhouse_utils.make_device_name(birdhouse_number)
cust = tbapi.get_customer(name)
devices = tbapi.get_customer_devices(cust)
print(devices)
device = tbapi.get_device_by_name(name)
customer = tbapi.update_customer(cust, None, cust_info["address"],
cust_info["address2"], cust_info["city"],
cust_info["state"], cust_info["zip"],
cust_info["country"])
server_attributes = {
"latitude": cust_info["lat"],
"longitude": cust_info["lon"],
"address": birdhouse_utils.one_line_address(cust_info)
}
tbapi.set_server_attributes(device, server_attributes)
exit()
cust = tbapi.get_customer(name)
def ensure_devices_exist(device_nums):
"""
Ensure all devices we're going to process exist on both the source and destination machines
"""
tbapi_old = TbApi(birdhouse_utils.make_mothership_url(old_server_ip),
thingsboard_username, thingsboard_password)
tbapi_new = TbApi(birdhouse_utils.make_mothership_url(new_server_ip),
thingsboard_username, thingsboard_password)
print(
"Making sure every device we want to migrate exists on both source and desitnation machines"
)
for num in device_nums:
print(".", end="")
name = birdhouse_utils.make_device_name(num)
old_device = tbapi_old.get_device_by_name(name)
if old_device is None:
print(f"\nDevice {name} does not exist on old server!")
exit()
new_device = tbapi_new.get_device_by_name(name)
if new_device is None:
print(f"\nDevice {name} does not exist on new server!")
exit()
old_key = tbapi_old.get_id(old_device)
new_key = tbapi_new.get_id(new_device)
if old_key != new_key:
print("\nDevice keys are different on old and new servers!")
exit()
print()
def main():
mothership_url = birdhouse_utils.make_mothership_url(args, config)
tbapi = TbApi(mothership_url, thingsboard_username, thingsboard_password)
for num in args["<num>"]:
print(f"Retrieving details for device {num}... ", end='', flush=True)
name = birdhouse_utils.make_device_name(num)
device = tbapi.get_device_by_name(name)
if device is None:
print(f"Failed.\nCould not find device {num}... Aborting.")
exit()
token = tbapi.get_device_token(device)
print("done.")
print(token)
def make_params(nums):
mothership_url = birdhouse_utils.make_mothership_url(args)
tbapi = TbApi(mothership_url, thingsboard_username, thingsboard_password)
params = []
for num in nums:
print(f"Retrieving details for device {num}... ", end='', flush=True)
device_name = birdhouse_utils.make_device_name(num)
dash_name = birdhouse_utils.make_dash_name(num)
device = tbapi.get_device_by_name(device_name)
dash = tbapi.get_dashboard_by_name(dash_name)
dash_url = tbapi.get_public_dash_url(dash)
tiny_url = make_tiny_url(dash_url)
if device is None:
print(f"Failed.\nCould not find device {num}... Aborting.")
exit()
token = tbapi.get_device_token(device)
params.append((
birdhouse_utils.get_sensor_type(num)[1],
birdhouse_utils.make_device_number(num),
token,
tiny_url
))
print("done.")
return params
def find_unmigrated_devices(old_ip, new_ip):
tbapi_old = TbApi(birdhouse_utils.make_mothership_url(old_ip),
thingsboard_username, thingsboard_password)
tbapi_new = TbApi(birdhouse_utils.make_mothership_url(new_ip),
thingsboard_username, thingsboard_password)
old_client = create_client(old_ip)
new_client = create_client(new_ip)
devices_on_old = tbapi_old.get_all_devices()
devices_on_new = tbapi_new.get_all_devices()
unmigrated_ready_to_go, migrated, unmigrated, no_data, no_data_2, new_only = list(
), list(), list(), list(), list(), list()
for dev in devices_on_old:
name = dev["name"]
print(f"Device {name}")
has_telem_on_old = has_telemetry(tbapi_old, dev)
has_telem_on_new = has_telemetry(tbapi_new, dev)
if not device_has_data(tbapi_old, old_client, name):
no_data_2.append(name)
if has_telem_on_old:
if has_telem_on_new:
# Telemetry exists on both servers; has it been migrated, or merely switched itself over?
if device_has_been_migrated(tbapi_old, old_client, tbapi_new,
new_client, name):
migrated.append(name)
else:
unmigrated_ready_to_go.append(name)
else: # on old, not on new
unmigrated.append(name)
else: # not old
if has_telem_on_new:
new_only.append(name)
else:
no_data.append(name)
for dev in devices_on_new:
if dev not in devices_on_old:
new_only.append(dev["name"])
print(f"Migrated {sorted(migrated)}")
print(
f"Unmigrated and unswitched (possibly no longer collecting data): {sorted(unmigrated)}"
)
print(f"Ready to migrate: {sorted(unmigrated_ready_to_go)}")
# Not sure which of these is more reliable -- they generate their results using different methods
print(f"No data: {sorted(no_data)}")
print(f"No data2: {sorted(no_data_2)}")
print(f"New only, nothing to do: {sorted(new_only)}")
exit()
def main():
global args, tbapi, port
tbapi = TbApi(motherShipUrl, config.thingsboard_username, config.thingsboard_password)
parser = argparse.ArgumentParser(description='Configurate your birdhouse!')
parser.add_argument('--number', '-n', metavar='NNN', type=str, help='the number of your birdhouse')
parser.add_argument('--localssid', metavar='ssid', type=str, help='The local SSID for the birdhouse wifi')
parser.add_argument('--localpass', metavar='pass', type=str, help='The local password for the birdhouse wifi')
parser.add_argument('--wifissid', metavar='ssid', type=str, help='The SSID for the local wifi')
parser.add_argument('--wifipass', metavar='pass', type=str, help='The password for the local wifi')
parser.add_argument('--mqtturl', metavar='url', type=str, help='The url for sending telemetry to')
parser.add_argument('--mqttport', metavar='port', type=str, help='The server\'s MQTT port')
parser.add_argument('--nocompile', action='store_true', help="skip compilation, upload stored binary")
parser.add_argument('--noupload', action='store_true', help="skip compilation and upload, rely on previously uploaded binary")
args = parser.parse_args()
# Find out what port the Birdhouse is on:
try:
port = get_best_guess_port()
if not port:
print("Plug birdhouse in, please!")
port = polling.poll(lambda: get_best_guess_port(), timeout=30, step=0.5)
except Exception as ex:
print("Could not find any COM ports")
raise ex
# compile_and_upload_firmware(compile=(not args.nocompile), upload=(not args.noupload))
# Validate
if args.number is not None:
if len(args.number) != 3:
print("Birdhouse number must be 3 digits (include leading 0s)")
sys.exit(1)
# else we'll try to get it from the birdhouse itself
# sys.exit()
runUi()
#!/usr/bin/env python
import json
import web # sudo pip3 install git+https://github.com/webpy/webpy#egg=web.py
import googlemaps # sudo pip install googlemaps
import geopy.distance # sudo pip install geopy
import re
import os
import base64
# pip install git+git://github.com/eykamp/thingsboard_api_tools.git --upgrade
# sudo pip install git+git://github.com/eykamp/thingsboard_api_tools.git --upgrade
from thingsboard_api_tools import TbApi
from redlight_greenlight_config import motherShipUrl, username, password, data_encoding, google_geolocation_key
tbapi = TbApi(motherShipUrl, username, password)
gmaps = googlemaps.Client(key=google_geolocation_key)
urls = ('/', 'set_led_color', '/hotspots/', 'handle_hotspots', '/update/',
'handle_update')
app = web.application(urls, globals())
class handle_hotspots:
def POST(self):
try:
data = web.data()
decoded = data.decode(data_encoding)
incoming_data = json.loads(decoded)
# print(incoming_data)
CFG = toml.load(configpath)
except FileNotFoundError:
show_config_help(configpath)
print("Using defaults...")
CFG = dict(motherShipUrl="FIXME",
username="******",
password="******",
data_encoding="utf-8",
google_geolocation_key="FIXME",
firmware_images_folder="FIXME")
print(CFG)
return CFG
CFG = load_config(build_config_path())
tbapi = TbApi(CFG['motherShipUrl'], CFG['username'], CFG['password'])
app = Flask(__name__)
def get_immediate_subdirectories(a_dir):
return [
name for name in os.listdir(a_dir)
if os.path.isdir(os.path.join(a_dir, name))
]
def get_geolocate():
# helper function to assist with testability, so it can be mocked
return googlemaps.Client(key=CFG['google_geolocation_key']).geolocate
def main():
cleanup = True
tbapi = TbApi(motherShipUrl, username, password)
# Get a definition of our template dashboard
template_dash = tbapi.get_dashboard_by_name(dashboard_template_name)
dash_def = tbapi.get_dashboard_definition(tbapi.get_id(template_dash))
# Lookup missing fields, such as zip, lat, and lon
update_customer_data()
if cust_lat is None or cust_lon is None:
print("Must have valid lat/lon in order to add device!")
exit(1)
# Create new customer and device records on the server
customer = tbapi.add_customer(cust_name, cust_address, cust_address2,
cust_city, cust_state, cust_zip,
cust_country, cust_email, cust_phone)
server_attributes = {"latitude": cust_lat, "longitude": cust_lon}
shared_attributes = {"LED": "Unknown", "nonce": 0}
device = tbapi.add_device(make_device_name(cust_name), sensor_type,
shared_attributes, server_attributes)
device_id = tbapi.get_id(device)
# We need to store the device token as a server attribute so our REST services can get access to it
device_token = tbapi.get_device_token(device_id)
server_attributes = {"device_token": device_token}
tbapi.set_server_attributes(device_id, server_attributes)
# Upate the dash def. to point at the device we just created (modifies dash_def)
update_dash_def(dash_def, cust_name, device_id)
# Create a new dash with our definition, and assign it to the new customer
dash = tbapi.create_dashboard_for_customer(cust_name + ' Dash', dash_def)
tbapi.assign_dash_to_user(tbapi.get_id(dash), tbapi.get_id(customer))
if cleanup:
# input("Press Enter to continue...") # Don't run from Sublime with this line enabled!!!
print("Cleaning up!")
tbapi.delete_dashboard(tbapi.get_id(dash))
tbapi.delete_device(device_id)
tbapi.delete_customer_by_id(tbapi.get_id(customer))
from thingsboard_api_tools import TbApi # pip install git+git://github.com/eykamp/thingsboard_api_tools.git --upgrade
from config import thingsboard_username, thingsboard_password # You'll need to create this... Be sure to gitignore it!
import config
args = docopt(__doc__)
winscp_profile = args['<winscpprofile>']
devices = args['<devices>'] if not args['all'] else "all"
clean = args['--clean']
build_only = args['build']
# clean = True
# build_only = False
mothership_url = birdhouse_utils.make_mothership_url(args, config)
tbapi = TbApi(mothership_url, thingsboard_username, thingsboard_password)
winscp_program_location = r"c:\Program Files (x86)\WinSCP\WinSCP.com"
arduino_build_exe_location = r"c:\Program Files (x86)\Arduino\arduino_debug.exe"
source_name = "firmware"
source_file = source_name + ".ino"
build_folder = r"C:\Temp\BirdhouseFirmwareBuildFolder" # Where we store the firmware image we're building
remote_dir = "/sensorbot/firmware_images" # Where we store the firmware images on the update server
def main():
version = extract_version(source_file)
build_target_file = source_name + "_" + str(version) + ".bin"
# Important: Use the .com version here, not the .exe
def main(settings):
print("Sensorbot Device Provisioning Script Version " + __version__)
if upload_img or should_upload_firmware:
settings.esp = find_birdhouse_port()
if settings.esp is None:
if ui_mode:
start_ui_no_device()
else:
exit()
if upload_img:
if not os.path.isfile(firmware_image):
print(f"Could not find firmware image {firmware_image}")
print("Aborting.")
exit()
upload_firmware(print, settings.esp, firmware_image)
exit()
validate_led_style(settings.led_style)
# Instantiate our API helper if needed
if should_set_up_thingsboard:
tbapi = TbApi(mothership_url, thingsboard_username,
thingsboard_password)
if should_delete_only:
birdhouse_utils.purge_server_objects_for_device(
tbapi, settings.birdhouse_number)
exit()
device_name = birdhouse_utils.make_device_name(settings.birdhouse_number)
if settings.birdhouse_number is not None:
print("Configuring device '" + device_name + "'\n")
# Get the port first to ensure our local house is in order before we validate things over the network
validated_token = False
if should_upload_firmware:
if settings.birdhouse_number is not None and settings.device_token is not None:
if not validate_device_token(settings.birdhouse_number,
settings.device_token):
print(
"Passed an invalid device token for the specified device.")
print("Aborting.")
exit()
validated_token = True
if should_create_thingsboard_objects:
settings.device_token = create_server_objects(
tbapi, settings.birdhouse_number
) # This will fail if server objects already exist
if testmode:
input("Press Enter to cleanup created objects...")
settings.birdhouse_utils.purge_server_objects_for_device(
tbapi, settings.birdhouse_number)
exit()
# assign_device_to_public_user(token, device_id)
if ui_mode:
start_ui(tbapi, settings.esp._port)
exit()
if not validated_token:
settings.device_token = get_or_validate_device_token(
tbapi, settings.birdhouse_number, settings.device_token,
should_create_thingsboard_objects)
if should_upload_firmware:
upload_firmware_and_configure(print, settings)
if attributes_only:
set_params(print, settings)
def main():
mothership_url = birdhouse_utils.make_mothership_url(args, config)
tbapi = TbApi(mothership_url, thingsboard_username, thingsboard_password)
print(f"Retrieving template dashboard {template_dash}... ",
end='',
flush=True)
template_dash_def = tbapi.get_dashboard_definition(
tbapi.get_dashboard_by_name(template_dash))
# We also need the id of the device being swapped out
template_device_id = tbapi.get_id(
tbapi.get_device_by_name(
birdhouse_utils.make_device_name(args["<device>"])))
print(" done.")
all_devices = tbapi.get_devices_by_name(copy_to_pattern)
for device in all_devices:
num = birdhouse_utils.get_device_number_from_name(device["name"])
print(f"Updating dashboard for {device['name']}")
dash_name_being_replaced = birdhouse_utils.make_dash_name(num)
device_name = birdhouse_utils.make_device_name(num)
device = tbapi.get_device_by_name(device_name)
device_id = tbapi.get_id(device)
# The dash we are replacing:
dash_being_replaced = tbapi.get_dashboard_by_name(
dash_name_being_replaced)
dash_id_being_replaced = tbapi.get_id(dash_being_replaced)
dash_def = tbapi.get_dashboard_definition(
tbapi.get_dashboard_by_name(
template_dash)) # dash_def will be modified
birdhouse_utils.reassign_dash_to_new_device(dash_def,
dash_name_being_replaced,
template_device_id,
device_id, device_name)
dash_def["id"]["id"] = dash_id_being_replaced
# del_humidity(dash_def)
# exit()
tbapi.save_dashboard(dash_def)
# Software. # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. import re import datetime, time, pytz import logging import deq_tools # pip install deq_tools from thingsboard_api_tools import TbApi # pip install git+git://github.com/eykamp/thingsboard_api_tools.git --upgrade from config import motherShipUrl, username, password, deq_logfile # You'll need to create this... Be sure to gitignore it! tbapi = TbApi(motherShipUrl, username, password) device_name = 'DEQ (SEL)' deq_tz_name = 'US/Pacific' logging.basicConfig(filename=deq_logfile, format='%(asctime)s %(message)s', level=logging.INFO) # WARN, INFO, DEBUG # Data is stored as if it were coming from one of our devices device = tbapi.get_device_by_name(device_name) device_token = tbapi.get_device_token(tbapi.get_id(device)) # This is the earliest timestamp we're interested in. The first time we run this script, all data since this date will be imported. # Making the date unnecessarily early will make this script run very slowly. earliest_ts = "2018/04/28T00:00" # DEQ uses ISO datetime format: YYYY/MM/DDTHH:MM
def verify_devices_remapped(device_nums,
min_interval=120,
age_considered_offline=7 * DAYS): # In seconds
"""
Look at the ages of the most recent telemetry, and ensure each device is sending current telemetry to the new server.
The theory is that once a device is talking to the new server, it will never go back to the old, and we can safely
shift all telemetry from the old server.
min_interval is the minimum time, in seconds, that the new server must be ahead of the old server for this check to pass.
Note that if the most recent telemetry on the old server is older than age_considered_offline, we'll assume the device is out of contact
(perhaps disconnceted or off), and we'll go ahead and migrate the data over.
"""
tbapi_old = TbApi(birdhouse_utils.make_mothership_url(old_server_ip),
thingsboard_username, thingsboard_password)
tbapi_new = TbApi(birdhouse_utils.make_mothership_url(new_server_ip),
thingsboard_username, thingsboard_password)
old_client = create_client(old_server_ip)
new_client = create_client(new_server_ip)
print(
"Verifying devices have attached themselves to the new server and aren't still sending data to the old"
)
now = int(time.time() * 1000)
for num in device_nums:
print(".", end="")
name = birdhouse_utils.make_device_name(num)
latest_old = get_latest_telemetry_date(tbapi_old, old_client,
name) or 0
if latest_old == 0: # There is no data on the old server; no need to migrate, but no problem moving forward, either
continue
latest_new = get_latest_telemetry_date(tbapi_new, new_client,
name) or 0
age_of_old = int(
(now - latest_old) /
1000) # Age of most recent telemetry on old server in seconds
diff = int(
(latest_new - latest_old) / 1000
) # Diffrence in ages between most recent telmetry on new and old server, in seconds
if age_of_old < age_considered_offline and diff < min_interval:
print(
f"\nIt looks like device {name} is still active and hasn't switched to new server yet. Not ready to migrate data!"
)
print(
f"\tLast telemetry on old server was {format_time_delta(age_of_old)} ago."
)
if latest_new == 0:
print(
"\tThe device has not yet sent any data to the new server."
)
else:
print(
f"\tLast telemetry on new server was {format_time_delta(now - latest_new)} ago."
)
exit()
print()