def test_send_batch_calls(httpserver, response):
""" testing issuing calls via Batch interface
"""
httpserver.serve_content(dumps(response))
client = JSONRPC(httpserver.url)
with pytest.raises(ProtocolError) as exc:
with client.batch() as batch:
batch = Batch(client)
batch.call("foo")
assert str(exc.value) == 'Expected a list of response from the server'
response = [response, {'jsonrpc': '2.0', 'result': 'test-result', 'id': 3}]
httpserver.serve_content(dumps(response))
client.call_id = 1
with client.batch() as foo:
foo.call("Foo")
foo.call("FFF")
assert foo.get_result(2) is None
assert foo.get_result(3) == 'test-result'
with pytest.raises(KeyError) as exc:
foo.get_result(1111)
expected_message = 'No such call_id in response'
if six.PY2:
assert str(exc.value.message) == expected_message
elif six.PY3:
assert exc.value.args[0] == expected_message
def test_batch_factory():
""" testing Batch object creation
"""
client = JSONRPC(None)
batch = client.batch()
assert isinstance(batch, Batch)
batch.call("foo")
batch.call("bar")
assert len(batch.calls) == 2
class Client(object):
"""The main interface to the dataplicity server"""
def __init__(self, conf_paths, check_firmware=True, log=None):
self.check_firmware = check_firmware
if log is None:
log = logging.getLogger('dataplicity.client')
self.log = log
conf_paths = conf_paths or []
if not isinstance(conf_paths, list):
conf_paths = [conf_paths]
self.conf_paths = conf_paths
self._sync_lock = Lock()
self._init()
def _init(self):
try:
conf = self.conf = settings.read(*self.conf_paths)
conf_dir = os.path.dirname(conf.path)
self.firmware_conf = settings.read_default(
os.path.join(conf_dir, 'firmware.conf'))
self.current_firmware_version = int(
self.firmware_conf.get('firmware', 'version', 1))
self.firmware_path = conf.get('firmware', 'path')
self.log.info('running firmware {:010}'.format(
self.current_firmware_version))
self.rpc_url = conf.get('server', 'url', constants.SERVER_URL)
self.push_url = conf.get('server', 'push_url', constants.PUSH_URL)
self.remote = JSONRPC(self.rpc_url)
self.serial = conf.get('device', 'serial', None)
if self.serial is None:
self.serial = serial.get_default_serial()
self.log.info('auto generated device serial, %r', self.serial)
self.name = conf.get('device', 'name', self.serial)
self.log.info('device name "{}", "serial" {}'.format(
self.name, self.serial))
self.device_class = conf.get('device', 'class')
self.subdomain = conf.get('device', 'subdomain', None)
if not self.subdomain:
# try legacy settings
self.subdomain = conf.get('device', 'company', None)
self._auth_token = conf.get('device', 'auth')
self.auto_register_info = conf.get('device', 'auto_device_text',
None)
self.tasks = TaskManager.init_from_conf(self, conf)
self.samplers = SamplerManager.init_from_conf(self, conf)
self.livesettings = LiveSettingsManager.init_from_conf(self, conf)
self.timelines = TimelineManager.init_from_conf(self, conf)
self.sample_now = self.samplers.sample_now
self.sample = self.samplers.sample
self.get_timeline = self.timelines.get_timeline
except:
self.log.exception('unable to start')
raise
def connect_wait(self, closing_event, sync_func):
def do_wait():
for _ in xrange(CONNECT_WAIT):
if closing_event.is_set():
return True
sleep(1)
return False
try:
while not closing_event.is_set():
if not self.serial or not self._auth_token or not self.push_url:
do_wait()
continue
push_url = "{}?serial={}&auth={}".format(
self.push_url, self.serial, self._auth_token)
response = _wait_on_url(push_url, closing_event, self.log)
if response is not None:
response = response.strip()
if response == "SYNCNOW":
self.log.debug("server requested sync")
try:
sync_func()
except:
self.log.exception("push sync callback failed")
elif response == "TIMEOUT":
# Timed out, just connect again
continue
else:
self.log.debug('push wait received: "{}"'.format(response))
# Some error occurred, or invalid response
# Wait for a moment, so as not to hammer the server
do_wait()
finally:
self.log.debug('connect_wait thread exiting')
@property
def auth_token(self):
"""get the auth_token, which may be in dataplicity.cfg, or reference another file"""
if self._auth_token.startswith('file:'):
auth_token_path = self._auth_token.split(':', 1)[-1]
try:
with open(auth_token_path, 'rt') as f:
auth_token = f.read()
except IOError:
return None
else:
self._auth_token = auth_token
return auth_token
else:
return self._auth_token
def get_settings(self, name):
self.livesettings.get(name, reload=True)
def sync(self):
# Serialize syncing
with self._sync_lock:
self._sync()
def _sync(self):
start = time()
self.log.debug("syncing...")
# If we don't have an auth_token, we are waiting for permission
if not self.auth_token and self._auth_token.startswith('file:'):
auth_token_path = self._auth_token.split(':', 1)[-1]
approval = self.remote.call('device.check_approval',
device_class=self.device_class,
subdomain=self.subdomain,
serial=self.serial,
name=self.name,
info=self.auto_register_info)
if approval['state'] != 'approved':
# Device is not yet approved, can't continue with sync
state = approval['state']
if state == 'pending':
# Waiting on approval
self.log.debug('device approval pending...')
else:
# denied
self.log.error('device approval {}'.format(state))
return
else:
# Device is approved. Write the auth_token.
try:
os.makedirs(os.path.dirname(auth_token_path))
except OSError:
pass
try:
with open(auth_token_path, 'wb') as f:
self._auth_token = approval['auth_token']
f.write(self._auth_token)
except:
self.log.exception('unable to write auth token')
# Will error out on the next command
if not self.auth_token:
self.log.error(
"sync failed -- no auth token, have you run 'dataplicity register'?"
)
return
if not os.path.exists(self.firmware_path):
self.log.debug("no firmware installed")
try:
self.deploy()
except:
self.log.exception("unable to deploy firmware")
raise ForceRestart("new firmware")
samplers_updated = []
random.seed()
sync_id = ''.join(
random.choice('abcdefghijklmnopqrstuvwxyz0123456789')
for _ in xrange(12))
with self.remote.batch() as batch:
# Authenticate
batch.call_with_id('authenticate_result',
'device.check_auth',
device_class=self.device_class,
serial=self.serial,
auth_token=self.auth_token,
sync_id=sync_id)
# Tell the server which firmware we're running
batch.call_with_id('set_firmware_result',
'device.set_firmware',
version=self.current_firmware_version)
# Check for new firmware (if required)
if self.check_firmware:
batch.call_with_id(
'firmware_result',
'device.check_firmware',
current_version=self.current_firmware_version)
# Add samples
for sampler_name in self.samplers.enumerate_samplers():
sampler = self.samplers.get_sampler(sampler_name)
samples = sampler.snapshot_samples()
if samples:
batch.call_with_id("samples.{}".format(sampler_name),
"device.add_samples",
device_class=self.device_class,
serial=self.serial,
sampler_name=sampler_name,
samples=samples)
samplers_updated.append(sampler_name)
else:
sampler.remove_snapshot()
# Update conf
conf_map = self.livesettings.contents_map
batch.call_with_id("conf_result",
"device.update_conf_map",
conf_map=conf_map)
# Update timeline(s)
if self.timelines:
for timeline in self.timelines:
batch.call_with_id('timeline_result_{}'.format(
timeline.name),
'device.add_events',
name=timeline.name,
events=timeline.get_events())
# get_result will throw exceptions with (hopefully) helpful error messages if they fail
batch.get_result('authenticate_result')
# If the server doesn't have the current firmware, we don't want to break the rest of the sync
try:
batch.get_result('set_firmware_result')
except Exception as e:
self.log.warning("unable to set firmware ({})".format(e))
# Remove snapshots that were successfully synced
# Unsuccessful snapshots remain on disk, so the next sync will re-attempt them.
for sampler_name in samplers_updated:
sampler = self.samplers.get_sampler(sampler_name)
try:
if not batch.get_result("samples.{}".format(sampler_name)):
self.log("failed to get sampler results '{}'".format(
sampler_name))
except Exception as e:
self.log.exception("error adding samples to {} ({})".format(
sampler_name, e))
else:
sampler.remove_snapshot()
try:
changed_conf = batch.get_result("conf_result")
except:
self.log.exception('error sending settings')
else:
if changed_conf:
self.livesettings.update(changed_conf, self.tasks)
changed_conf_names = ", ".join(sorted(changed_conf.keys()))
self.log.debug(
"settings file(s) changed: {}".format(changed_conf_names))
for timeline in self.timelines:
try:
timeline_result = batch.get_result('timeline_result_{}'.format(
timeline.name))
except:
self.log.exception('error sending timeline')
else:
timeline.clear_events(timeline_result)
ellapsed = time() - start
self.log.debug('sync complete {:0.2f}s'.format(ellapsed))
if self.check_firmware:
firmware_result = batch.get_result('firmware_result')
if firmware_result['current']:
self.log.debug('firmware is current')
else:
firmware_b64 = firmware_result['firmware']
device_class = firmware_result['device_class']
version = firmware_result['version']
self.log.debug(
"new firmware, version v{} for device class '{}'".format(
version, device_class))
self.log.info("installing firmware v{}".format(version))
install_path = firmware.install_encoded(
device_class, version, firmware_b64)
self.log.info(
'firmware installed in "{}"'.format(install_path))
comms.Comms().restart()
def deploy(self):
"""Deploy latest firmware"""
self.log.info("requesting firmware...")
with self.remote.batch() as batch:
batch.call_with_id('register_result',
'device.register',
auth_token=self.auth_token,
name=self.name or self.serial,
serial=self.serial,
device_class_name=self.device_class)
batch.call_with_id('auth_result',
'device.check_auth',
device_class=self.device_class,
serial=self.serial,
auth_token=self.auth_token)
batch.call_with_id('firmware_result', 'device.get_firmware')
try:
batch.get_result('register_result')
except Exception as e:
self.log.warning(e)
batch.get_result('auth_result')
fw = batch.get_result('firmware_result')
if not fw['firmware']:
self.log.warning('no firmware available!')
return False
version = fw['version']
firmware_bin = b64decode(fw['firmware'])
firmware_file = StringIO(firmware_bin)
firmware_fs = ZipFS(firmware_file)
dst_fs = OSFS(constants.FIRMWARE_PATH, create=True)
firmware.install(self.device_class, version, firmware_fs, dst_fs)
fw_path = dst_fs.getsyspath('/')
self.log.info("installed firmware {:010} to {}".format(
version, fw_path))
firmware.activate(self.device_class, version, dst_fs)
self.log.info("activated firmware {:010}".format(version))
class Client(object):
"""The main interface to the dataplicity server"""
def __init__(self, conf_paths, check_firmware=True, log=None):
self.check_firmware = check_firmware
if log is None:
log = logging.getLogger('dataplicity.client')
self.log = log
conf_paths = conf_paths or []
if not isinstance(conf_paths, list):
conf_paths = [conf_paths]
self.conf_paths = conf_paths
self._sync_lock = Lock()
self._init()
def _init(self):
try:
conf = self.conf = settings.read(*self.conf_paths)
conf_dir = os.path.dirname(conf.path)
self.firmware_conf = settings.read_default(os.path.join(conf_dir, 'firmware.conf'))
self.current_firmware_version = int(self.firmware_conf.get('firmware', 'version', 1))
self.firmware_path = conf.get('firmware', 'path')
self.log.info('running firmware {:010}'.format(self.current_firmware_version))
self.rpc_url = conf.get('server',
'url',
constants.SERVER_URL)
self.push_url = conf.get('server',
'push_url',
constants.PUSH_URL)
self.remote = JSONRPC(self.rpc_url)
self.serial = conf.get('device', 'serial', None)
if self.serial is None:
self.serial = serial.get_default_serial()
self.log.info('auto generated device serial, %r', self.serial)
self.name = conf.get('device', 'name', self.serial)
self.log.info('device name "{}", "serial" {}'.format(self.name, self.serial))
self.device_class = conf.get('device', 'class')
self.subdomain = conf.get('device', 'subdomain', None)
if not self.subdomain:
# try legacy settings
self.subdomain = conf.get('device', 'company', None)
self._auth_token = conf.get('device', 'auth')
self.auto_register_info = conf.get('device', 'auto_device_text', None)
self.tasks = TaskManager.init_from_conf(self, conf)
self.samplers = SamplerManager.init_from_conf(self, conf)
self.livesettings = LiveSettingsManager.init_from_conf(self, conf)
self.timelines = TimelineManager.init_from_conf(self, conf)
self.sample_now = self.samplers.sample_now
self.sample = self.samplers.sample
self.get_timeline = self.timelines.get_timeline
except:
self.log.exception('unable to start')
raise
def connect_wait(self, closing_event, sync_func):
def do_wait():
for _ in xrange(CONNECT_WAIT):
if closing_event.is_set():
return True
sleep(1)
return False
try:
while not closing_event.is_set():
if not self.serial or not self._auth_token or not self.push_url:
do_wait()
continue
push_url = "{}?serial={}&auth={}".format(self.push_url,
self.serial,
self._auth_token)
response = _wait_on_url(push_url, closing_event, self.log)
if response is not None:
response = response.strip()
if response == "SYNCNOW":
self.log.debug("server requested sync")
try:
sync_func()
except:
self.log.exception("push sync callback failed")
elif response == "TIMEOUT":
# Timed out, just connect again
continue
else:
self.log.debug('push wait received: "{}"'.format(response))
# Some error occurred, or invalid response
# Wait for a moment, so as not to hammer the server
do_wait()
finally:
self.log.debug('connect_wait thread exiting')
@property
def auth_token(self):
"""get the auth_token, which may be in dataplicity.cfg, or reference another file"""
if self._auth_token.startswith('file:'):
auth_token_path = self._auth_token.split(':', 1)[-1]
try:
with open(auth_token_path, 'rt') as f:
auth_token = f.read()
except IOError:
return None
else:
self._auth_token = auth_token
return auth_token
else:
return self._auth_token
def get_settings(self, name):
self.livesettings.get(name, reload=True)
def sync(self):
# Serialize syncing
with self._sync_lock:
self._sync()
def _sync(self):
start = time()
self.log.debug("syncing...")
# If we don't have an auth_token, we are waiting for permission
if not self.auth_token and self._auth_token.startswith('file:'):
auth_token_path = self._auth_token.split(':', 1)[-1]
approval = self.remote.call('device.check_approval',
device_class=self.device_class,
subdomain=self.subdomain,
serial=self.serial,
name=self.name,
info=self.auto_register_info)
if approval['state'] != 'approved':
# Device is not yet approved, can't continue with sync
state = approval['state']
if state == 'pending':
# Waiting on approval
self.log.debug('device approval pending...')
else:
# denied
self.log.error('device approval {}'.format(state))
return
else:
# Device is approved. Write the auth_token.
try:
os.makedirs(os.path.dirname(auth_token_path))
except OSError:
pass
try:
with open(auth_token_path, 'wb') as f:
self._auth_token = approval['auth_token']
f.write(self._auth_token)
except:
self.log.exception('unable to write auth token')
# Will error out on the next command
if not self.auth_token:
self.log.error("sync failed -- no auth token, have you run 'dataplicity register'?")
return
if not os.path.exists(self.firmware_path):
self.log.debug("no firmware installed")
try:
self.deploy()
except:
self.log.exception("unable to deploy firmware")
raise ForceRestart("new firmware")
samplers_updated = []
random.seed()
sync_id = ''.join(random.choice('abcdefghijklmnopqrstuvwxyz0123456789') for _ in xrange(12))
with self.remote.batch() as batch:
# Authenticate
batch.call_with_id('authenticate_result',
'device.check_auth',
device_class=self.device_class,
serial=self.serial,
auth_token=self.auth_token,
sync_id=sync_id)
# Tell the server which firmware we're running
batch.call_with_id('set_firmware_result',
'device.set_firmware',
version=self.current_firmware_version)
# Check for new firmware (if required)
if self.check_firmware:
batch.call_with_id('firmware_result',
'device.check_firmware',
current_version=self.current_firmware_version)
# Add samples
for sampler_name in self.samplers.enumerate_samplers():
sampler = self.samplers.get_sampler(sampler_name)
samples = sampler.snapshot_samples()
if samples:
batch.call_with_id("samples.{}".format(sampler_name),
"device.add_samples",
device_class=self.device_class,
serial=self.serial,
sampler_name=sampler_name,
samples=samples)
samplers_updated.append(sampler_name)
else:
sampler.remove_snapshot()
# Update conf
conf_map = self.livesettings.contents_map
batch.call_with_id("conf_result",
"device.update_conf_map",
conf_map=conf_map)
# Update timeline(s)
if self.timelines:
for timeline in self.timelines:
batch.call_with_id('timeline_result_{}'.format(timeline.name),
'device.add_events',
name=timeline.name,
events=timeline.get_events())
# get_result will throw exceptions with (hopefully) helpful error messages if they fail
batch.get_result('authenticate_result')
# If the server doesn't have the current firmware, we don't want to break the rest of the sync
try:
batch.get_result('set_firmware_result')
except Exception as e:
self.log.warning("unable to set firmware ({})".format(e))
# Remove snapshots that were successfully synced
# Unsuccessful snapshots remain on disk, so the next sync will re-attempt them.
for sampler_name in samplers_updated:
sampler = self.samplers.get_sampler(sampler_name)
try:
if not batch.get_result("samples.{}".format(sampler_name)):
self.log("failed to get sampler results '{}'".format(sampler_name))
except Exception as e:
self.log.exception("error adding samples to {} ({})".format(sampler_name, e))
else:
sampler.remove_snapshot()
try:
changed_conf = batch.get_result("conf_result")
except:
self.log.exception('error sending settings')
else:
if changed_conf:
self.livesettings.update(changed_conf, self.tasks)
changed_conf_names = ", ".join(sorted(changed_conf.keys()))
self.log.debug("settings file(s) changed: {}".format(changed_conf_names))
for timeline in self.timelines:
try:
timeline_result = batch.get_result('timeline_result_{}'.format(timeline.name))
except:
self.log.exception('error sending timeline')
else:
timeline.clear_events(timeline_result)
ellapsed = time() - start
self.log.debug('sync complete {:0.2f}s'.format(ellapsed))
if self.check_firmware:
firmware_result = batch.get_result('firmware_result')
if firmware_result['current']:
self.log.debug('firmware is current')
else:
firmware_b64 = firmware_result['firmware']
device_class = firmware_result['device_class']
version = firmware_result['version']
self.log.debug("new firmware, version v{} for device class '{}'".format(version, device_class))
self.log.info("installing firmware v{}".format(version))
install_path = firmware.install_encoded(device_class, version, firmware_b64)
self.log.info('firmware installed in "{}"'.format(install_path))
comms.Comms().restart()
def deploy(self):
"""Deploy latest firmware"""
self.log.info("requesting firmware...")
with self.remote.batch() as batch:
batch.call_with_id('register_result',
'device.register',
auth_token=self.auth_token,
name=self.name or self.serial,
serial=self.serial,
device_class_name=self.device_class)
batch.call_with_id('auth_result',
'device.check_auth',
device_class=self.device_class,
serial=self.serial,
auth_token=self.auth_token)
batch.call_with_id('firmware_result',
'device.get_firmware')
try:
batch.get_result('register_result')
except Exception as e:
self.log.warning(e)
batch.get_result('auth_result')
fw = batch.get_result('firmware_result')
if not fw['firmware']:
self.log.warning('no firmware available!')
return False
version = fw['version']
firmware_bin = b64decode(fw['firmware'])
firmware_file = StringIO(firmware_bin)
firmware_fs = ZipFS(firmware_file)
dst_fs = OSFS(constants.FIRMWARE_PATH, create=True)
firmware.install(self.device_class,
version,
firmware_fs,
dst_fs)
fw_path = dst_fs.getsyspath('/')
self.log.info("installed firmware {:010} to {}".format(version, fw_path))
firmware.activate(self.device_class, version, dst_fs)
self.log.info("activated firmware {:010}".format(version))