def refresh_token(self):
"""Attempt to refresh out cloud token with iotile.cloud."""
if self.token_type != 'jwt':
raise DataError(
"Attempting to refresh a token that does not need to be refreshed",
token_type=self.token_type)
conf = ConfigManager()
domain = conf.get('cloud:server')
url = '{}/api/v1/auth/api-jwt-refresh/'.format(domain)
resp = self.api.session.post(url, json={'token': self.token})
if resp.status_code != 200:
raise ExternalError("Could not refresh token",
error_code=resp.status_code)
data = resp.json()
# Save token that we just refreshed to the registry and update our own token
self.token = data['token']
reg = ComponentRegistry()
reg.set_config('arch:cloud_token', self.token)
def _setup_proxies(self):
"""Load in proxy module objects for all of the registered components on this system."""
# Find all of the registered IOTile components and see if we need to add any proxies for them
reg = ComponentRegistry()
proxy_classes = reg.load_extensions('iotile.proxy',
class_filter=TileBusProxyObject,
product_name="proxy_module")
for name, obj in proxy_classes:
proxy_key = obj.__name__ + ':' + name
# awu_10/01/19 - we want to add all proxies even if duplicate but diff version
# if proxy_key in self._proxies:
# continue
self._proxies[proxy_key] = obj
# Check if this object matches a specific shortened name so that we can
# automatically match a hw module to a proxy without user intervention
try:
short_name = obj.ModuleName()
if short_name in self._name_map:
self._name_map[short_name].append(obj)
else:
self._name_map[short_name] = [obj]
except Exception: # pylint: disable=broad-except;
# We don't want this to die if someone loads a misbehaving plugin
self.logger.exception(
"Error importing misbehaving proxy object %s, skipping.",
obj)
def run(self):
component = os.path.abspath('.')
deps = self._get_dependencies(component)
reg = ComponentRegistry()
found = []
not_found = []
for name in deps:
try:
tile = reg.find_component(name)
found.append(tile)
except ArgumentError:
not_found.append(name)
deplines = []
for tile in found:
deplines.append('- %s' % tile.name)
for name in not_found:
deplines.append('- %s (NOT INSTALLED)' % name)
view = ViewList(deplines, 'dependencies-directive')
node = docutils.nodes.paragraph()
sphinx.util.nodes.nested_parse_with_titles(self.state, view, node)
return node.children
def _load_providers(self):
"""Load all config_variables providers using pkg_resources
"""
reg = ComponentRegistry()
for name, provider in reg.load_extensions('iotile.config_variables'):
try:
prefix, conf_vars = provider()
except (ValueError, TypeError) as exc:
raise ExternalError("Error loading config variables",
package=name,
error=str(exc))
for var in conf_vars:
if len(var) != 3 and len(var) != 4:
raise ExternalError(
"Error loading config variable, invalid length",
data=var,
package=name)
name = prefix + ':' + var[0]
if len(var) == 3:
var_obj = ConfigVariable(var[0], var[1], var[2], MISSING)
else:
var_obj = ConfigVariable(name, var[1], var[2], var[3])
if name in self._known_variables:
raise ExternalError(
"The same config variable was defined twice",
name=name)
self._known_variables[name] = var_obj
def check(self, depinfo, deptile, depsettings):
from iotile.core.dev.registry import ComponentRegistry
reg = ComponentRegistry()
try:
comp = reg.find_component(depinfo['name'])
except ArgumentError:
return True
#If the component does not have a matching version, we cannot assess up-to-date-ness
#FIXME: We should return that we cannot assess up to dateness, not that we are up to date
reqver = depinfo['required_version']
if not reqver.check(comp.parsed_version):
return True
#If the component in the registry has a higher version or a newer release date, it should
#be updated
if comp.parsed_version > deptile.parsed_version:
return False
if comp.release_date is not None and comp.release_date > deptile.release_date:
return False
return True
def update_local(self, path='.'):
"""Attempt to resolve all LOCAL dependencies in this IOTile by installing them into build/deps
"""
tile = IOTile(path)
if tile.release:
raise ArgumentError(
"Cannot update dependencies on a release mode tile that cannot have dependencies"
)
depdir = os.path.join(tile.folder, 'build', 'deps')
#FIXME: Read resolver_settings.json file
resolver_chain = DependencyResolverChain()
reg = ComponentRegistry()
for dep in tile.dependencies:
# Check each dependency to see if it's local
try:
local_tile = reg.find_component(dep['name'])
if local_tile.release:
continue
if local_tile is not None:
result = resolver_chain.update_dependency(tile, dep)
except ArgumentError:
continue
iprint("Resolving %s: %s" % (dep['name'], result))
def find_proxy_plugin(component, plugin_name):
""" Attempt to find a proxy plugin provided by a specific component
Args:
component (string): The name of the component that provides the plugin
plugin_name (string): The name of the plugin to load
Returns:
TileBuxProxyPlugin: The plugin, if found, otherwise raises DataError
"""
reg = ComponentRegistry()
plugins = reg.load_extensions('iotile.proxy_plugin',
comp_filter=component,
class_filter=TileBusProxyPlugin,
product_name='proxy_plugin')
for _name, plugin in plugins:
if plugin.__name__ == plugin_name:
return plugin
raise DataError(
"Could not find proxy plugin module in registered components or installed distributions",
component=component,
name=plugin_name)
def _setup_apps(self):
"""Load in all iotile app objects for all registered or installed components on this system."""
reg = ComponentRegistry()
app_classes = reg.load_extensions('iotile.app',
class_filter=IOTileApp,
product_name="app_module")
for _name, app in app_classes:
try:
matches = app.MatchInfo()
name = app.AppName()
for tag, ver_range, quality in matches:
if tag not in self._known_apps:
self._known_apps[tag] = []
self._known_apps[tag].append((ver_range, quality, app))
if name in self._named_apps:
self.logger.warning(
"Added an app module with an existing name, overriding previous app, name=%s",
name)
self._named_apps[name] = app
except Exception: #pylint: disable=broad-except;
# We don't want this to die if someone loads a misbehaving plugin
self.logger.exception(
"Error importing misbehaving app module %s, skipping.",
app)
def __init__(self, domain=None, username=None, **kwargs):
reg = ComponentRegistry()
conf = ConfigManager()
if domain is None:
domain = conf.get('cloud:server')
self.api = Api(domain=domain, **kwargs)
self._domain = self.api.domain
try:
token = reg.get_config('arch:cloud_token')
token_type = reg.get_config('arch:cloud_token_type', default='jwt')
self.api.set_token(token, token_type=token_type)
except ArgumentError:
# If we are interactive, try to get the user to login for a single
# session rather than making them call link_cloud to store a cloud token
if type_system.interactive:
username, password = self._prompt_user_pass(username, domain)
ok_resp = self.api.login(email=username, password=password)
if not ok_resp:
raise ExternalError("Could not login to %s as user %s" %
(domain, username))
else:
raise ExternalError(
"No stored iotile cloud authentication information",
suggestion=
'Call iotile config link_cloud with your username and password'
)
self.token = self.api.token
self.token_type = self.api.token_type
def test_nonexistent():
"""Make sure nonexistent config vars throw an error
"""
reg = ComponentRegistry()
with pytest.raises(ArgumentError):
reg.get_config('test1_nonexistent')
def registry():
reg = ComponentRegistry()
reg.clear()
reg.clear_components()
reg.add_component(os.getcwd())
yield reg
def impersonate_device(self, device_id):
"""Convert our token to a permanent device token.
This function is most useful for creating virtual IOTile devices whose access to iotile.cloud
is based on their device id, not any particular user's account.
There are a few differences between device tokens and user tokens:
- Device tokens never expire and don't need to be refreshed
- Device tokens are more restricted in what they can access in IOTile.cloud than user tokens
Args:
device_id (int): The id of the device that we want to get a token for.
"""
slug = device_id_to_slug(device_id)
token_type = IOTileCloud.DEVICE_TOKEN_TYPE
try:
resp = self.api.device(slug).key.get(
type=IOTileCloud.DEVICE_TOKEN_TYPE)
token = resp['key']
except RestHttpBaseException as exc:
raise ExternalError("Error calling method on iotile.cloud",
exception=exc,
response=exc.response.status_code)
self.api.set_token(token, token_type=token_type)
self.token = token
self.token_type = token_type
reg = ComponentRegistry()
reg.set_config('arch:cloud_token', self.token)
reg.set_config('arch:cloud_token_type', self.token_type)
reg.set_config('arch:cloud_device', slug)
def link_cloud(self, username=None, password=None, device_id=None):
"""Create and store a token for interacting with the IOTile Cloud API.
You will need to call link_cloud once for each virtualenv that
you create and want to use with any api calls that touch iotile cloud.
Note that this method is called on a ConfigManager instance
If you do not pass your username or password it will be prompted from
you securely on stdin.
If you are logging in for a user, the token will expire periodically and you
will have to relogin.
If you pass a device_id, you can obtain a limited token for that device
that will never expire, assuming you have access to that device.
Args:
username (string): Your iotile.cloud username. This is prompted
from stdin if not provided.
password (string): Your iotile.cloud password. This is prompted
from stdin if not provided.
device_id (int): Optional device id to obtain permanent credentials
for a device.
"""
reg = ComponentRegistry()
domain = self.get('cloud:server')
if username is None:
# Both python 2 and 3 require native strings to be passed into getpass
prompt_str = "Please enter your IOTile.cloud email: "
if sys.version_info.major < 3:
prompt_str = prompt_str.encode('utf-8')
username = input(prompt_str)
if password is None:
# Both python 2 and 3 require native strings to be passed into getpass
prompt_str = "Please enter your IOTile.cloud password: "******"Could not login to iotile.cloud as user %s" %
username)
reg.set_config('arch:cloud_user', cloud.username)
reg.set_config('arch:cloud_token', cloud.token)
reg.set_config('arch:cloud_token_type', cloud.token_type)
if device_id is not None:
cloud = IOTileCloud()
cloud.impersonate_device(device_id)
def registry():
reg = ComponentRegistry()
reg.clear()
reg.clear_components()
path = os.path.join(os.path.dirname(__file__), 'rpc_proxy.py')
reg.register_extension('iotile.proxy', 'virtual_tile', path)
yield reg
def _load_functions(self):
"""Load all config functions that should be bound to this ConfigManager
Config functions allow you to add functions that will appear under ConfigManager
but call your specified function. This is useful for adding complex configuration
behavior that is callable from the iotile command line tool
"""
reg = ComponentRegistry()
for _, conf_func in reg.load_extensions('iotile.config_function'):
try:
name = conf_func.__name__
self.add_function(name, conf_func)
except (ValueError, TypeError) as exc:
raise ExternalError("Error loading config function", name=name, error=str(exc))
def basic_cloud(mock_cloud_private_nossl):
"""A basic mock iotile.cloud initialized with default information.
There is a single project with 5 devices that have ids 1-5 and
a second inaccessible project with 1 device (id 6) in it.
"""
ComponentRegistry.SetBackingStore('memory')
domain, cloud = mock_cloud_private_nossl
reg = ComponentRegistry()
reg.set_config('cloud:server', domain)
reg.set_config('arch:cloud_token', 'JWT_USER')
reg.set_config('arch:cloud_token_type', 'jwt')
cloud.quick_add_user('*****@*****.**', 'test')
proj_id, _proj_slug = cloud.quick_add_project()
proj2_id, _proj_slug = cloud.quick_add_project()
devs = [
cloud.quick_add_device(proj_id, x, streamers=[100, 200])
for x in range(1, 6)
]
client = IOTileCloud()
cloud.quick_add_device(proj2_id)
cloud.quick_add_sg(slug="water-meter-v1-1-0", app_tag=123)
cloud.quick_add_sg(slug="water-meter-v1-1-1", app_tag=124)
cloud.quick_add_dt(slug="internaltestingtemplate-v0-1-0", os_tag=234)
cloud.quick_add_dt(slug="internaltestingtemplate-v0-1-1", os_tag=235)
yield client, proj_id, cloud
def find_proxy_plugin(component, plugin_name):
""" Attempt to find a proxy plugin provided by a specifc component
Args:
component (string): The name of the component that provides the plugin
plugin_name (string): The name of the plugin to load
Returns:
TileBuxProxPlugin: The plugin, if found, otherwise raises DataError
"""
reg = ComponentRegistry()
#Try to find plugin in an installed component, otherwise look through installed
#packages
try:
comp = reg.find_component(component)
plugins = comp.proxy_plugins()
for plugin in plugins:
objs = import_proxy(plugin, TileBusProxyPlugin)
for obj in objs:
if obj.__name__ == plugin_name:
return obj
except ArgumentError:
pass
for entry in pkg_resources.iter_entry_points('iotile.proxy_plugin'):
module = entry.load()
objs = [
obj for obj in itervalues(module.__dict__)
if inspect.isclass(obj) and issubclass(obj, TileBusProxyPlugin)
and obj != TileBusProxyPlugin
]
for obj in objs:
if obj.__name__ == plugin_name:
return obj
raise DataError(
"Could not find proxy plugin module in registered components or installed distributions",
component=component,
name=plugin_name)
def find_proxy(component, proxy_name=None, obj_type=TileBusProxyObject):
"""
Search through the registered component data base for a proxy or proxy plugin module provided by
a specific component optionally having a specific name.
Proxy modules must inherit from TileBusProxyObject. Proxy plugin modules must inherit from TileBusProxyPlugin
"""
# Find all of the registered IOTile components and see if we need to add any proxies for them
reg = ComponentRegistry()
comp = reg.find_component(component)
proxies = comp.proxy_modules()
if len(proxies) != 1:
raise DataError(
"Attempting to find proxy module from component that does not provide exactly 1 proxy",
proxies=proxies)
proxy_mod = proxies[0]
proxy_objs = import_proxy(proxy_mod, obj_type)
if len(proxy_objs) == 0:
raise DataError(
"Specified component did not define any proxies but said that it did provide them",
component=component)
elif len(proxy_objs) > 1 and proxy_name is None:
raise DataError(
"Sepcific component defined multiple proxy objects and a name was not specified",
component=component,
proxies=proxy_objs)
elif len(proxy_objs) == 1 and proxy_name is None:
return proxy_objs[0]
for obj in proxy_objs:
if obj.__name__ == proxy_name:
return obj
raise DataError("Named proxy could not be found",
component=component,
proxies=proxy_objs,
desired_name=proxy_name)
def registry(request):
ComponentRegistry.SetBackingStore(request.param)
reg = ComponentRegistry()
reg.clear()
yield reg
reg.clear()
def __init__(self, args):
cert = args.get('certificate', None)
key = args.get('private_key', None)
root = args.get('root_certificate', None)
endpoint = args.get('endpoint', None)
iamkey = args.get('iam_key', None)
iamsecret = args.get('iam_secret', None)
iamsession = args.get('iam_session', None)
use_websockets = args.get('use_websockets', False)
try:
if not use_websockets:
if cert is None:
raise ExternalError("Certificate for AWS IOT not passed in certificate key")
elif key is None:
raise ExternalError("Private key for certificate not passed in private_key key")
else:
if iamkey is None or iamsecret is None:
raise ExternalError("IAM Credentials need to be provided for websockets auth")
except ExternalError:
# If the correct information is not passed in, try and see if we get it from our environment
# try to pull in root certs, endpoint name and iam or cognito session information
reg = ComponentRegistry()
if endpoint is None:
endpoint = reg.get_config('awsiot-endpoint', default=None)
if root is None:
root = reg.get_config('awsiot-rootcert', default=None)
iamkey = reg.get_config('awsiot-iamkey', default=None)
iamsecret = reg.get_config('awsiot-iamtoken', default=None)
iamsession = reg.get_config('awsiot-session', default=None)
if iamkey is None or iamsecret is None:
raise
use_websockets = True
if root is None:
raise ExternalError("Root of certificate chain not passed in root_certificate key (and not in registry)")
elif endpoint is None:
raise ExternalError("AWS IOT endpoint not passed in endpoint key (and not in registry)")
self.websockets = use_websockets
self.iam_key = iamkey
self.iam_secret = iamsecret
self.iam_session = iamsession
self.cert = cert
self.key = key
self.root = root
self.endpoint = endpoint
self.client = None
self.sequencer = TopicSequencer()
self.queues = {}
self.wildcard_queues = []
self._logger = logging.getLogger(__name__)
def load_external_components(typesys):
"""Load all external types defined by iotile plugins.
This allows plugins to register their own types for type annotations and
allows all registered iotile components that have associated type libraries to
add themselves to the global type system.
"""
# Find all of the registered IOTile components and see if we need to add any type libraries for them
from iotile.core.dev.registry import ComponentRegistry
reg = ComponentRegistry()
modules = reg.list_components()
typelibs = reduce(lambda x, y: x+y, [reg.find_component(x).find_products('type_package') for x in modules], [])
for lib in typelibs:
if lib.endswith('.py'):
lib = lib[:-3]
typesys.load_external_types(lib)
def _setup_proxies(self):
"""Load in proxy module objects for all of the registered components on this system."""
# Find all of the registered IOTile components and see if we need to add any proxies for them
reg = ComponentRegistry()
modules = reg.list_components()
proxies = reduce(
lambda x, y: x + y,
[reg.find_component(x).proxy_modules() for x in modules], [])
proxy_classes = []
for prox in proxies:
proxy_classes += self._load_module_classes(prox,
TileBusProxyObject)
# Find all installed proxy objects through registered entry points
for entry in pkg_resources.iter_entry_points('iotile.proxy'):
mod = entry.load()
proxy_classes += [
x for x in itervalues(mod.__dict__)
if inspect.isclass(x) and issubclass(x, TileBusProxyObject)
and x != TileBusProxyObject
]
for obj in proxy_classes:
if obj.__name__ in self._proxies:
continue #Don't readd proxies that we already know about
self._proxies[obj.__name__] = obj
#Check if this object matches a specific shortened name so that we can
#automatically match a hw module to a proxy without user intervention
try:
short_name = obj.ModuleName()
if short_name in self._name_map:
self._name_map[short_name].append(obj)
else:
self._name_map[short_name] = [obj]
except Exception: #pylint: disable=broad-except;We don't want this to die if someone loads a misbehaving plugin
self.logger.exception(
"Error importing misbehaving proxy module, skipping.")
def _create_stream(self, force_adapter=None, record=None):
conn_string = None
port = self.port
if port is not None:
port = port.strip()
# Check if we're supposed to use a specific device adapter
if force_adapter is not None:
return AdapterStream(force_adapter, record=record)
# Attempt to find a DeviceAdapter that can handle this transport type
reg = ComponentRegistry()
for _, adapter_factory in reg.load_extensions(
'iotile.device_adapter', name_filter=self.transport):
return AdapterStream(adapter_factory(port), record=record)
raise HardwareError(
"Could not find transport object registered to handle passed transport type",
transport=self.transport)
def registry():
reg = ComponentRegistry()
reg.clear()
yield reg
reg.clear()
def hw_man(gateway, local_broker):
"""Create a HardwareManager that can talk to our gateway over the local broker."""
reg = ComponentRegistry()
reg.set_config('awsiot-endpoint', '')
reg.set_config('awsiot-rootcert', '')
reg.set_config('awsiot-iamkey', '')
reg.set_config('awsiot-iamtoken', '')
hw_dev = HardwareManager(port="awsiot:devices/d--0000-0000-0000-0002")
yield hw_dev
hw_dev.close()
def _setup_apps(self):
"""Load in all iotile app objects for all registered or installed components on this system."""
reg = ComponentRegistry()
modules = reg.list_components()
apps = reduce(lambda x, y: x + y,
[reg.find_component(x).app_modules() for x in modules],
[])
app_classes = []
for app in apps:
app_classes += self._load_module_classes(app, IOTileApp)
# Find all installed proxy objects through registered entry points
for entry in pkg_resources.iter_entry_points('iotile.app'):
mod = entry.load()
app_classes += [
x for x in itervalues(mod.__dict__) if inspect.isclass(x)
and issubclass(x, IOTileApp) and x != IOTileApp
]
for app in app_classes:
try:
matches = app.MatchInfo()
name = app.AppName()
for tag, ver_range, quality in matches:
if tag not in self._known_apps:
self._known_apps[tag] = []
self._known_apps[tag].append((ver_range, quality, app))
if name in self._named_apps:
self.logger.warning(
"Added an app module with an existing name, overriding previous app, name=%s",
name)
self._named_apps[name] = app
except Exception: #pylint: disable=broad-except;We don't want this to die if someone loads a misbehaving plugin
self.logger.exception(
"Error importing misbehaving app module, skipping.")
def list_local(self, path='.'):
"""List the absolute path of all local dependencies of this tile.
A local dependency is defined as one that is located in the local
ComponentRegistry(). The primary utility of this function is to
figure out which dependencies could productively be built locally
before building this component, vs which of its dependencies are
pulled down from released code that is immutable.
Args:
path (str): The path to the iotile component that we wish to
check local dependencies on. If not specified, this
defaults to the current working directory.
Returns:
list(str): A list of paths to each locally installed dependency.
"""
tile = IOTile(path)
if tile.release:
raise ArgumentError(
"Cannot check dependencies on a release mode tile that cannot have dependencies"
)
dep_paths = []
reg = ComponentRegistry()
for dep in tile.dependencies:
try:
local_tile = reg.find_component(dep['name'])
if local_tile.release:
continue
dep_paths.append(os.path.abspath(local_tile.folder))
except ArgumentError:
continue
return dep_paths
def _create_stream(self, force_adapter=None):
conn_string = None
port = self.port
if port is not None and "," in port:
port, conn_string = port.split(',')
if port is not None:
port = port.strip()
if conn_string is not None:
conn_string = conn_string.strip()
# Check if we're supposed to use a specific device adapter
if force_adapter is not None:
return AdapterCMDStream(force_adapter,
port,
conn_string,
record=self._record)
# First check if this is the special none stream that creates a transport channel nowhere
if self.transport == 'none':
return CMDStream(port, conn_string, record=self._record)
# Next attempt to find a CMDStream that is registered for this transport type
reg = ComponentRegistry()
for _name, stream_factory in reg.load_extensions(
'iotile.cmdstream', name_filter=self.transport):
return stream_factory(port, conn_string, record=self._record)
# Otherwise attempt to find a DeviceAdapter that we can turn into a CMDStream
for _name, adapter_factory in reg.load_extensions(
'iotile.device_adapter', name_filter=self.transport):
return AdapterCMDStream(adapter_factory(port),
port,
conn_string,
record=self._record)
raise HardwareError(
"Could not find transport object registered to handle passed transport type",
transport=self.transport)
def resolve(self, depinfo, destdir):
from iotile.core.dev.registry import ComponentRegistry
reg = ComponentRegistry()
try:
comp = reg.find_component(depinfo['name'])
except ArgumentError:
return {'found': False}
# Make sure the tile we found in the registry has the required version
reqver = depinfo['required_version']
if not reqver.check(comp.parsed_version):
return {'found': False}
# If the component is in the local registry but hasn't been built,
# raise an error.
if not os.path.exists(comp.output_folder):
raise ExternalError(
"Component found in registry but has not been built",
path=comp.folder,
name=comp.name,
suggestion="Run iotile build on this component first")
try:
IOTile(comp.output_folder)
except IOTileException:
raise ExternalError(
"Component found in registry but its build/output folder is not valid",
path=comp.folder,
name=comp.name,
suggestion="Cleanly rebuild the component")
self._copy_folder_contents(comp.output_folder, destdir)
return {'found': True}
def chain_composite(request):
"""Create a multirule chain that has the registry followed by the mock resolver
"""
reg = ComponentRegistry()
reg.add_component(comp_path('comp3_v1.0'))
chain = DependencyResolverChain()
chain.rules.append([10, (re.compile('.*'), MockDependencyResolver, [comp_path('comp1_v1.1')])])
yield chain
reg.remove_component('comp3')
