def test_template_uid_parsing(self):
tt = [
{
'uid': 'github.com/account/repository/name/0.0.1',
'error': False,
'expected': ('github.com', 'account', 'repository', 'name', '0.0.1'),
'name': 'valid',
},
{
'uid': 'github.com/account/repository/name/0.0.1',
'error': False,
'expected': ('github.com', 'account', 'repository', 'name', '0.0.1'),
'name': 'no scheme',
},
{
'uid': 'account/repository/name/0.0.1',
'error': True,
'name': 'missing host',
},
{
'uid': 'github.com/repository/name/0.0.1',
'error': True,
'name': 'no account',
},
{
'uid': 'github.com/account/name/0.0.1',
'error': True,
'name': 'no repository',
},
{
'uid': 'github.com/account/repository/0.0.1',
'error': True,
'name': 'missing name',
},
{
'uid': 'github.com/account/repository/name',
'error': False,
'expected': ('github.com', 'account', 'repository', 'name'),
'name': 'missing version',
},
{
'uid': 'name/0.0.1',
'error': True,
'name': 'name and 0.0.1',
},
{
'uid': 'name',
'error': True,
'name': 'just name',
}
]
for tc in tt:
if tc['error'] is False:
uid = TemplateUID.parse(tc['uid'])
self.assertEqual(uid.tuple(), tc['expected'], tc['name'])
else:
with self.assertRaises(ValueError, msg=tc['name']):
uid = TemplateUID.parse(tc['uid'])
def test_template_uid_comparaison(self):
uid1 = TemplateUID.parse('github.com/account/repository/name/1.0.0')
uid2 = TemplateUID.parse('github.com/account/repository/name/1.0.0')
uid3 = TemplateUID.parse('github.com/account/repository/name/1.0.1')
uid4 = TemplateUID.parse('github.com/account/repository/name/0.9.1')
uid5 = TemplateUID.parse('github.com/account/repository/other/1.0.0')
self.assertTrue(uid1 == uid2)
self.assertTrue(uid1 != uid3)
self.assertTrue(uid1 < uid3)
self.assertTrue(uid1 <= uid3)
self.assertTrue(uid1 > uid4)
self.assertTrue(uid1 >= uid4)
with self.assertRaises(ValueError,
msg="should not compare 2 different template"):
uid1 < uid5
with self.assertRaises(ValueError,
msg="should not compare 2 different template"):
uid1 <= uid5
with self.assertRaises(ValueError,
msg="should not compare 2 different template"):
uid1 >= uid5
with self.assertRaises(ValueError,
msg="should not compare 2 different template"):
uid1 > uid5
def load(template, service_detail):
"""
load the service from it's file system serialized format
@param template: the template class to use to instantiate the service
@param service_detail:all the detail of a service in a dict (info, data, states, task list)
"""
template_uid = TemplateUID.parse(service_detail['service']['template'])
guid = service_detail['service']['guid']
try:
if template_uid > template.template_uid:
raise BadTemplateError(
"Trying to load service %s with template %s, while it requires %s or higher"
% (guid, template.template_uid,
service_detail['service']['template']))
except ValueError: # is the two template are not the same, ValueError is raised
raise BadTemplateError(
"Trying to load service %s with template %s, while it requires %s or higher"
% (guid, template.template_uid,
service_detail['service']['template']))
service = template(name=service_detail['service']['name'],
guid=guid,
data=service_detail['data'])
service._public = service_detail['service'].get('public', False)
service.state.load(service_detail['states'])
# srv.data.load(service_detail['data']) FIXME: should we need this since we pass the data in the constructor line 94
tasks = service_detail.get('tasks') or []
service.task_list.load(tasks)
add(service)
return service
def get_robot(self, template_uid):
"""
returns a instance of ZeroRobotClient that is managing the template identified by template_uid
It looks into all the know ZeroRobots which one manages the template_uid and return a client to it.
If not known robots managed the template_uid, then KeyError is raised
"""
if isinstance(template_uid, str):
template_uid = TemplateUID.parse(template_uid)
def compare(a, b):
if a.host and b.host and (a.host != b.host):
return False
if a.account and b.account and (a.account != b.account):
return False
if a.repo and b.repo and (a.repo != b.repo):
return False
if a.name and b.name and (a.name != b.name):
return False
if a.version and b.version and (a.version != b.version):
return False
return True
for robot in self.robots.values():
for uid in robot.templates.uids:
if compare(template_uid, uid):
return robot
raise KeyError("no robot that managed %s found" % template_uid)
def setUpClass(cls):
cls.valid_data = {
'zerotierClient': 'zt',
'wipeDisks': False,
'zerotierNetID': '',
'redisPassword': '',
'networks': ['storage']
}
cls.member = {
'nodeId': 'id',
'config': {
'authorized': False,
'ipAssignments': []
},
'online': False,
'name': 'name'
}
cls.member2 = {
'nodeId': 'id',
'config': {
'authorized': False,
'ipAssignments': ['127.0.0.1']
}
}
config.DATA_DIR = tempfile.mkdtemp(prefix='0-templates_')
ZeroosBootstrap.template_uid = TemplateUID.parse(
'github.com/zero-os/0-templates/%s/%s' %
(ZeroosBootstrap.template_name, ZeroosBootstrap.version))
def get(uid):
"""Get a template class
Arguments:
uid {string|TemplateUID} -- unique identifier for a template.
Can be a string with valid format as : 'github.com/account/repository/name/version'
or a zerorobot.template_collection.TemplateUID object
or a string which is the name of the template you want
if 2 templates are loaded that have the same name TemplateConflictError is raised
Raises:
TemplateNotFoundError -- template with specified uid is not found
TemplateConflictError -- raise if 2 templates have the same name is raised
Returns:
[TemplateBase] -- return the template class
"""
if isinstance(uid, str):
try:
uid = TemplateUID.parse(uid)
except ValueError:
# uid is not a full template uid, try with only its name
templates = find(name=uid)
size = len(templates)
if size > 1:
raise TemplateConflictError("tried to get template with name %s, but more then one template have this name (%s)" %
(uid, ', '.join([str(t.template_uid) for t in templates])))
elif size <= 0:
raise TemplateNotFoundError("template with name %s not found" % str(uid))
else:
return templates[0]
if uid not in _templates:
raise TemplateNotFoundError("template with name %s not found" % str(uid))
return _templates[uid]
def _load_template(url, template_dir):
"""
load a template in memory from a file
The file must contain a class that inherits from template.TemplateBase
and the name of the class must match the name of the file.
the name of the class is camelcased from the name of the file.
ex:
node.py -> Node
vm_manager.py -> VmManager
a_long_name.py -> ALongName
"""
template_name = os.path.basename(template_dir).split('.')[0]
class_name = template_name.replace('_', ' ').title().replace(' ', '')
class_path = os.path.join(template_dir, template_name + '.py')
spec = importlib.util.spec_from_file_location(template_name, class_path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
class_ = getattr(module, class_name)
_, host, account, repo = git.url.parse(url)
class_.template_uid = TemplateUID.parse("%s/%s/%s/%s/%s" % (host, account, repo, template_name, class_.version))
sys.modules[str(class_.template_uid)] = module
class_.template_dir = template_dir
_templates[class_.template_uid] = class_
logger = j.logger.get('zerorobot')
logger.debug("add template %s to collection" % class_.template_uid)
return _templates[class_.template_uid]
def _find_services_to_be_scheduled(actions):
services_guids = []
for action_item in actions:
template_uid = None
template = action_item.get("template")
if template:
template_uid = TemplateUID.parse(template)
service = action_item.get("service")
candidates = []
kwargs = {"name": service}
if template_uid:
kwargs.update({
"template_host": template_uid.host,
"template_account": template_uid.account,
"template_repo": template_uid.repo,
"template_name": template_uid.name,
"template_version": template_uid.version,
})
# filter out None value
kwargs = {k: v for k, v in kwargs.items() if v is not None}
if len(kwargs) > 0:
candidates = scol.find(**kwargs)
else:
candidates = scol.list_services()
services_guids.extend([s.guid for s in candidates])
return services_guids
def preTest(cls, path, template):
config.data_repo = config.DataRepo(
tempfile.mkdtemp(prefix='0-templates_'))
template_collection._load_template(
'https://github.com/zero-os/0-templates', path)
template.template_uid = TemplateUID.parse(
'github.com/zero-os/0-templates/%s/%s' %
(template.template_name, template.version))
def setUpClass(cls):
cls.valid_data = {
'node': 'node',
'templates': [],
'organization': None,
'nics': []
}
config.DATA_DIR = tempfile.mkdtemp(prefix='0-templates_')
Zrobot.template_uid = TemplateUID.parse(
'github.com/zero-os/0-templates/%s/%s' %
(Zrobot.template_name, Zrobot.version))
def _instantiate(self, data):
if hasattr(data, 'secret') and data.secret:
self._client = config_mgr.append_secret(self._client.instance,
data.secret)
# force re-creation of the connection with new secret added in the Authorization header
self._client._api = None
srv = ServiceProxy(data.name, data.guid, self._client)
srv.template_uid = TemplateUID.parse(data.template)
if data.data:
srv._data = data.as_dict()
return srv
def setUpClass(cls):
cls.valid_data = {
'nodePort': 9900,
'mode': 'user',
'sync': False,
'admin': '',
'disk': '/dev/sda1',
}
config.DATA_DIR = tempfile.mkdtemp(prefix='0-templates_')
Zerodb.template_uid = TemplateUID.parse(
'github.com/zero-os/0-templates/%s/%s' %
(Zerodb.template_name, Zerodb.version))
def setUpClass(cls):
cls.valid_data = {
'password': '******',
'timeout': 120,
'ssl': False,
'db': 0,
'port': 6379,
'host': '127.0.0.1',
'unixSocket': '',
}
config.DATA_DIR = tempfile.mkdtemp(prefix='0-templates_')
ZeroosClient.template_uid = TemplateUID.parse(
'github.com/zero-os/0-templates/%s/%s' %
(ZeroosClient.template_name, ZeroosClient.version))
def setUpClass(cls):
cls.valid_data = {
'url': 'url',
'db': 'db',
'username': '******',
'password': '******',
'productId': 'productId',
'botToken': 'botToken',
'chatId': 'chatId'
}
config.DATA_DIR = tempfile.mkdtemp(prefix='0-templates_')
ErpRegisteration.template_uid = TemplateUID.parse(
'github.com/zero-os/0-templates/%s/%s' %
(ErpRegisteration.template_name, ErpRegisteration.version))
def load(template, base_path):
"""
load the service from it's file system serialized format
@param template: the template class to use to instantiate the service
@param base_path: path of the directory where
to load the service state and data from
"""
if not os.path.exists(base_path):
raise FileNotFoundError(
"Trying to load service from %s, but directory doesn't exists" %
base_path)
guid = os.path.basename(base_path)
service_info = j.data.serializer.yaml.load(
os.path.join(base_path, 'service.yaml'))
service_data = j.data.serializer.yaml.load(
os.path.join(base_path, 'data.yaml'))
template_uid = TemplateUID.parse(service_info['template'])
try:
if template_uid > template.template_uid:
raise BadTemplateError(
"Trying to load service %s with template %s, while it requires %s or higher"
% (guid, template.template_uid, service_info['template']))
except ValueError: # is the two template are not the same, ValueError is raised
raise BadTemplateError(
"Trying to load service %s with template %s, while it requires %s or higher"
% (guid, template.template_uid, service_info['template']))
if service_info['guid'] != guid:
raise BadTemplateError(
"Trying to load service from folder %s, but name of the service is %s"
% (base_path, service_info['name']))
srv = template(name=service_info['name'],
guid=service_info['guid'],
data=service_data)
srv._public = service_info.get('public', False)
srv.state.load(os.path.join(base_path, 'state.yaml'))
srv.data.load(os.path.join(base_path, 'data.yaml'))
srv.task_list.load(os.path.join(base_path, 'tasks.yaml'))
srv._path = base_path
add(srv)
return srv
def load_services(data_dir):
if not os.path.exists(data_dir):
os.makedirs(data_dir)
for srv_dir in j.sal.fs.listDirsInDir(data_dir, recursive=True):
info_path = os.path.join(srv_dir, 'service.yaml')
if not os.path.exists(info_path):
continue
service_info = j.data.serializer.yaml.load(info_path)
tmpl_uid = TemplateUID.parse(service_info['template'])
try:
tmplClass = tcol.get(str(tmpl_uid))
except tcol.TemplateNotFoundError:
# template of the service not found, could be we have the template but not the same version
# try to get the template without specifiying version
tmplClasses = tcol.find(host=tmpl_uid.host, account=tmpl_uid.account, repo=tmpl_uid.repo, name=tmpl_uid.name)
size = len(tmplClasses)
if size > 1:
raise RuntimeError("more then one template version found, this should never happens")
elif size < 1:
# if the template is not found, try to add the repo using the info of the service template uid
url = "http://%s/%s/%s" % (tmpl_uid.host, tmpl_uid.account, tmpl_uid.repo)
tcol.add_repo(url)
tmplClass = tcol.get(service_info['template'])
else:
# template of another version found, use newer version to load the service
tmplClass = tmplClasses[0]
srv = scol.load(tmplClass, srv_dir)
loading_failed = []
for service in scol.list_services():
try:
service.validate()
except Exception as err:
logger.error("fail to load %s: %s" % (service.guid, str(err)))
# the service is not going to process its task list until it can
# execute validate() without problem
service.gl_mgr.stop('executor')
loading_failed.append(service)
if len(loading_failed) > 0:
gevent.spawn(_try_load_service, loading_failed)
def setUpClass(cls):
cls.valid_data = {
'cpu': 1,
'flist': 'flist',
'memory': 128,
'nics': [],
'vnc': -1,
'ports': [],
'media': [],
'mount': {},
'tags': [],
'uuid': '444d10d7-77f8-4b33-a6df-feb76e34dbc4',
}
config.DATA_DIR = tempfile.mkdtemp(prefix='0-templates_')
Vm.template_uid = TemplateUID.parse(
'github.com/zero-os/0-templates/%s/%s' %
(Vm.template_name, Vm.version))
cls.vnc_port = 5900
def setUpClass(cls):
cls.valid_data = {
'bridges': [],
'env': {},
'flist': 'flist',
'hostNetworking': False,
'hostname': '',
'identity': '',
'initProcesses': [],
'mounts': [],
'nics': [],
'ports': ['80:80'],
'privileged': False,
'storage': '',
'zerotierNetwork': ''
}
config.DATA_DIR = tempfile.mkdtemp(prefix='0-templates_')
Container.template_uid = TemplateUID.parse(
'github.com/zero-os/0-templates/%s/%s' % (Container.template_name, Container.version))
def setUpClass(cls):
cls.valid_data = {
'chatId':
'chatId',
'supported': [{
'hddCount': 2,
'ram': 8,
'cpu': 'cpu',
'ssdCount': 2,
'name': 'name'
}],
'botToken':
'botToken'
}
config.DATA_DIR = tempfile.mkdtemp(prefix='0-templates_')
HardwareCheck.template_uid = TemplateUID.parse(
'github.com/zero-os/0-templates/%s/%s' %
(HardwareCheck.template_name, HardwareCheck.version))
patch('js9.j.tools')
def setUpClass(cls):
cls.valid_data = {
'container': 'container_minio',
'node': 'node',
'listenPort': 9000,
'namespace': 'namespace',
'nsSecret': 'nsSecret',
'login': '******',
'password': '******',
'zerodbs': ['192.24.121.42:9900'],
'privateKey': '',
'resticPassword': '******',
'resticRepo': 'repo/',
'resticRepoPassword': '',
'resticUsername': '******'
}
config.DATA_DIR = tempfile.mkdtemp(prefix='0-templates_')
Minio.template_uid = TemplateUID.parse(
'github.com/zero-os/0-templates/%s/%s' %
(Minio.template_name, Minio.version))
def load_services(config):
for service_details in storage.list():
tmpl_uid = TemplateUID.parse(service_details['service']['template'])
try:
tmplClass = tcol.get(str(tmpl_uid))
except tcol.TemplateNotFoundError:
# template of the service not found, could be we have the template but not the same version
# try to get the template without specifiying version
tmplClasses = tcol.find(host=tmpl_uid.host, account=tmpl_uid.account, repo=tmpl_uid.repo, name=tmpl_uid.name)
size = len(tmplClasses)
if size > 1:
raise RuntimeError("more then one template version found, this should never happens")
elif size < 1:
# if the template is not found, try to add the repo using the info of the service template uid
url = "http://%s/%s/%s" % (tmpl_uid.host, tmpl_uid.account, tmpl_uid.repo)
tcol.add_repo(url)
tmplClass = tcol.get(service_details['service']['template'])
else:
# template of another version found, use newer version to load the service
tmplClass = tmplClasses[0]
scol.load(tmplClass, service_details)
loading_failed = []
logger = j.logger.get('zerorobot')
for service in scol.list_services():
try:
service.validate()
except Exception as err:
logger.error("fail to load %s: %s" % (service.guid, str(err)))
# the service is not going to process its task list until it can
# execute validate() without problem
service.gl_mgr.stop('executor')
loading_failed.append(service)
if len(loading_failed) > 0:
gevent.spawn(_try_load_service, loading_failed)
def _schedule_action(action_item):
template_uid = None
template = action_item.get("template")
if template:
template_uid = TemplateUID.parse(template)
service = action_item.get("service")
action = action_item.get("action")
args = action_item.get("args")
if args and not isinstance(args, dict):
raise TypeError("args should be a dict not %s" % type(args))
candidates = []
kwargs = {"name": service}
if template_uid:
kwargs.update({
"template_host": template_uid.host,
"template_account": template_uid.account,
"template_repo": template_uid.repo,
"template_name": template_uid.name,
"template_version": template_uid.version,
})
# filter out None value
kwargs = {k: v for k, v in kwargs.items() if v is not None}
if len(kwargs) > 0:
candidates = scol.find(**kwargs)
else:
candidates = scol.list_services()
tasks = []
for service in candidates:
t = service.schedule_action(action, args=args)
tasks.append((t, service))
return tasks
def setUpClass(cls):
config.DATA_DIR = tempfile.mkdtemp(prefix='0-templates_')
ZerotierClient.template_uid = TemplateUID.parse(
'github.com/zero-os/0-templates/%s/%s' % (ZerotierClient.template_name, ZerotierClient.version))
def setUpClass(cls):
cls.valid_data = {'node': 'node', 'alerta': ['alerta']}
config.DATA_DIR = tempfile.mkdtemp(prefix='0-templates_')
Healthcheck.template_uid = TemplateUID.parse(
'github.com/zero-os/0-templates/%s/%s' %
(Healthcheck.template_name, Healthcheck.version))
def uids(self):
"""
Returns a list of template UID present on the ZeroRobot
"""
templates, _ = self._client.api.templates.ListTemplates()
return {TemplateUID.parse(t.uid): t for t in templates}
def __init__(self, guid, name):
self.name = name
self.guid = guid
self.template_uid = TemplateUID.parse('github.com/threefoldtech/0-robot/fakeservice/0.0.1')
self._public = False
def __init__(self, guid, name):
self.name = name
self.guid = guid
self.template_uid = TemplateUID.parse('github.com/threefoldtech/0-robot/other/0.0.1')
