def setUpClass(cls): # TODO: Mock all tests so etcd doesn't need to be running ( lambda res: res if res is True else raise_f(HTTPException, "Failed to connect to etcd, errno: {}".format(res)) )(ping_port(port=2379))
def add_to_cluster(self, cluster, res): """ Specification: 0. Search and handle `master` tag in `cluster_name` 1. Imports `cluster_name`, seeks and sets (`install` xor `setup`) and (serve` or `start`) callables 2. Installs `cluster_name` 3. Serves `cluster_name` """ args = cluster['args'] if 'args' in cluster else tuple() kwargs = update_d({ 'domain': self.dns_name, 'node_name': self.node_name, 'public_ipv4': self.node.public_ips[-1], 'cache': {}, 'cluster_name': cluster.get('cluster_name') }, cluster['kwargs'] if 'kwargs' in cluster else {}) cluster_type = cluster['module'].replace('-', '_') cluster_path = '/'.join(ifilter(None, (cluster_type, kwargs['cluster_name']))) kwargs.update(cluster_path=cluster_path) if ':' in cluster_type: cluster_type, _, tag = cluster_type.rpartition(':') del _ else: tag = None kwargs.update(tag=tag) if tag == 'master': kwargs.update(master=True) if hasattr(self.node, 'private_ips') and len(self.node.private_ips): kwargs.update(private_ipv4=self.node.private_ips[-1]) guessed_os = self.guess_os() # import `cluster_type` try: setattr(self, 'fab', getattr(__import__(cluster_type, globals(), locals(), [guessed_os], -1), guessed_os)) except AttributeError as e: if e.message != "'module' object has no attribute '{os}'".format(os=guessed_os): raise raise ImportError('Cannot `import {os} from {cluster_type}`'.format(os=guessed_os, cluster_type=cluster_type)) fab_dir = dir(self.fab) # Sort functions like so: `step0`, `step1` func_names = sorted( (j for j in fab_dir if not j.startswith('_') and str.isdigit(j[-1])), key=lambda s: int(''.join(takewhile(str.isdigit, s[::-1]))[::-1] or -1) ) if 'run_cmds' in cluster: mapping = {'>=': operator.ge, '<': operator.lt, '>': operator.gt, '<=': operator.le} # TODO: There must be a full list somewhere! def dict_type(run_cmds, func_names): op = mapping[run_cmds['op']] return [func_name for func_name in func_names if op(int(''.join(takewhile(str.isdigit, func_name[::-1]))[::-1]), int(run_cmds['val']))] run_cmds_type = type(cluster['run_cmds']) if 'exclude' in cluster['run_cmds']: func_names = tuple(ifilter(lambda func: func not in cluster['run_cmds']['exclude'], func_names)) func_names = dict_type(cluster['run_cmds'], func_names) '''{ DictType: dict_type(cluster['run_cmds'], func_names) }.get(run_cmds_type, raise_f(NotImplementedError, '{!s} unexpected for run_cmds'.format(run_cmds_type)))''' if not func_names: try: get_attr = lambda a, b: a if hasattr(self.fab, a) else b if hasattr(self.fab, b) else raise_f( AttributeError, '`{a}` nor `{b}`'.format(a=a, b=b)) func_names = ( get_attr('install', 'setup'), get_attr('serve', 'start') ) except AttributeError as e: logger.error('{e} found in {cluster_type}'.format(e=e, cluster_type=cluster_type)) raise AttributeError( 'Function names in {cluster_type} must end in a number'.format(cluster_type=cluster_type) ) # 'must'! logger.warn('Deprecation: Function names in {cluster_type} should end in a number'.format( cluster_type=cluster_type) ) self.handle_deprecations(func_names) for idx, step in enumerate(func_names): exec_output = execute(getattr(self.fab, step), *args, **kwargs)[self.dns_name] if idx == 0: res[self.dns_name] = {cluster_path: {step: exec_output}} if tag == 'master': save_node_info('master', [self.node_name], folder=cluster_type, marshall=json) else: res[self.dns_name][cluster_path][step] = exec_output save_node_info(self.node_name, node_to_dict(self.node), folder=cluster_path, marshall=json)
def prepare_cluster_obj(self, cluster, res): cluster_args = cluster["args"] if "args" in cluster else tuple() cluster_kwargs = update_d( { "domain": self.dns_name, "node_name": self.node_name, "public_ipv4": self.node.public_ips[-1], "cache": {}, "cluster_name": cluster.get("cluster_name"), }, cluster["kwargs"] if "kwargs" in cluster else {}, ) cluster_type = cluster["module"].replace("-", "_") cluster_path = "/".join(_f for _f in (cluster_type, cluster_kwargs["cluster_name"]) if _f) cluster_kwargs.update(cluster_path=cluster_path) if "cache" not in cluster_kwargs: cluster_kwargs["cache"] = {} if ":" in cluster_type: cluster_type, _, tag = cluster_type.rpartition(":") del _ else: tag = None cluster_kwargs.update(tag=tag) if tag == "master": cluster_kwargs.update(master=True) if hasattr(self.node, "private_ips") and len(self.node.private_ips): cluster_kwargs.update(private_ipv4=self.node.private_ips[-1]) guessed_os = guess_os(node=self.node) # import `cluster_type` try: setattr( self, "fab", getattr( __import__(cluster_type, globals(), locals(), [guessed_os]), guessed_os, ), ) except AttributeError as e: if e.message != "'module' object has no attribute '{os}'".format( os=guessed_os): raise raise ImportError( "Cannot `import {os} from {cluster_type}`".format( os=guessed_os, cluster_type=cluster_type)) fab_dir = dir(self.fab) # Sort functions like so: `step0`, `step1` func_names = get_sorted_strnum(fab_dir) func_names = self.filter_funcs(cluster, func_names) if not func_names: try: get_attr = ( lambda a, b: a if hasattr(self.fab, a) else b if hasattr(self.fab, b) else raise_f( AttributeError, "`{a}` nor `{b}`".format(a=a, b=b))) func_names = (get_attr("install", "setup"), get_attr("serve", "start")) except AttributeError as e: root_logger.error("{e} found in {cluster_type}".format( e=e, cluster_type=cluster_type)) raise AttributeError( "Function names in {cluster_type} must end in a number". format(cluster_type=cluster_type)) # 'must'! root_logger.warn( "Deprecation: Function names in {cluster_type} should end in a number" .format(cluster_type=cluster_type)) self.handle_deprecations(func_names) self.func_names = func_names return PreparedClusterObj( cluster_path=cluster_path, cluster_type=cluster_type, cluster_args=cluster_args, cluster_kwargs=cluster_kwargs, res=res, tag=tag, )