def _get_script_action_method(self): task = WrappedCallableTask(self._run_script_with_settings, name=self.name, alias=self.action_exec_id, parallel=self.parallel, sudo=self.sudo) task.parallel = self.parallel task.serial = not self.parallel return task
def execute_pseudo_task(fn=None, obj=None, name=None): '''Safely call a function as a WrappedCallableTask This alters the function `fn` and creates a :py:class:`WrappedCallableTask`, also updating host and role parameters to match the environment :param fn: function to alter :param obj: object to search on :param name: object function name as string ''' try: # Alternate form of call if obj is not None and name is not None: fn = getattr(obj, name) if not callable(fn): raise ValueError( "Function `fn` must be callable, " "or function `name` must exist on object `obj`" ) if not _is_task(fn): fn = WrappedCallableTask(fn) fn.hosts = env.hosts except AttributeError: pass else: execute(fn)
def test_calling_the_object_is_the_same_as_run(self): random_return = random.randint(1000, 2000) def foo(): return random_return task = WrappedCallableTask(foo) eq_(task(), task.run())
def get_fabric_task(self): action_method = self._get_action_method() LOG.debug('action_method is %s', action_method) task = WrappedCallableTask(action_method, name=self.name, alias=self.action_exec_id, parallel=self.parallel, sudo=self.sudo) # We need to explicitly set that since WrappedCallableTask abuses kwargs # and doesn't do anything with "parallel" and "serial" kwarg. # We also need to explicitly set serial since we default to # parallel=True in the environment so just "parallel" won't do. task.parallel = self.parallel task.serial = not self.parallel return task
def test_name_is_the_name_of_the_wrapped_callable(self): def foo(): pass foo.__name__ = "random_name_%d" % random.randint(1000, 2000) task = WrappedCallableTask(foo) eq_(task.name, foo.__name__)
def deploy_ids(pr, owner, repository): from apply_pr import fabfile configure_logging() get_deploys_task = WrappedCallableTask(fabfile.print_deploys) execute(get_deploys_task, pr, owner=owner, repository=repository)
def task_with_pkg_log(task_function): @wraps(task_function) def wrapper(*args, **kwargs): task_name = task_function.__name__ # Before timestamp = datetime.now().strftime("%Y%m%d-%H%M%S") before_file_path = "{}/{}-{}-before.txt".format( log_dir, timestamp, task_name) cmd = "zcat -f /var/log/dpkg.log* | grep \"\\ install\\ \" > %s" % before_file_path require.directory(log_dir, owner='root', use_sudo=True) sudo(cmd) result = task_function(*args, **kwargs) # After timestamp = datetime.now().strftime("%Y%m%d-%H%M%S") after_file_path = "{}/{}-{}-after.txt".format(log_dir, timestamp, task_name) cmd = "zcat -f /var/log/dpkg.log* | grep \"\\ install\\ \" > %s" % after_file_path sudo(cmd) return result return WrappedCallableTask(wrapper)
def test_passes_all_regular_args_to_run(self): def foo(*args): return args random_args = tuple( [random.randint(1000, 2000) for i in range(random.randint(1, 5))] ) task = WrappedCallableTask(foo) eq_(random_args, task(*random_args))
def test_reads_double_under_doc_from_callable(self): def foo(): pass foo.__doc__ = "Some random __doc__: %d" % random.randint(1000, 2000) task = WrappedCallableTask(foo) eq_(task.__doc__, foo.__doc__)
def get_tasks(self): return [ WrappedCallableTask(lcd_task(task, self._cwd)) for task in [ self.upload, self.deploy, self.install, self.build, self.tail, self.rollback, self.upstart, self.link_latest_release ] ]
def test_allows_any_number_of_args(self): args = [i for i in range(random.randint(0, 10))] def foo(): pass task = WrappedCallableTask(foo, *args)
def create_host_task(key, host_config): """ Generate host tasks dynamically from config """ # do validation *before* dynamic task function generation # allowing for hostname to avoid a breaking change if 'hostname' in host_config and 'hostnames' in host_config: raise ValueError(red('cannot specify both \'hostname\' and \'hostnames\'')) if 'hostname' not in host_config and 'hostnames' not in host_config: raise ValueError(red('must supply \'hostnames\' section')) hosts_key = 'hostname' if 'hostname' in host_config else 'hostnames' def f(): hosts = None if 'hostname' in host_config: warn('\'hostname\' is being deprecated in favor of \'hostnames\' so you can provide a csv-list\n') hostname = host_config['hostname'] hosts = [hostname] if 'hostnames' in host_config: hosts = [h.strip() for h in host_config['hostnames'].split(',')] env.hosts = hosts env.port = host_config.get('port', 22) # convenience for local deployment to Vagrantfile VM if hosts[0] in {'localhost', '127.0.0.1'}: hostname = '127.0.0.1' # sometimes fabric just fails with 'localhost' env.user = '******' env.password = '******' env.port = host_config.get('port', 2222) f.__name__ = key f.__doc__ = "[hosts] \tsets deploy hosts to %s" % green(host_config[hosts_key]) return WrappedCallableTask(f)
def test_allows_any_number_of_kwargs(self): kwargs = dict([("key%d" % i, i) for i in range(random.randint(0, 10))]) def foo(): pass task = WrappedCallableTask(foo, **kwargs)
def test_passes_all_keyword_args_to_run(self): def foo(**kwargs): return kwargs random_kwargs = {} for i in range(random.randint(1, 5)): random_key = ("foo", "bar", "baz", "foobar", "barfoo")[i] random_kwargs[random_key] = random.randint(1000, 2000) task = WrappedCallableTask(foo) eq_(random_kwargs, task(**random_kwargs))
def test_dispatches_to_wrapped_callable_on_run(self): random_value = "some random value %d" % random.randint(1000, 2000) def foo(): return random_value task = WrappedCallableTask(foo) eq_(random_value, task())
def test_passes_unused_args_to_parent(self): args = [i for i in range(random.randint(1, 10))] def foo(): pass try: task = WrappedCallableTask(foo, *args) except TypeError: msg = "__init__ raised a TypeError, meaning args weren't handled" self.fail(msg)
def test_passes_unused_kwargs_to_parent(self): random_range = range(random.randint(1, 10)) kwargs = dict([("key_%s" % i, i) for i in random_range]) def foo(): pass try: task = WrappedCallableTask(foo, **kwargs) except TypeError: self.fail( "__init__ raised a TypeError, meaning kwargs weren't handled")
def apply_pr(pr, host, from_number=0, from_commit=None, force_hostname=False, owner='gisce', repository='erp', src='/home/erp/src', sudo_user='******', auto_exit=False, force_name=None): """ Deploy a PR into a remote server via Fabric :param pr: Number of the PR to deploy :type pr: str :param host: Host to connect :type host: str :param from_number: Number of the commit to deploy from :type from_number: str :param from_commit: Hash of the commit to deploy from :type from_commit: str :param force_hostname: Hostname used in GitHub :type force_hostname: str :param owner: Owner of the repository of GitHub :type owner: str :param repository: Name of the repository of GitHub :type repository: str :param src: Source path to the repository directory :type src: str :param sudo_user: Remote user with sudo :type sudo_user str """ from apply_pr import fabfile if 'ssh' not in host and host[:2] != '//': host = '//{}'.format(host) url = urlparse(host, scheme='ssh') env.user = url.username env.password = url.password configure_logging() apply_pr_task = WrappedCallableTask(fabfile.apply_pr) result = execute(apply_pr_task, pr, from_number, from_commit, hostname=force_hostname, src=src, owner=owner, repository=repository, sudo_user=sudo_user, host='{}:{}'.format(url.hostname, (url.port or 22)), auto_exit=auto_exit, force_name=force_name) return result
def status_pr(deploy_id, status, owner, repository): """Update the status of a deploy into GitHub""" from apply_pr import fabfile configure_logging() mark_deploy_status = WrappedCallableTask(fabfile.mark_deploy_status) execute(mark_deploy_status, deploy_id, status, owner=owner, repository=repository)
def create_changelog(milestone, issues, changelog_path, owner, repository): """Create a changelog for the given milestone""" from apply_pr import fabfile log_level = getattr(logging, os.environ.get('LOG_LEVEL', 'INFO').upper()) logging.basicConfig(level=log_level) changelog_task = WrappedCallableTask(fabfile.create_changelog) execute(changelog_task, milestone, issues, changelog_path, owner=owner, repository=repository)
def check_prs_status(prs, separator, version, owner, repository): """Check the status of the PRs for a set of PRs""" from apply_pr import fabfile log_level = getattr(logging, os.environ.get('LOG_LEVEL', 'INFO').upper()) logging.basicConfig(level=log_level) check_pr_task = WrappedCallableTask(fabfile.prs_status) execute(check_pr_task, prs, owner=owner, repository=repository, separator=separator, version=version)
def mark_deployed_backend(pr, force_hostname=False, owner='gisce', repository='erp'): from apply_pr import fabfile configure_logging() mark_deployed_task = WrappedCallableTask(fabfile.mark_deployed) execute(mark_deployed_task, pr, hostname=force_hostname, owner=owner, repository=repository) click.echo( colors.green( u"Marking PR#{} as deployed success! \U0001F680".format(pr)))
def check_pr(pr, force, src, owner, repository, host): """DEPRECATED - Check for applied commits on PR""" print(colors.red("This option has been deprecated as it doesn't work")) if not force: print( colors.red( "Use '--force' to force the usage for this command (as is)")) exit() from apply_pr import fabfile url = urlparse(host, scheme='ssh') env.user = url.username env.password = url.password configure_logging() check_pr_task = WrappedCallableTask(fabfile.check_pr) execute(check_pr_task, pr, src=src, owner=owner, repository=repository, host='{}:{}'.format(url.hostname, (url.port or 22)))
def notify(t): """Decorates a fabric task""" @wraps(t) def wrapper(*args, **kwargs): start = time.time() error = None output = None try: r = t(*args, **kwargs) if r: if not isinstance(r, list): r = [r] output = '\n\n'.join(['%s\n%s\n%s' % (res.command, res.stdout, res.stderr) for res in r] ) except Exception as e: error = e end = time.time() duration = end - start try: dog_http_api.event(_title(t, args, kwargs, error), _text(t, args, kwargs, duration, output, error), source_type_name="fabric", alert_type="error" if error else "success", priority="normal", aggregation_key=_aggregation_key(t, args, kwargs, error), tags=_tags(t, args, kwargs, error)) except Exception as e: logger.warn("Datadog notification on task {0} failed with {1}".format(t.__name__, e)) if error: raise error else: return r return WrappedCallableTask(wrapper)
def test_name_can_be_overridden(self): def foo(): pass eq_(WrappedCallableTask(foo).name, 'foo') eq_(WrappedCallableTask(foo, name='notfoo').name, 'notfoo')
def get_hosts(command, *args): return WrappedCallableTask(command).get_hosts(*args)
def test_run_is_wrapped_callable(self): def foo(): pass task = WrappedCallableTask(foo) eq_(task.wrapped, foo)
def execute(task, *args, **kwargs): """ Patched version of fabric's execute task with alternative error handling """ my_env = {'clean_revert': True} results = {} # Obtain task is_callable = callable(task) if not (is_callable or _is_task(task)): # Assume string, set env.command to it my_env['command'] = task task = crawl(task, state.commands) if task is None: msg = "%r is not callable or a valid task name" % ( my_env['command'],) if state.env.get('skip_unknown_tasks', False): warn(msg) return else: abort(msg) # Set env.command if we were given a real function or callable task obj else: dunder_name = getattr(task, '__name__', None) my_env['command'] = getattr(task, 'name', dunder_name) # Normalize to Task instance if we ended up with a regular callable if not _is_task(task): task = WrappedCallableTask(task) # Filter out hosts/roles kwargs new_kwargs, hosts, roles, exclude_hosts = parse_kwargs(kwargs) # Set up host list my_env['all_hosts'], my_env[ 'effective_roles'] = task.get_hosts_and_effective_roles(hosts, roles, exclude_hosts, state.env) parallel = requires_parallel(task) if parallel: # Import multiprocessing if needed, erroring out usefully # if it can't. try: import multiprocessing except ImportError: import traceback tb = traceback.format_exc() abort(tb + """ At least one task needs to be run in parallel, but the multiprocessing module cannot be imported (see above traceback.) Please make sure the module is installed or that the above ImportError is fixed.""") else: multiprocessing = None # Get pool size for this task pool_size = task.get_pool_size(my_env['all_hosts'], state.env.pool_size) # Set up job queue in case parallel is needed queue = multiprocessing.Queue() if parallel else None jobs = JobQueue(pool_size, queue) if state.output.debug: jobs._debug = True # Call on host list if my_env['all_hosts']: # Attempt to cycle on hosts, skipping if needed for host in my_env['all_hosts']: try: results[host] = _execute( task, host, my_env, args, new_kwargs, jobs, queue, multiprocessing ) except NetworkError, e: results[host] = e # Backwards compat test re: whether to use an exception or # abort func = warn if state.env.skip_bad_hosts or state.env.warn_only \ else abort error(e.message, func=func, exception=e.wrapped) except SystemExit, e: pass # If requested, clear out connections here and not just at the end. if state.env.eagerly_disconnect: disconnect_all()
if os.path.isfile(_common_fn): _common = yaml.safe_load(open(_common_fn)) if os.path.isdir(common.ROLE_DIR): for _name in os.listdir(common.ROLE_DIR): #print 'checking',_name _settings_fn = os.path.join(common.ROLE_DIR, _name, 'settings.yaml') if _name == 'all' or not os.path.isfile(_settings_fn): continue _config = copy.deepcopy(_common) _config.update(yaml.safe_load(open(_settings_fn)) or type(env)()) _settings_local_fn = os.path.join(common.ROLE_DIR, _name, 'settings_local.yaml') if os.path.isfile(_settings_local_fn): _config.update(yaml.safe_load(open(_settings_local_fn)) or type(env)()) _f = _get_environ_handler(_name, _config) _var_name = 'role_'+_name _f = WrappedCallableTask(_f, name=_name) exec "%s = _f" % (_var_name,) role_commands[_var_name] = _f # Auto-import all sub-modules. sub_modules = {} sub_modules['common'] = common __all__ = [] for loader, module_name, is_pkg in pkgutil.walk_packages(__path__): if module_name in locals(): continue __all__.append(module_name) module = loader.find_module(module_name).load_module(module_name) sub_modules[module_name] = module #print module
alert_type="success", priority="normal", aggregation_key=task_full_name) except: logger.warn( "Datadog notification failed but task {0} completed". format(t.wrapped.func_name)) return r except Exception, e: # If notification is on, create an error event end = time.time() duration = end - start if notify_datadog: try: task_full_name = "%s.%s" % (t.__module__, t.wrapped.func_name) dog_http_api.event( "{0}".format(task_full_name), "{0} failed after {1} because of {2}.".format( task_full_name, human_duration(duration), e), source_type_name="fabric", alert_type="error", priority="normal", aggregation_key=task_full_name) except: logger.exception("Datadog notification failed") # Reraise raise return WrappedCallableTask(wrapper)
def execute(task, *args, **kwargs): """ Patched version of fabric's execute task with alternative error handling """ my_env = {'clean_revert': True} results = {} # Obtain task is_callable = callable(task) if not (is_callable or _is_task(task)): # Assume string, set env.command to it my_env['command'] = task task = crawl(task, state.commands) if task is None: msg = "%r is not callable or a valid task name" % ( my_env['command'], ) if state.env.get('skip_unknown_tasks', False): warn(msg) return else: abort(msg) # Set env.command if we were given a real function or callable task obj else: dunder_name = getattr(task, '__name__', None) my_env['command'] = getattr(task, 'name', dunder_name) # Normalize to Task instance if we ended up with a regular callable if not _is_task(task): task = WrappedCallableTask(task) # Filter out hosts/roles kwargs new_kwargs, hosts, roles, exclude_hosts = parse_kwargs(kwargs) # Set up host list my_env['all_hosts'], my_env[ 'effective_roles'] = task.get_hosts_and_effective_roles( hosts, roles, exclude_hosts, state.env) parallel = requires_parallel(task) if parallel: # Import multiprocessing if needed, erroring out usefully # if it can't. try: import multiprocessing except ImportError: import traceback tb = traceback.format_exc() abort(tb + """ At least one task needs to be run in parallel, but the multiprocessing module cannot be imported (see above traceback.) Please make sure the module is installed or that the above ImportError is fixed.""") else: multiprocessing = None # Get pool size for this task pool_size = task.get_pool_size(my_env['all_hosts'], state.env.pool_size) # Set up job queue in case parallel is needed queue = multiprocessing.Queue() if parallel else None jobs = JobQueue(pool_size, queue) if state.output.debug: jobs._debug = True # Call on host list if my_env['all_hosts']: # Attempt to cycle on hosts, skipping if needed for host in my_env['all_hosts']: try: results[host] = _execute(task, host, my_env, args, new_kwargs, jobs, queue, multiprocessing) except NetworkError, e: results[host] = e # Backwards compat test re: whether to use an exception or # abort func = warn if state.env.skip_bad_hosts or state.env.warn_only \ else abort error(e.message, func=func, exception=e.wrapped) except SystemExit, e: results[host] = e # If requested, clear out connections here and not just at the end. if state.env.eagerly_disconnect: disconnect_all()
def get_hosts_and_effective_roles(command, *args): return WrappedCallableTask(command).get_hosts_and_effective_roles(*args)