def get_job_config(job_spec, config_file, options): try: job_key = AuroraJobKey.from_path(job_spec) select_cluster = job_key.cluster select_env = job_key.env select_role = job_key.role jobname = job_key.name except AuroraJobKey.Error: deprecation_warning('Please refer to your job in CLUSTER/ROLE/ENV/NAME format.') select_cluster = options.cluster if options.cluster else None select_env = options.env select_role = None jobname = job_spec try: json_option = options.json except AttributeError: json_option = False try: bindings = options.bindings except AttributeError: bindings = () return get_config( jobname, config_file, json_option, bindings, select_cluster=select_cluster, select_role=select_role, select_env=select_env)
def setUp(self): self.RETURN_VALUE = 'foo' test_obj = self class FakeAuroraClientAPI(object): def cancel_update(self, job_key): test_obj.API_CALL = functools.partial(self.cancel_update, job_key) return test_obj.RETURN_VALUE def kill_job(self, job_key, instances=None, lock=None): test_obj.API_CALL = functools.partial(self.kill_job, job_key, instances, lock) return test_obj.RETURN_VALUE def restart(self, job_key, shards, updater_config, health_check_interval_seconds): test_obj.API_CALL = functools.partial(self.restart, job_key, shards, updater_config, health_check_interval_seconds) return test_obj.RETURN_VALUE def start_cronjob(self, job_key): test_obj.API_CALL = functools.partial(self.start_cronjob, job_key) return test_obj.RETURN_VALUE self._patch_bases(NonHookedAuroraClientAPI, (FakeAuroraClientAPI, )) self.api = NonHookedAuroraClientAPI() # Test args passed in to check that these are proxied un-modified self.test_job_key = AuroraJobKey.from_path('a/b/c/d') self.test_config = 'bar' self.test_shards = 'baz' self.test_lock = 'lock' self.test_updater_config = 'blah' self.health_check_interval_seconds = 'baa'
def disambiguate_args_or_die(cls, args, options, client_factory=AuroraClientAPI): """ Returns a (AuroraClientAPI, AuroraJobKey, AuroraConfigFile:str) tuple if one can be found given the args, potentially querying the scheduler with the returned client. Calls die() with an appropriate error message otherwise. Arguments: args: args from app command invocation. options: options from app command invocation. must have env and cluster attributes. client_factory: a callable (cluster) -> AuroraClientAPI. """ if not len(args) > 0: die('job path is required') try: job_key = AuroraJobKey.from_path(args[0]) client = client_factory(job_key.cluster) config_file = args[1] if len( args) > 1 else None # the config for hooks return client, job_key, config_file except AuroraJobKey.Error: log.warning( "Failed to parse job path, falling back to compatibility mode") role = args[0] if len(args) > 0 else None name = args[1] if len(args) > 1 else None env = None config_file = None # deprecated form does not support hooks functionality cluster = options.cluster if not cluster: die('cluster is required') client = client_factory(cluster) return client, cls._disambiguate_or_die(client, role, env, name), config_file
def run(args, options): """usage: run cluster/role/env/job cmd Runs a shell command on all machines currently hosting shards of a single job. This feature supports the same command line wildcards that are used to populate a job's commands. This means anything in the {{mesos.*}} and {{thermos.*}} namespaces. """ # TODO(William Farner): Add support for invoking on individual shards. # TODO(Kevin Sweeney): Restore the ability to run across jobs with globs (See MESOS-3010). if not args: die('job path is required') job_path = args.pop(0) try: cluster_name, role, env, name = AuroraJobKey.from_path(job_path) except AuroraJobKey.Error as e: die('Invalid job path "%s": %s' % (job_path, e)) command = ' '.join(args) cluster = CLUSTERS[cluster_name] dcr = DistributedCommandRunner(cluster, role, env, [name], options.ssh_user) dcr.run(command, parallelism=options.num_threads, executor_sandbox=options.executor_sandbox)
def disambiguate_args_or_die(cls, args, options, client_factory=AuroraClientAPI): """ Returns a (AuroraClientAPI, AuroraJobKey, AuroraConfigFile:str) tuple if one can be found given the args, potentially querying the scheduler with the returned client. Calls die() with an appropriate error message otherwise. Arguments: args: args from app command invocation. options: options from app command invocation. must have env and cluster attributes. client_factory: a callable (cluster) -> AuroraClientAPI. """ if not len(args) > 0: die('job path is required') try: job_key = AuroraJobKey.from_path(args[0]) client = client_factory(job_key.cluster) config_file = args[1] if len(args) > 1 else None # the config for hooks return client, job_key, config_file except AuroraJobKey.Error: log.warning("Failed to parse job path, falling back to compatibility mode") role = args[0] if len(args) > 0 else None name = args[1] if len(args) > 1 else None env = None config_file = None # deprecated form does not support hooks functionality cluster = options.cluster if not cluster: die('cluster is required') client = client_factory(cluster) return client, cls._disambiguate_or_die(client, role, env, name), config_file
def ssh(args, options): """usage: ssh cluster/role/env/job shard [args...] Initiate an SSH session on the machine that a shard is running on. """ if not args: die('Job path is required') job_path = args.pop(0) try: cluster_name, role, env, name = AuroraJobKey.from_path(job_path) except AuroraJobKey.Error as e: die('Invalid job path "%s": %s' % (job_path, e)) if not args: die('Shard is required') try: shard = int(args.pop(0)) except ValueError: die('Shard must be an integer') api = make_client(cluster_name) resp = api.query(api.build_query(role, name, set([int(shard)]), env=env)) check_and_log_response(resp) first_task = resp.result.scheduleStatusResult.tasks[0] remote_cmd = 'bash' if not args else ' '.join(args) command = DistributedCommandRunner.substitute(remote_cmd, first_task, api.cluster, executor_sandbox=options.executor_sandbox) ssh_command = ['ssh', '-t'] role = first_task.assignedTask.task.owner.role slave_host = first_task.assignedTask.slaveHost for tunnel in options.tunnels: try: port, name = tunnel.split(':') port = int(port) except ValueError: die('Could not parse tunnel: %s. Must be of form PORT:NAME' % tunnel) if name not in first_task.assignedTask.assignedPorts: die('Task %s has no port named %s' % (first_task.assignedTask.taskId, name)) ssh_command += [ '-L', '%d:%s:%d' % (port, slave_host, first_task.assignedTask.assignedPorts[name])] ssh_command += ['%s@%s' % (options.ssh_user or role, slave_host), command] return subprocess.call(ssh_command)
def test_kill_job_with_instances(self): """Test kill client-side API logic.""" mock_context = FakeAuroraCommandContext() with contextlib.nested( patch('twitter.aurora.client.cli.jobs.Job.create_context', return_value=mock_context), patch('twitter.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS)): api = mock_context.get_api('west') api.kill_job.return_value = self.get_kill_job_response() with temporary_file() as fp: fp.write(self.get_valid_config()) fp.flush() cmd = AuroraCommandLine() cmd.execute(['job', 'kill', '--config=%s' % fp.name, '--instances=0,2,4-6', 'west/mchucarroll/test/hello']) # Now check that the right API calls got made. assert api.kill_job.call_count == 1 api.kill_job.assert_called_with(AuroraJobKey.from_path('west/mchucarroll/test/hello'), [0, 2, 4, 5, 6])
def setUp(self): self.RETURN_VALUE = 'foo' test_obj = self class FakeAuroraClientAPI(object): def cancel_update(self, job_key): test_obj.API_CALL = functools.partial(self.cancel_update, job_key) return test_obj.RETURN_VALUE def kill_job(self, job_key, instances=None, lock=None): test_obj.API_CALL = functools.partial(self.kill_job, job_key, instances, lock) return test_obj.RETURN_VALUE def restart(self, job_key, shards, updater_config, health_check_interval_seconds): test_obj.API_CALL = functools.partial( self.restart, job_key, shards, updater_config, health_check_interval_seconds) return test_obj.RETURN_VALUE def start_cronjob(self, job_key): test_obj.API_CALL = functools.partial(self.start_cronjob, job_key) return test_obj.RETURN_VALUE self._patch_bases(NonHookedAuroraClientAPI, (FakeAuroraClientAPI, )) self.api = NonHookedAuroraClientAPI() # Test args passed in to check that these are proxied un-modified self.test_job_key = AuroraJobKey.from_path('a/b/c/d') self.test_config = 'bar' self.test_shards = 'baz' self.test_lock = 'lock' self.test_updater_config = 'blah' self.health_check_interval_seconds = 'baa'
def run(args, options): """usage: run cluster/role/env/job cmd Runs a shell command on all machines currently hosting shards of a single job. This feature supports the same command line wildcards that are used to populate a job's commands. This means anything in the {{mesos.*}} and {{thermos.*}} namespaces. """ # TODO(William Farner): Add support for invoking on individual shards. # TODO(Kevin Sweeney): Restore the ability to run across jobs with globs (See MESOS-3010). if not args: die('job path is required') job_path = args.pop(0) try: cluster_name, role, env, name = AuroraJobKey.from_path(job_path) except AuroraJobKey.Error as e: die('Invalid job path "%s": %s' % (job_path, e)) command = ' '.join(args) cluster = CLUSTERS[cluster_name] dcr = DistributedCommandRunner(cluster, role, env, [name], options.ssh_user) dcr.run(command, parallelism=options.num_threads, executor_sandbox=options.executor_sandbox)
def parse_aurora_job_key_into(option, opt, value, parser): try: setattr(parser.values, option.dest, AuroraJobKey.from_path(value)) except AuroraJobKey.Error as e: raise optparse.OptionValueError('Failed to parse: %s' % e)
def test_basic(self): AuroraJobKey.from_path("smf1/mesos/test/labrat")
def test_basic(self): AuroraJobKey.from_path("smf1/mesos/test/labrat")