def get_job(self, job_name):
""" Returns a Job by name, or None if none exists. """
for job in self.jobs:
if job.name == job_name:
return job
logger.warn('Tried to find job with name {0}, but job not found'.format(job_name))
return None
def remote_ssh(self, host):
""" Execute a command on SSH. Takes a paramiko host dict """
logger.info('Starting remote execution of task {0} on host {1}'.format(self.name, host['hostname']))
try:
self.remote_client = paramiko.SSHClient()
self.remote_client.load_system_host_keys()
self.remote_client.set_missing_host_key_policy(
paramiko.AutoAddPolicy())
self.remote_client.connect(host['hostname'], username=host['user'],
key_filename=host['identityfile'][0],
timeout=82800)
transport = self.remote_client.get_transport()
transport.set_keepalive(10)
self.remote_channel = transport.open_session()
self.remote_channel.get_pty()
self.remote_channel.exec_command(self.command)
except Exception as e:
logger.warn('Exception encountered in remote task execution')
self.remote_failure = True
self.stderr += 'Exception when trying to SSH related to: '
self.stderr += '{0}: {1}\n"'.format(type(e).__name__, str(e))
self.stderr += 'Was looking for host "{0}"\n'.format(str(host))
self.stderr += 'Found in config:\n'
self.stderr += 'host: "{0}"\n'.format(str(host))
self.stderr += 'hostname: "{0}"\n'.format(str(host.get('hostname')))
self.stderr += 'user: "******"\n'.format(str(host.get('user')))
self.stderr += 'identityfile: "{0}"\n'.format(str(host.get('identityfile')))
self.remote_client.close()
def get_host(self, hostname):
""" Returns a Host dict with config options, or None if none exists"""
if hostname in self.get_hosts():
return self.load_ssh_conf().lookup(hostname)
logger.warn(
'Tried to find host with name {0}, but host not found'.format(
hostname))
return None
def load_ssh_conf(self):
try:
conf_file = open(os.path.expanduser(self.ssh_config))
ssh_config = paramiko.SSHConfig()
ssh_config.parse(conf_file)
conf_file.close()
return ssh_config
except IOError:
logger.warn('Tried to load SSH config but failed, probably file not found')
return None
def initialize_snapshot(self):
""" Copy the DAG and validate """
logger.debug('Initializing DAG snapshot for job {0}'.format(self.name))
if self.snapshot is not None:
logger.warn("Attempting to initialize DAG snapshot without " +
"first destroying old snapshot.")
snapshot_to_validate = deepcopy(self.graph)
is_valid, reason = self.validate(snapshot_to_validate)
if not is_valid:
raise DagobahError(reason)
self.snapshot = snapshot_to_validate
def init_dagobah(config, testing=False):
backend = get_backend(config)
event_handler = configure_event_hooks(config)
ssh_config = get_conf(config, 'Dagobahd.ssh_config', '~/.ssh/config')
if not os.path.isfile(os.path.expanduser(ssh_config)):
logger.warn("SSH config doesn't exist, no remote hosts will be listed")
dagobah = Dagobah(backend, event_handler, ssh_config)
known_ids = [
id for id in backend.get_known_dagobah_ids()
if id != dagobah.dagobah_id
]
if len(known_ids) > 1:
# need a way to handle this intelligently through config
raise ValueError('could not infer dagobah ID, ' +
'multiple available in backend')
if known_ids:
dagobah.from_backend(known_ids[0])
return dagobah