def ssh_login(host, username, password):
r = SSH_Client()
r.load_system_host_keys()
r.set_missing_host_key_policy(AutoAddPolicy())
info("Trying to open a SSH connection to {}".format(host))
try:
r.connect(host, username=username, password=password)
except SSH_BadHostKeyException as errstr:
error("SSH host key for {0} could not be verified: {1}".format(host, errstr))
return False
except SSH_AuthenticationException:
error("SSH authentication failed for {}".format(host))
return False
except SSH_SSHException as errstr:
error("Unknown SSH error while connecting to {0}: {1}".format(host, errstr))
return False
except OSError as err:
error("Can't connect to SSH server {0}: '{1}'".format(host, err))
return False
except:
error("Unknown error encountered while connecting to SSH server {}".format(host))
return False
info("SSH connection to {} opened successfully".format(host))
return r
def try_ssh(virtual_machine: VirtualMachine, retries: int = 10):
"""
Try to connect to a virtual machine using ssh
:param virtual_machine: the virtual machine
:param retries: the maximum of retries
"""
retry = 0
connected = False
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.load_system_host_keys()
while retry < retries and not connected:
try:
logger.debug('Trying ssh connection to %s %s out of %s retries',
virtual_machine.hostname, retry, retries)
ssh.connect(virtual_machine.hostname,
port=virtual_machine.ssh_port,
username=virtual_machine.ssh_username,
password=virtual_machine.ssh_password)
connected = True
except (BadHostKeyException, AuthenticationException, SSHException,
socket.error):
time.sleep(10)
retry += 1
if not connected:
raise Exception(
'[{}] Unable to connect to the machine after {} tries'.format(
virtual_machine.hostname, retry))
logger.info('Connection established to %s after %d out of %d retries',
virtual_machine.name, retry, retries)
def run(self):
if self.GITHUB_TOKEN is None:
logging.critical('No github OAuth token defined in the GITHUB_TOKEN env variable')
sys.exit(1)
if self.SSH_PKEY is None:
logging.critical('SSH_KEY not configured, please set it to you private SSH key file')
sys.exit(1)
github = Github(self.GITHUB_TOKEN)
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.SSH_PKEY = os.path.expanduser(self.SSH_PKEY)
orga = github.get_organization('nuxeo') # type: Organization
repo = orga.get_repo('nuxeo.com') # type: Repository
opened_pulls = [('/var/www/nuxeo.com/pr-%d.' % pull.number) + self.PREVIEW_DOMAIN for pull in repo.get_pulls()]
try:
proxy = ProxyCommand(('ssh -i %s -W 10.10.0.63:22 ' % self.SSH_PKEY) + self.BASTION_IP)
ssh.connect('10.10.0.63', username='******', sock=proxy, key_filename=self.SSH_PKEY)
_, stdout, _ = ssh.exec_command('ls -d /var/www/nuxeo.com/pr-*')
[ssh.exec_command('rm -rf ' + line.strip()) for line in stdout.readlines() if line.strip() not in opened_pulls]
ssh.close()
except SSHException, e:
logging.critical('Could work on remote: %s', e)
sys.exit(1)
def create_worker(host):
config = SSHConfig()
proxy = None
if os.path.exists(os.path.expanduser('~/.ssh/config')):
with open(os.path.expanduser('~/.ssh/config')) as f:
config.parse(f)
if host.hostname is not None and 'proxycommand' in config.lookup(
host.hostname):
proxy = ProxyCommand(config.lookup(host.hostname)['proxycommand'])
worker = SSHClient()
worker.load_system_host_keys()
worker.set_missing_host_key_policy(AutoAddPolicy())
# store data for later reference
worker.host = host.hostname
worker.username = host.username
worker.password = host.password
worker.key_filename = host.key_filename
worker.connect(hostname=host.hostname,
username=host.username,
password=host.password,
key_filename=host.key_filename,
sock=proxy)
return worker
def create_ssh_client(host, account, password, port=22):
client = SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(AutoAddPolicy())
client.connect(host, port=port, username=account, password=password)
return client
class PRISMASSHClient:
def __init__(self, address, user=None, passwd=None):
self.client = SSHClient()
self.client.set_missing_host_key_policy(AutoAddPolicy())
self.client.load_system_host_keys()
if user is None:
self.client.connect(address)
else:
if passwd is not None:
self.client.connect(address, username=user, password=passwd)
else:
self.client.connect(address, username=user)
def close(self):
self.client.close()
def list_from_directory(self, directory):
stdin, stdout, stderr = self.client.exec_command("ls " + directory)
result = stdout.read().splitlines()
return result
def download_file(self, remote_filepath, local_filepath):
sftp = self.client.open_sftp()
sftp.get(remote_filepath, local_filepath)
sftp.close()
def size_of_file(self, remote_filepath):
sftp = self.client.open_sftp()
stat = sftp.stat(remote_filepath)
sftp.close()
return stat.st_size
class Host:
def __init__(self, name):
self.name = name
self.ip = None
self.ssh_pkey = None
self._ssh = None
def __str__(self):
return self.name
def transport(self):
"""Returns the ssh transport for this host."""
if self._ssh == None:
self._ssh = SSHClient()
# import keys using paramikos method
self._ssh.load_system_host_keys()
# import openssh system host keys
host_keys_file = '/etc/ssh/ssh_known_hosts'
if os.path.exists(host_keys_file):
self._ssh.load_system_host_keys(host_keys_file)
# import saved host keys from/for multiapt
host_keys_file = os.getenv('HOME',
'/') + '/.ssh/known_hosts_multiapt'
if os.path.exists(host_keys_file):
self._ssh.load_host_keys(host_keys_file)
# now set our own filename for key save purposes
self._ssh._host_keys_filename = host_keys_file
# enable our own policy for host key problems
self._ssh.set_missing_host_key_policy(SSHAskHostKeyPolicy())
if Main.debug: print 'D: ssh.connect()'
self._ssh.connect(self.ip,
username=config.remote_username,
pkey=self.ssh_pkey)
return self._ssh
def turn_off(id):
server = SERVERS.get(id)
if not server:
return server_not_found()
exit_status = -1
ssh_settings = server['ssh']
client = SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(AutoAddPolicy())
client.connect(ssh_settings['address'],
username=ssh_settings['username'],
password=ssh_settings['password'])
stdin, stdout, stderr = client.exec_command('shutdown -p now')
#print("stdout: " + str(stdout.readlines()))
#print("stderr: " + str(stderr.readlines()))
exit_status = stdout.channel.recv_exit_status()
print("Shutdown, exit status: %s" % exit_status)
client.close()
return jsonify({"success": exit_status == 0})
def create_new_connection(self, index):
"""Attempts to create a new SSHThread object at the given index"""
# Spawns a paramiko SSHClient
ssh_client = SSHClient()
ssh_client.load_system_host_keys()
ssh_client.set_missing_host_key_policy(AutoAddPolicy())
# We loop until we can make a connection
while True:
# Get the next ssh connection name (hostname) to attempt
ssh_name = next(self.ssh_gen)
# Ignore if we already have a connection with this computer
if ssh_name in self.active_connections:
continue
# Try the connection and continue on a fail
try:
ssh_client.connect(ssh_name, timeout=TIMEOUT, port=PORT)
except Exception as e:
#print(e)
time.sleep(0.1)
continue
# Store connection in array and add hostname to set
self.ssh_threads[index] = SSHThread(ssh_client, self.command,
ssh_name, self.print_finish)
self.active_connections.add(ssh_name)
print("Connection made with %s." % ssh_name)
break
class Host:
def __init__(self, name):
self.name = name
self.ip = None
self.ssh_pkey = None
self._ssh = None
def __str__(self):
return self.name
def transport(self):
"""Returns the ssh transport for this host."""
if self._ssh == None:
self._ssh = SSHClient()
# import keys using paramikos method
self._ssh.load_system_host_keys()
# import openssh system host keys
host_keys_file = '/etc/ssh/ssh_known_hosts'
if os.path.exists(host_keys_file):
self._ssh.load_system_host_keys(host_keys_file)
# import saved host keys from/for multiapt
host_keys_file = os.getenv('HOME','/')+'/.ssh/known_hosts_multiapt'
if os.path.exists(host_keys_file):
self._ssh.load_host_keys(host_keys_file)
# now set our own filename for key save purposes
self._ssh._host_keys_filename = host_keys_file
# enable our own policy for host key problems
self._ssh.set_missing_host_key_policy(SSHAskHostKeyPolicy())
if Main.debug: print 'D: ssh.connect()'
self._ssh.connect(self.ip, username=config.remote_username, pkey=self.ssh_pkey)
return self._ssh
def run_remote_command(args, hostgroup_name, hostgroup, command):
"""Run the appropriate command on hosts in a given host group based on
the action being taken"""
client = SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(AutoAddPolicy())
ssh_key = None
host_results = {}
if args.ssh_key:
ssh_key = "%s/.ssh/%s" % (os.environ['HOME'], args.ssh_key)
for host in hostgroup:
try:
client.connect(host.address,
allow_agent=True,
username=os.getenv('USER'))
except Exception, e:
print "Error running remote command on (%s:%s) (%s)" % (
host.name, host.address, e)
continue
print "(%s:%s) => (%s)" % (host.name, host.address, command)
chan = client.get_transport().open_session()
chan.set_combine_stderr(True)
chan.exec_command(command)
dump_channel(chan)
rv = chan.recv_exit_status()
host_results[host.name] = rv
chan.close()
client.close()
_summarize_exit_code(rv)
class RemoteRunner(Runner):
def __init__(self, *args, **kwargs):
super(RemoteRunner, self).__init__(*args, **kwargs)
self.context
def start(self, command):
self.ssh_client = SSHClient()
self.ssh_client.load_system_host_keys()
self.ssh_client.set_missing_host_key_policy(AutoAddPolicy())
self.ssh_client.connect(self.context.remote_runner.hostname, username=self.context.remote_runner.username)
self.ssh_channel = self.ssh_client.get_transport().open_session()
if self.using_pty:
self.ssh_channel.get_pty()
self.ssh_channel.exec_command(command)
def stdout_reader(self):
return self.ssh_channel.recv
def stderr_reader(self):
return self.ssh_channel.recv_stderr
def default_encoding(self):
return locale.getpreferredencoding(True)
def wait(self):
return self.ssh_channel.recv_exit_status()
def returncode(self):
return self.ssh_channel.recv_exit_status()
def create_backup_repository(service):
"""
- create filesystem folders
- store ssh key
- create subaccount
"""
# Create folder and SSH key
client = SSHClient()
client.load_system_host_keys()
client.connect(**settings.STORAGE_SERVER)
ftp = client.open_sftp()
dirname = str(uuid4())
ftp.mkdir(dirname)
ftp.chdir(dirname)
ftp.mkdir(".ssh")
ftp.chdir(".ssh")
with ftp.open("authorized_keys", "w") as handle:
handle.write(service.last_report.ssh_key)
# Create account on the service
url = "https://robot-ws.your-server.de/storagebox/{}/subaccount".format(
settings.STORAGE_BOX)
response = requests.post(
url,
data={
"homedirectory": "weblate/{}".format(dirname),
"ssh": "1",
"external_reachability": "1",
"comment": "Weblate backup service {}".format(service.pk),
},
auth=(settings.STORAGE_USER, settings.STORAGE_PASSWORD),
)
data = response.json()
return "ssh://{}@{}:23/./backups".format(data["subaccount"]["username"],
data["subaccount"]["server"])
def create_worker(host):
config = SSHConfig()
proxy = None
if os.path.exists(os.path.expanduser('~/.ssh/config')):
config.parse(open(os.path.expanduser('~/.ssh/config')))
if host.hostname is not None and \
'proxycommand' in config.lookup(host.hostname):
proxy = ProxyCommand(config.lookup(host.hostname)['proxycommand'])
# proxy = paramiko.ProxyCommand("ssh -o StrictHostKeyChecking=no [email protected] nc 118.138.239.241 22")
worker = SSHClient()
worker.load_system_host_keys()
worker.set_missing_host_key_policy(AutoAddPolicy())
worker.hostname = host.hostname # store all this for later reference (e.g., logging, reconnection)
worker.username = host.username
worker.password = host.password
worker.proxy = proxy
if not host.key_filename is None:
worker.pkey = RSAKey.from_private_key_file(host.key_filename,
host.key_password)
else:
worker.pkey = None
# time.sleep(4)
# worker.connect(hostname=host.hostname, username=host.username, password=host.password, key_filename=host.key_filename, sock=proxy, timeout=3600)
worker.connect(hostname=host.hostname,
username=host.username,
password=host.password,
pkey=worker.pkey,
sock=proxy)
return worker
class RemoteClient(object):
def __init__(self, host, ip=None, user='******'):
self.host = host
self.ip = ip
self.user = user
self.client = SSHClient()
self.sftpclient = None
self.client.set_missing_host_key_policy(AutoAddPolicy())
self.client.load_system_host_keys()
logging.debug("RemoteClient created for host: %s", host)
def startup(self):
try:
logging.debug("Trying to connect to remote server %s", self.host)
self.client.connect(self.host, port=22, username=self.user)
self.sftpclient = self.client.open_sftp()
except PasswordRequiredException:
raise ClientNotSetupException('Pubkey is encrypted.')
except SSHException as e:
raise ClientNotSetupException(e)
except:
if self.ip:
logging.warning("Connection with hostname failed. Retrying "
"with IP")
self._try_with_ip()
else:
logging.error("Connection to %s failed.", self.host)
raise ClientNotSetupException('Could not connect to the host.')
def _try_with_ip(self):
try:
logging.debug("Connecting to IP:%s User:%s", self.ip, self.user)
self.client.connect(self.ip, port=22, username=self.user)
self.sftpclient = self.client.open_sftp()
except PasswordRequiredException:
raise ClientNotSetupException('Pubkey is encrypted.')
except SSHException as e:
raise ClientNotSetupException(e)
except socket.error:
logging.error("Connection with IP (%s) failed.", self.ip)
raise ClientNotSetupException('Could not connect to the host.')
def run(self, command):
if not self.client:
raise ClientNotSetupException(
'Cannot run procedure. Client not initialized')
buffers = self.client.exec_command(command)
output = []
for buf in buffers:
try:
output.append(buf.read())
except IOError:
output.append('')
return tuple(output)
def _remote(self):
client = SSHClient()
client.load_system_host_keys()
client.connect(hostname=self.host,
port=self.port,
username=self.username,
password=self.password)
return client
class TransferAgent(object):
def __init__(self):
self.client = SSHClient()
self.client.load_system_host_keys()
self.sftp_client = None
def load_host_keys(self, filepath):
self.client.load_host_keys(filepath)
def connect(self, host, username=None, port=22, private_key_file=None):
self.client.connect(host, username=username, port=port,
key_filename=private_key_file)
self.sftp_client = self.client.open_sftp()
return self
def close(self):
self.sftp_client.close()
self.client.close()
def dest_file_exists(self, destpath):
try:
self.sftp_client.stat(destpath)
except:
return False
return True
def transfer(self, srcpath, destdir, callback=None, randomize=True,
name_length=7, max_attempts=4):
if not randomize:
destpath = self.transform_path(srcpath, destdir)
return self.do_transfer(srcpath, destpath, callback=callback)
for _ in range(max_attempts):
name = generate_random_name(name_length)
destpath = self.transform_path(srcpath, destdir, rename=name)
if self.dest_file_exists(destpath):
continue
return self.do_transfer(srcpath, destpath, callback=callback)
raise TransferError("Exceeded max transfer attempts")
def do_transfer(self, src, dest, callback=None):
self.sftp_client.put(src, dest, callback=callback)
return dest
def transform_path(self, srcpath, destdir, rename=None):
basename = os.path.basename(srcpath)
_, ext = os.path.splitext(basename)
if rename:
basename = rename + ext
return posixpath.join(destdir, basename)
def execute_cmd(ip_addr, cmd):
client = SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(AutoAddPolicy())
client.connect(ip_addr,
username=dtacq_un,
password=dtacq_pw)
_, aout, aerr = client.exec_command(cmd)
return aout.read(), aerr.read()
def ssh_connection(self, storage):
connection = SSHClient()
connection.load_system_host_keys()
vals = storage.config
connection.connect(hostname=vals.get('hostname', None),
port=vals.get('port', 0),
username=vals.get('user', None),
password=vals.get('password', None))
sftp = connection.open_sftp()
return connection, sftp
def _connect(self):
client = SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(AutoAddPolicy())
try:
client.connect(hostname=self.host, port=self.port, username=self.user, password=self.passwd, timeout=3)
self.closed = False
return client
except AuthenticationException, e:
return e
def run(self, project):
if not self.vm.start_VM():
return -1, ''
if not os.path.exists(os.path.join(project.tempdir, project.target)):
raise FileNotFoundError('Error: Executable file has not been created!')
copy_to_vm = [os.path.join(project.tempdir, project.target)]
copy_from_vm = ['CUnitAutomated-Results.xml']
print('Connecting to remote machine...')
client = SSHClient()
client.set_missing_host_key_policy(AutoAddPolicy())
client.load_system_host_keys()
client.connect(self.host, username=self.username, password=self.password, timeout=10)
return_code = 0
data = ''
with client.open_sftp() as sftp:
try:
self.rmtree(sftp, self.remote_path)
except FileNotFoundError:
pass
try:
sftp.mkdir(self.remote_path)
except OSError:
pass
for f in copy_to_vm:
remote_file = os.path.join(self.remote_path, os.path.basename(f))
sftp.put(f, remote_file)
sftp.chmod(remote_file, 0o777)
stdin, stdout, stderr = client.exec_command('cd {}; timeout {}s {}'.format(self.remote_path, self.timeout, remote_file))
return_code = stdout.channel.recv_exit_status()
print('[Remote] Error code: {}'.format(return_code))
stdout_string = '[Remote] ' + ''.join(stdout)
if stdout_string:
print('[Remote] STDOUT:')
print(stdout_string)
stderr_string = '[Remote] ' + ''.join(stderr)
if stderr_string:
print('[Remote] STDERR:')
print(stderr_string)
for f in copy_from_vm:
# get all result files
remote_file = os.path.join(self.remote_path, os.path.basename(f))
try:
with tempfile.TemporaryFile() as local_file:
sftp.getfo(remote_file, local_file)
local_file.seek(0)
data = local_file.read()
except FileNotFoundError:
print('Remote file not found!')
# delete all files in home directory
self.rmtree(sftp, self.remote_path)
client.close()
if self.shutdown_vm_after:
self.vm.stop_VM()
return return_code, data
def ssh_instance_pass(step, instance_name, password):
out = bash('ssh %s -A -f -L 11112:%s:22 root@%s -N' % (SSH_OPTS, world.instances[instance_name]['ip'], config['cloud']['master']))
if out.successful():
from paramiko.client import SSHClient
client = SSHClient()
client.load_system_host_keys()
client.connect('localhost', username='******', password=password, port=11112)
#stdin, stdout, stderr = client.exec_command('hostname')
client.exec_command('hostname')
print 'done'
client.close()
def place_code_on_workers():
client = SSHClient()
client.load_system_host_keys()
for worker in worker_list.keys():
client.connect(worker, username=USERNAME)
scp = SCPClient(client.get_transport())
scp.put(['worker.py'], remote_path='/home/{}/'.format(USERNAME))
print("{color}[{}] {}{end_color}".format(
worker, "Copied worker.py",
color=worker_list[worker]['text_color'],
end_color=Colors.ENDC))
client.close()
def _sftp_connect(self):
connection = SSHClient()
connection.load_system_host_keys()
connection.connect(
hostname=self.integration_id.invoice_id.partner_id.ssh_server,
port=int(self.integration_id.invoice_id.partner_id.ssh_port),
username=self.integration_id.invoice_id.partner_id.ssh_name,
password=self._decrypt_value(
self.integration_id.invoice_id.partner_id.ssh_pass),
)
sftp = connection.open_sftp()
return connection, sftp
class datastore_sftp(datastore):
""" collect of independent functions, not really a Class """
def __init__(self):
# Remote path for saving all resources
self.base_folder = settings.SFTP_DATASTORE_REMOTEBASEFOLDER
# Local base folder for saving temporary files before upload
self.tmp_folder = settings.SFTP_DATASTORE_LOCALTMPFOLDER
# url for donwloading resources
self.public_base_url = settings.SFTP_BASE_URL
self.buckets = []
self.connection = None
self.ssh_client = SSHClient()
self.ssh_client.set_missing_host_key_policy(AutoAddPolicy())
self.ssh_client.load_system_host_keys()
self.sftp = None
def connect(self):
""" don't use at INIT because it hangs all application"""
logger = logging.getLogger(__name__)
logger.error('Connecting SFTP %s:%s (%s, %s)' %(
settings.SFTP_DATASTORE_HOSTNAME,
settings.SFTP_DATASTORE_PORT,
settings.SFTP_DATASTORE_USER,
settings.SFTP_DATASTORE_PASSWORD)
)
# TODO: Remove
con = sftp.Connection(
host=settings.SFTP_DATASTORE_HOSTNAME,
port=settings.SFTP_DATASTORE_PORT,
username=settings.SFTP_DATASTORE_USER,
password=settings.SFTP_DATASTORE_PASSWORD,
log=True
)
self.connection = con
#
self.ssh_client.connect(
settings.SFTP_DATASTORE_HOSTNAME,
port=settings.SFTP_DATASTORE_PORT,
username=settings.SFTP_DATASTORE_USER,
password=settings.SFTP_DATASTORE_PASSWORD
)
self.sftp = self.ssh_client.open_sftp()
# list all buckets (folders)
try:
self.buckets = self.sftp.listdir(path=self.base_folder)
logger.error('Buckets: %s' %str(self.buckets))
except Exception, e:
logger.error('Error Connecting SFTP %s' % str(e))
self.sftp.close()
class datastore_sftp(datastore):
""" collect of independent functions, not really a Class """
def __init__(self):
# Remote path for saving all resources
self.base_folder = settings.SFTP_DATASTORE_REMOTEBASEFOLDER
# Local base folder for saving temporary files before upload
self.tmp_folder = settings.SFTP_DATASTORE_LOCALTMPFOLDER
# url for donwloading resources
self.public_base_url = settings.SFTP_BASE_URL
self.buckets = []
self.connection = None
self.ssh_client = SSHClient()
self.ssh_client.set_missing_host_key_policy(AutoAddPolicy())
self.ssh_client.load_system_host_keys()
self.sftp = None
def connect(self):
""" don't use at INIT because it hangs all application"""
logger = logging.getLogger(__name__)
logger.info('Connecting SFTP %s:%s (%s, %s)' %(
settings.SFTP_DATASTORE_HOSTNAME,
settings.SFTP_DATASTORE_PORT,
settings.SFTP_DATASTORE_USER,
settings.SFTP_DATASTORE_PASSWORD)
)
# TODO: Remove
con = sftp.Connection(
host=settings.SFTP_DATASTORE_HOSTNAME,
port=settings.SFTP_DATASTORE_PORT,
username=settings.SFTP_DATASTORE_USER,
password=settings.SFTP_DATASTORE_PASSWORD,
log=True
)
self.connection = con
#
self.ssh_client.connect(
settings.SFTP_DATASTORE_HOSTNAME,
port=settings.SFTP_DATASTORE_PORT,
username=settings.SFTP_DATASTORE_USER,
password=settings.SFTP_DATASTORE_PASSWORD
)
self.sftp = self.ssh_client.open_sftp()
# list all buckets (folders)
try:
self.buckets = self.sftp.listdir(path=self.base_folder)
logger.info('Buckets: %s' %str(self.buckets))
except Exception, e:
logger.error('Error Connecting SFTP %s' % str(e))
self.sftp.close()
def _efact_connect(self):
connection = SSHClient()
connection.load_system_host_keys()
connection.connect(hostname=self.env["ir.config_parameter"].get_param(
"account.invoice.efact.server", default=None),
port=int(self.env["ir.config_parameter"].get_param(
"account.invoice.efact.port", default=None)),
username=self.env["ir.config_parameter"].get_param(
"account.invoice.efact.user", default=None),
password=self.env["ir.config_parameter"].get_param(
"account.invoice.efact.password", default=None))
sftp = connection.open_sftp()
return connection, sftp
class SSHOps:
def __init__(self):
self.client = SSHClient()
self.client.load_system_host_keys()
self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.client.connect("192.168.0.2", username='******', password='******')
def runCommand(self, comando):
stdin, stdout, stderr = self.client.exec_command(comando)
if stderr.channel.recv_exit_status() != 0:
return {"status": 1, "message": stderr.read()}
else:
return {"status": 0, "message": stdout.read()}
def fetch_moosefs_drive_content(self, remote_host_ip: str) -> dict:
# A sample result dictionary
# result_dict = {
# 'files': ['temp_file101', 'temp_file102'],
# 'cotent': [file 1 content here', 'file 2 content here' ]
# }
try:
result_dict = dict()
result_dict['files'] = list()
result_dict['content'] = list()
mfsClientVM = SSHClient()
mfsClientVM.set_missing_host_key_policy(paramiko.AutoAddPolicy())
mfsClientVM.load_system_host_keys()
mfsClientVM.connect(hostname=remote_host_ip,
username=self.remote_host_username)
print("Fetching file and its content on VM with IP: " +
remote_host_ip)
# Fetch file name on client VM
stdin, stdout, stderr = mfsClientVM.exec_command(
'cd /mnt/mfs/; ls')
outlines = stdout.readlines()
stdin.close()
file_name = ''.join(outlines)
file_name = str(file_name).rstrip('\n')
result_dict['files'].append(file_name)
print('File name is: ' + file_name)
# Fetch file content on client VM
stdin, stdout, stderr = mfsClientVM.exec_command(
'cd /mnt/mfs/; cat ' + file_name)
outlines = stdout.readlines()
stdin.close()
file_content = ''.join(outlines)
file_content = str(file_content).rstrip('\n')
result_dict['content'].append(file_content)
print('File Content is: ' + file_content)
mfsClientVM.close()
return result_dict
except Exception as e:
print(
"Something went wrong while fetching the moosefs drive content"
)
print("Error Details: " + str(e))
return None
def connect_ssh(host):
""" Function to connect remote machine using SSH"""
hostname, port, username, password = parsehost(host)
client = SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(AutoAddPolicy())
try:
client.connect(hostname=hostname,
port=port,
username=username,
password=password)
return True, client
except AuthenticationException as e_ex:
return False, str(e_ex)
def fromPrivateKey(host,port=22,username=None,password=None,private_key=None):
"""deprecated see connect_v2 """
#ssh -i /path/to//private_key user@localhost -p 2222
#ssh -i /path/to//private_key username@host -p port
if SSHClient is None:
raise Exception("Paramiko not installed")
src = SSHClientSource()
print(host,port,username,password,private_key)
pkey = None
if private_key: # non-null, non-empty
passphrase = "" # TODO: support passphrases
pkey=paramiko.RSAKey.from_private_key_file(\
private_key,passphrase)
client = SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
print("\n---\nConnect With: host=%s:%s user=%s privatekey=`%s`\n"%(\
host,port,username,private_key))
try:
if pkey:
client.connect(host,port=port,
username=username,
pkey=pkey,timeout=1.0,compress=True)
else:
client.connect(host,port=port,
username=username,password=password,
timeout=1.0,compress=True)
except BadHostKeyException as e:
sys.stderr.write("got: %s\n"%e.key.asbytes());
sys.stderr.write("expected: %s\n"%e.expected_key.asbytes());
msg = "Connection error on %s:%s using privatekey=%s\n"%(\
host,port,private_key)
msg += "you may need to clear ~/.ssh/known_hosts "+\
"for entries related to %s:%s\n"%(host,port);
sys.stderr.write(msg);
raise Exception(msg);
src.client = client
src.ftp = client.open_sftp()
src.host=host
src.port=port
return src
def start_training():
client_list = []
for worker_name in worker_list.keys():
client = SSHClient()
client.load_system_host_keys()
client.connect(worker_name, username=USERNAME)
channel = client.get_transport().open_session()
client_list.append((worker_name, client, channel, {'stdout': "", 'stderr': ""}))
for worker_name, client, channel, outs in client_list:
channel.exec_command('python3 worker.py')
at_least_one_worker_still_running = True
while at_least_one_worker_still_running:
at_least_one_worker_still_running = False
for worker_name, client, channel, outs in client_list:
rl, wl, xl = select([channel], [], [], 0.0)
if not channel.exit_status_ready():
at_least_one_worker_still_running = True
if len(rl) > 0:
outs['stdout'] += channel.recv(RECV_BUFFER_SIZE).decode('utf-8')
outs['stderr'] += channel.recv_stderr(RECV_BUFFER_SIZE).decode('utf-8')
if channel.exit_status_ready():
if len(outs['stdout']) > 0:
print("{color}[{}] {}{end_color}".format(
worker_name, outs['stdout'],
color=worker_list[worker_name]['text_color'],
end_color=Colors.ENDC))
outs['stdout'] = ""
if len(outs['stderr']) > 0:
print("{color}[{}] {}{end_color}".format(
worker_name, outs['stderr'],
color=worker_list[worker_name]['text_color'],
end_color=Colors.ENDC))
outs['stderr'] = ""
lines_stdout = outs['stdout'].split("\n")
outs['stdout'] = lines_stdout[-1]
lines_stderr = outs['stderr'].split("\n")
outs['stderr'] = lines_stderr[-1]
for line in lines_stdout[:-1]:
print("{color}[{}] {}{end_color}".format(
worker_name, line,
color=worker_list[worker_name]['text_color'],
end_color=Colors.ENDC))
for line in lines_stderr[:-1]:
print("{color}[{}] {}{end_color}".format(
worker_name, line,
color=worker_list[worker_name]['warn_color'],
end_color=Colors.ENDC))
for worker_name, client, channel, outs in client_list:
client.close()
def executar_comando_remoto(com):
hosts = ["192.168.0.2"]
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
for h in hosts:
try:
ssh.connect(h)
stdin,stdout,stderr = ssh.exec_command(com)
if stderr.channel.recv_exit_status() != 0:
print stderr.read()
else:
print stdout.read()
except Exception as e:
print "Nao conseguiu conectar ao servidor %s"%e
def executar_comando_remoto(com):
hosts = ["192.168.0.2"]
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
for h in hosts:
try:
ssh.connect(h)
stdin, stdout, stderr = ssh.exec_command(com)
if stderr.channel.recv_exit_status() != 0:
print stderr.read()
else:
print stdout.read()
except Exception as e:
print "Nao conseguiu conectar ao servidor %s" % e
class SshOps:
def __init__(self):
self.ssh = SSHClient()
self.ssh.load_system_host_keys()
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.ssh.connect(hostname='127.0.0.1',
username='******',
password='******')
def runCommand(self, command):
stdin, stdout, stderr = self.ssh.exec_command(command)
if stderr.channel.recv_exit_status() != 0:
return {"status": 1, "message": stderr.read()}
else:
return {"status": 0, "message": stdout.read()}
class SshOps:
def __init__(self):
self.client = SSHClient()
self.client.load_system_host_keys()
self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
config = ConfigParser.ConfigParser()
config.read("/home/forlinux/www/4803-Python/ProjetoDexter/config.cfg")
self.client.connect(config.get("docker","address"))
def runCommand(self,command):
stdin,stdout,stderr = self.client.exec_command(command)
if stderr.channel.recv_exit_status() != 0:
return {"status":1,"message":stderr.read()}
else:
return {"status":0,"message":stdout.read()}
class SshOps:
def __init__(self):
self.ssh = SSHClient()
self.ssh.load_system_host_keys()
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
self.ssh.connect(hostname='xxx', username='******', password='******')
except Exception as e:
print("Erro {}".format(e))
def run_command(self, command):
stdin, stdout, stderr = self.ssh.exec_command(command)
if stderr.channel.recv_exit_status() != 0:
return {"status": 1, "message": stderr.read()}
else:
return {"status": 0, "message": stdout.read()}
def stop_instance(instance):
try:
stop_script_location = instance.tags['stop_script']
except KeyError:
stop_script_location = '~/shutdown.sh'
client = SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.load_system_host_keys()
client.connect(instance.ip_address, username="******")
stdin, stdout, stderr = client.exec_command('%s "%s" "%s"' % (stop_script_location, app.config["API_KEY"], app.config["MY_URL"] + "/api/v1/stats"))
try:
for s in [stdin, stdout, stderr]:
app.logger.info(s.read())
except IOError:
pass
client.close()
def init_ssh_connection(host,
username=None,
password=None,
port=22,
dryrun=False):
import paramiko
from paramiko.client import SSHClient
if dryrun:
return {'ssh_conn': None}
ssh_conn = SSHClient()
ssh_conn.load_system_host_keys()
ssh_conn.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_conn.connect(host, username=username, password=password)
return {'ssh_conn': ssh_conn}
def executar_comando(usuario, senha):
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect('localhost', username =usuario, password=senha)
stdin, stdout, stderr = ssh.exec_command('ls -la')
if stderr.channel.recv_exit_status() != 0:
print stderr.read()
else:
print stdout.read()
except Exception as e:
print 'Falhou ao conectar: ', e
class SSH:
def __init__(self):
self.servidor = "192.168.0.2"
self.ssh = SSHClient()
self.ssh.load_system_host_keys()
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.ssh.connect(self.servidor)
def executarComandoRemoto(self,com):
try:
stdin,stdout,stderr = self.ssh.exec_command(com)
if stderr.channel.recv_exit_status() != 0:
return stderr.read()
else:
return stdout.read()
except Exception as e:
print "Nao conseguiu conectar ao servidor %s"%e
def executar_comando(servidor,senha):
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect("localhost",username="******",password=senha)
stdin, stdout, stderr = ssh.exec_command("ls -la")
if stderr.channel_recv_exit_status() != 0:
print stderr.read()
else:
print stdout.read()
except Exception as e:
print "A conexao falhou"
def get_iodef(
iodef_name_list,
server='140.117.101.15',
port=22, username='******',
password=None,
file_locate='/etc/iodef/{}/{}.xml'
):
logging.debug('Generating ssh client...')
client = SSHClient()
logging.debug('Loading system host key...')
client.load_system_host_keys()
# disable following if known_hosts
logging.debug('Setting missing host key policy to AutoAddPolicy...')
client.set_missing_host_key_policy(AutoAddPolicy())
logging.debug('Connecting to ssh {}@{}:{}...'.format(
username, server, int(port),
))
client.connect(
server, port=int(port), username=username, password=password,
timeout=5,
)
logging.debug('Generating sftp...')
sftp_client = client.open_sftp()
return_dict = dict()
for each_iodef_name in iodef_name_list:
try:
quote = '/'.join(each_iodef_name.split('-')[:-1])
file_path = file_locate.format(quote, each_iodef_name)
logging.debug('Touching remote file {}'.format(file_path))
with sftp_client.open(file_path) as f:
logging.debug('Loading remote file {}'.format(file_path))
return_dict[each_iodef_name] = f.read()
except Exception as e:
logging.exception('Unable to retrieve {}'.format(each_iodef_name))
return return_dict
class GraderConfiguration:
def __init__(self, single_class_name=None, on_grading_server=False):
self.on_grading_server = on_grading_server
self.config_dir = os.path.expanduser('~/.config/grader')
if not os.path.isdir(self.config_dir):
error = '{0} does not exist'.format(self.config_dir)
raise ConfigurationError(error)
self.config_filename = os.path.join(self.config_dir, 'grader.conf')
if not os.path.isfile(self.config_filename):
error = '{0} does not exist'.format(self.config_filename)
raise ConfigurationError(error)
parser = configparser.ConfigParser()
try:
parser.read(self.config_filename)
except configparser.MissingSectionHeaderError:
error = 'Error reading {0}'.format(self.config_filename)
raise ConfigurationError(error)
sections = parser.sections()
sections.sort()
if sections != ['email', 'server']:
error = 'Config file requires two sections: server and email'
raise ConfigurationError(error)
try:
self.username = parser.get('server', 'username')
self.host = parser.get('server', 'host')
self.group = parser.get('server', 'group')
self.from_name = parser.get('email', 'from_name')
self.from_address = parser.get('email', 'from_address')
if parser.has_option('email', 'smtp_server'):
self.smtp_server = parser.get('email', 'smtp_server')
else:
self.smtp_server = None
if parser.has_option('email', 'smtp_port'):
self.smtp_port = parser.get('email', 'smtp_port')
else:
self.smtp_port = None
if parser.has_option('email', 'email_username'):
self.email_username = parser.get('email', 'email_username')
else:
self.email_username = None
if parser.has_option('email', 'email_password'):
self.email_password = parser.get('email', 'email_password')
else:
self.email_password = None
except configparser.NoOptionError as e:
raise ConfigurationError(e.message)
if self.on_grading_server:
self.ssh = None
else:
try:
self.ssh = SSHClient()
self.ssh.load_system_host_keys()
self.ssh.connect(self.host, username=self.username)
except OSError as e:
error = ('Error opening SSH connection to {0}:\n{1}'
.format(self.host, e))
raise ConfigurationError(error)
try:
self.home_dir = home_dir_from_username(self.username, ssh=self.ssh)
except CommandError as e:
raise ConfigurationError('Error connecting to {0} via SSH:\n{1}'
.format(self.host, e))
self.students_by_class = {}
self.students_csv_filenames_by_class = {}
self.students_by_username = {}
for filename in os.listdir(self.config_dir):
if not filename.endswith('.csv'):
continue
class_name, _ = os.path.splitext(filename)
if single_class_name is not None \
and class_name != single_class_name:
continue
if ' ' in class_name:
error = 'Error in class name "{0}".\n'.format(class_name)
error += 'Class names may not contain spaces'
raise ConfigurationError(error)
self.students_by_class[class_name] = []
filepath = os.path.join(self.config_dir, filename)
self.students_csv_filenames_by_class[class_name] = filepath
try:
with open(filepath) as f:
rows = list(csv.reader(f))
except OSError as e:
error = 'Error opening {0}:\n{1}'.format(filepath, e)
raise ConfigurationError(error)
for row in rows:
try:
student_row = row
student = Student(*student_row, ssh=self.ssh)
self.students_by_class[class_name].append(student)
self.students_by_username[student.username] = student
except TypeError:
row_str = ','.join(row)
error = 'Error in {0} at this row:\n{1}'.format(filepath,
row_str)
raise ConfigurationError(error)
if len(self.students_by_class) == 0:
raise ConfigurationError('No classes defined')
for class_name, students in self.students_by_class.items():
if len(students) == 0:
error = 'No students in {0}'.format(class_name)
raise ConfigurationError(error)
def get_assignments(self, class_name):
assignments_dir = os.path.join(self.home_dir, class_name, 'assignments')
if not directory_exists(assignments_dir, ssh=self.ssh):
raise ConfigurationError('{0} does not exist'
.format(assignments_dir))
return list_directory(assignments_dir, ssh=self.ssh)
def get_reports_repo_dir(self, class_name, assignment):
repo_dir = os.path.join(self.home_dir, class_name, 'assignments',
assignment, assignment + '_reports.git')
return repo_dir
#!/usr/bin/python
from task_constants import *
from paramiko.client import SSHClient
client = SSHClient()
client.load_system_host_keys()
client.connect(hostname = opt_server,
username = opt_username,
key_filename = opt_keyfile)
output = client.exec_command('ls')
print output
from scp import SCPClient
scp = SCPClient(client.get_transport())
scp.put("ssh.py", "ssh.py")
scp.put("scp.py", "scp.py")
print client.exec_command('md5sum ssh.py scp.py')
client.close()
"""
The UUID of a task will serve as the filename for all the files associated with the task.
As well as the handle for marking/handling the task.
A task has these states
- TODO (we should verify that the file has been generated, if not, alert)
copy the file on WEB01 to OPT01
mark the task as GOING_TO_RUN
submit the job as qsub (a shell script with param)
(qsub should be able to mark something as READY_TO_MAIL
class SFTPClient(object):
"""Dynamic extension on paramiko's SFTPClient."""
MAX_PACKET_SIZE = SFTPFile.__dict__['MAX_REQUEST_SIZE']
ssh_client = None
client = None
raise_exceptions = False
original_arguments = {}
debug = False
_log = logging.getLogger(LOG_NAME)
_dircache = []
def __init__(self, **kwargs):
"""Constructor."""
self.original_arguments = kwargs.copy()
self._connect(**kwargs)
def __enter__(self):
"""For use with a with statement."""
return self
def __exit__(self, exc_type, exc_val, exc_tb):
"""For use with a with statement."""
self.close_all()
def _connect(self, **kwargs):
kwargs_to_paramiko = dict(
look_for_keys=kwargs.pop('look_for_keys', True),
username=kwargs.pop('username'),
port=kwargs.pop('port', 22),
allow_agent=False,
timeout=kwargs.pop('timeout', None),
)
host = kwargs.pop('hostname', 'localhost')
password = kwargs.pop('password')
keepalive = kwargs.pop('keepalive', 5)
if password:
kwargs_to_paramiko['password'] = password
self.raise_exceptions = kwargs.pop('raise_exceptions', False)
self.ssh_client = SSHClient()
self.ssh_client.load_system_host_keys()
self.ssh_client.connect(host, **kwargs_to_paramiko)
self.client = self.ssh_client.open_sftp()
channel = self.client.get_channel()
channel.settimeout(kwargs_to_paramiko['timeout'])
channel.get_transport().set_keepalive(keepalive)
# 'Extend' the SFTPClient class
is_reconnect = kwargs.pop('is_reconnect', False)
members = inspect.getmembers(self.client,
predicate=inspect.ismethod)
self._log.debug('Dynamically adding methods from original SFTPClient')
for (method_name, method) in members:
if method_name[0:2] == '__' or method_name == '_log':
self._log.debug('Ignorning {}()'.format(method_name))
continue
if not is_reconnect and hasattr(self, method_name):
raise AttributeError('Not overwriting property "{}". This '
'version of Paramiko is not '
'supported.'.format(method_name))
self._log.debug('Adding method {}()'.format(method_name))
setattr(self, method_name, method)
def close_all(self):
"""Close client and SSH client handles."""
self.client.close()
self.ssh_client.close()
def clear_directory_cache(self):
"""Reset directory cache."""
self._dircache = []
def listdir_attr_recurse(self, path='.'):
"""List directory attributes recursively."""
for da in self.client.listdir_attr(path=path):
is_dir = da.st_mode & 0o700 == 0o700
if is_dir:
try:
for x in self.listdir_attr_recurse(
path_join(path, da.filename)):
yield x
except IOError as e:
if self.raise_exceptions:
raise e
else:
yield (path_join(path, da.filename), da,)
def _get_callback(self, start_time, _log):
def cb(tx_bytes, total_bytes):
total_time = datetime.now() - start_time
total_time = total_time.total_seconds()
total_time_s = floor(total_time)
if (total_time_s % LOG_INTERVAL) != 0:
return
nsize_tx = naturalsize(tx_bytes,
binary=True,
format='%.2f')
nsize_total = naturalsize(total_bytes,
binary=True,
format='%.2f')
speed_in_s = tx_bytes / total_time
speed_in_s = naturalsize(speed_in_s,
binary=True,
format='%.2f')
_log.info('Downloaded {} / {} in {} ({}/s)'.format(
nsize_tx,
nsize_total,
naturaldelta(datetime.now() - start_time),
speed_in_s,
total_time_s))
return cb
def mirror(self,
path='.',
destroot='.',
keep_modes=True,
keep_times=True,
resume=True):
"""
Mirror a remote directory to a local location.
path is the remote directory. destroot must be the location where
destroot/path will be created (the path must not already exist).
keep_modes and keep_times are boolean to ensure permissions and time
are retained respectively.
Pass resume=False to disable file resumption.
"""
n = 0
resume_seek = None
cwd = self.getcwd()
for _path, info in self.listdir_attr_recurse(path=path):
if info.st_mode & 0o700 == 0o700:
continue
dest_path = path_join(destroot, dirname(_path))
dest = path_join(dest_path, basename(_path))
if dest_path not in self._dircache:
try:
makedirs(dest_path)
except OSError:
pass
self._dircache.append(dest_path)
if isdir(dest):
continue
try:
with open(dest, 'rb'):
current_size = os.stat(dest).st_size
if current_size != info.st_size:
resume_seek = current_size
if resume:
self._log.info('Resuming file {} at {} '
'bytes'.format(dest, current_size))
raise IOError() # ugly goto
except IOError:
while True:
try:
# Only size is used to determine complete-ness here
# Hash verification is in the util module
if resume_seek and resume:
read_tuples = []
n_reads = ceil((info.st_size - resume_seek) /
self.MAX_PACKET_SIZE) - 1
n_left = ((info.st_size - resume_seek) %
self.MAX_PACKET_SIZE)
offset = 0
for n in range(n_reads):
read_tuples.append((resume_seek + offset,
self.MAX_PACKET_SIZE,))
offset += self.MAX_PACKET_SIZE
read_tuples.append((resume_seek + offset, n_left,))
with self.client.open(_path) as rf:
with open(dest, 'ab') as f:
f.seek(resume_seek)
resume_seek = None
for chunk in rf.readv(read_tuples):
f.write(chunk)
else:
dest = realpath(dest)
self._log.info('Downloading {} -> '
'{}'.format(_path, dest))
start_time = datetime.now()
self.client.get(_path, dest)
self._get_callback(start_time, self._log)(
info.st_size, info.st_size)
# Do not count files that were already downloaded
n += 1
break
except (socket.timeout, SFTPError) as e:
# Resume at position - 10 bytes
resume_seek = os.stat(dest).st_size - 10
if isinstance(e, socket.timeout):
self._log.error('Connection timed out')
else:
self._log.error('{!s}'.format(e))
if resume:
self._log.info('Resuming GET {} at {} '
'bytes'.format(_path,
resume_seek))
else:
self._log.debug('Not resuming (resume = {}, '
'exception: {})'.format(resume,
e))
raise e
self._log.debug('Re-establishing connection')
self.original_arguments['is_reconnect'] = True
self._connect(**self.original_arguments)
if cwd:
self.chdir(cwd)
# Okay to fix existing files even if they are already downloaded
try:
if keep_modes:
chmod(dest, info.st_mode)
if keep_times:
utime(dest, (info.st_atime, info.st_mtime,))
except IOError:
pass
return n
def __str__(self):
"""Return string representation."""
return '{} (wrapped by {}.SFTPClient)'.format(
str(self.client), __name__)
__unicode__ = __str__
#!/usr/bin/python
from paramiko.client import SSHClient
import paramiko
from datetime import datetime
hosts = ["192.168.1.10","192.168.1.106","192.168.1.100"]
ssh = SSHClient()
# ler as chaves
ssh.load_system_host_keys()
# se eh um server que nao conheco fazer add
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
for h in hosts:
try:
print "Conectando no servidor %s na data %s"%(h,datetime.now())
ssh.connect(h)
stdin,stdout,stderr = ssh.exec_command('w')
if stderr.channel.recv_exit_status() != 0:
print stderr.read()
else:
print stdout.read()
print "Saindo do servidor %s na data %s"%(h,datetime.now())
except Exception as e:
print "Nao conseguiu conectar ao servidor: %s"%e
def ssh_client(server_ip, server_user, server_key):
client = SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.load_system_host_keys()
client.connect(server_ip, username=server_user, key_filename=server_key)
return client
class SSHperformer(Performer):
provider_name = 'ssh'
settings_class = SSHPerformerSettings
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.client = None
self.logger = getLogger(__name__)
def _connect(self):
self.client = SSHClient()
self.client.set_missing_host_key_policy(AutoAddPolicy())
self.client.load_system_host_keys()
connection_details = {}
if self.settings.port:
connection_details['port'] = self.settings.port
if self.settings.username:
connection_details['username'] = self.settings.username
if self.settings.password:
connection_details['password'] = self.settings.password
try:
self.client.connect(self.settings.hostname, **connection_details)
except NoValidConnectionsError as e:
raise PerformerError('Cant connect to %s' % self.settings.hostname)
else:
#ssh agent forwarding
s = self.client.get_transport().open_session()
AgentRequestHandler(s)
def _paramiko_exec_command(self, command, bufsize=-1, timeout=None):
# replacement paramiko.client.exec_command(command) for binary output
# https://github.com/paramiko/paramiko/issues/291
# inspired by workaround https://gist.github.com/smurn/4d45a51b3a571fa0d35d
chan = self.client._transport.open_session(timeout=timeout)
chan.settimeout(timeout)
chan.exec_command(command)
stdin = chan.makefile('wb', bufsize)
stdout = chan.makefile('rb', bufsize)
stderr = chan.makefile_stderr('rb', bufsize)
return stdin, stdout, stderr
def execute(self, command, logger=None, writein=None, max_lines=None):
self.logger.debug("Execute command: '%s'" % command)
if not self.client:
self._connect()
stdin, stdout, stderr = self._paramiko_exec_command(command)
# read stdout asynchronously - in 'realtime'
output_reader = OutputReader(stdout, logger=logger or self.output_logger, max_lines=max_lines)
if writein:
# write writein to stdin
stdin.write(writein)
stdin.flush()
stdin.channel.shutdown_write()
# wait for end of output
output = output_reader.output()
# wait for exit code
exit_code = stdout.channel.recv_exit_status()
if exit_code:
err = stderr.read().decode('utf-8').strip()
self.logger.debug('command error: %s' % err)
raise CommandError(command, exit_code, err)
return output
def send_file(self, source, target):
self.logger.debug("Send file: '%s' '%s'" % (source, target))
source = expanduser(source)
sftp = self.client.open_sftp()
sftp.put(source, target)
sftp.close()
@contextmanager
def get_fo(self, remote_path):
from tempfile import SpooledTemporaryFile
self.logger.debug('SSH Get fo: %s' % remote_path)
sftp = self.client.open_sftp()
try:
with SpooledTemporaryFile(1024000) as fo:
sftp.getfo(remote_path, fo)
yield fo
finally:
sftp.close()
def main():
user_config_file = path.join(path.expanduser('~'), '.fernglas.cfg')
project_config_file = path.join(getcwd(), '.fernglas.cfg')
config = SafeConfigParser()
config.read([user_config_file, project_config_file])
servers = {}
if not config.has_section('main'):
print("No main config found")
exit(1)
if not config.has_option('main', 'servers'):
print("No servers defined")
exit(2)
server_sections = config.get('main', 'servers').split(',')
server_sections = [section.strip() for section in server_sections]
for server in server_sections:
servers[server] = dict(config.items(server))
if 'port' in servers[server]:
servers[server]['port'] = int(servers[server]['port'])
issue = sys.argv[1]
package = run_setup(path.join(getcwd(), 'setup.py'))
package_name = package.get_name()
repo = Repo(getcwd())
latest_issue_commit = repo.git.log(all=True, grep=issue, n=1, format='%H')
if not latest_issue_commit:
print("No commits found for " + issue)
exit(3)
config = SSHConfig()
ssh_config_path = path.expanduser('~/.ssh/config')
if path.exists(ssh_config_path):
try:
config.parse(open(ssh_config_path))
except Exception as e:
print("Could not parse ssh config: " + str(e))
client = SSHClient()
client.load_system_host_keys()
# XXX support ecdsa?
client.set_missing_host_key_policy(AutoAddPolicy())
for key, server in servers.items():
host_config = config.lookup(server['hostname'])
connect_opts = {}
for key_ssh, key_paramiko in SSH_CONFIG_MAPPING.items():
if key_ssh in host_config:
connect_opts[key_paramiko] = host_config[key_ssh]
connect_opts.update(dict(
(opt, servers[key][opt]) for opt in SSH_OPTIONS
if opt in servers[key]))
client.connect(**connect_opts)
stdin, stdout, stderr = client.exec_command('grep {0} {1}'.format(
package_name, server['versions-path']))
deployed_version = stdout.read().strip().replace(package_name, '')\
.replace('=', '').strip()
version_tags = [tag for tag in repo.tags
if tag.name == deployed_version]
tag = version_tags[0]
version_commit = tag.commit.hexsha
status = repo.is_ancestor(latest_issue_commit, version_commit)
print("{0} is {2}deployed to {1}".format(
issue, key, (not status) and 'not ' or ''))
client.close()
def update_assignment_tests(class_name, local_assignment_dir):
local_assignment_dir = os.path.expanduser(local_assignment_dir)
local_assignment_dir = os.path.abspath(local_assignment_dir)
assignment = os.path.basename(local_assignment_dir.rstrip('/'))
test_code_dir = os.path.join(local_assignment_dir, 'tests')
if not os.path.isdir(test_code_dir):
sys.exit('{0} does not exist'.format(test_code_dir))
action_file_path = os.path.join(test_code_dir, 'action.sh')
if not os.path.isfile(action_file_path):
sys.exit('No action.sh in {0}'.format(test_code_dir))
try:
config = GraderConfiguration(single_class_name=class_name)
except ConfigurationError as e:
sys.exit(e)
try:
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.connect(config.host, username=config.username)
except OSError as e:
sys.exit('Error opening SSH connection:\n{0}'.format(e))
if class_name not in config.students_by_class:
sys.exit('Class {0} does not exist'.format(class_name))
test_code_repo_tempdir = TemporaryDirectory()
try:
test_code_repo = copy_and_create_repo(test_code_dir,
test_code_repo_tempdir.name,
assignment)
except CommandError as e:
error = 'Error copying test code repo:\n{0}'.format(e)
sys.exit(error)
try:
remote_home_dir = home_dir_from_username(config.username, ssh=ssh)
except CommandError as e:
sys.exit('Error getting remote home dir for {0}:\n{1}'
.format(config.username, e))
remote_assignment_dir = os.path.join(remote_home_dir, class_name,
'assignments', assignment)
tests_bare_repo_dir = os.path.join(remote_assignment_dir,
'{0}_tests.git'.format(assignment))
if not directory_exists(tests_bare_repo_dir, ssh=ssh):
sys.exit('{0} does not exist on {1}'.format(tests_bare_repo_dir,
config.host))
print('Updating test repo for assignment', assignment)
tests_bare_repo = Repository(tests_bare_repo_dir, assignment,
is_local=False, is_bare=True,
remote_host=config.host,
remote_user=config.username)
try:
test_code_repo.push(tests_bare_repo, force=True)
print('Pushed tests to', tests_bare_repo_dir)
except CommandError as e:
sys.exit('Error pushing test repo:\n{0}'.format(e))
print(assignment, 'tests updated successfully')
class SSH(AbstractFs):
policy = AutoAddPolicy
recv_size = 1024
_ssh = None
_basepath = "/"
def __init__(self, host, username="******", basepath=None):
logger.debug("Creating SSH client")
self._ssh = SSHClient()
self._ssh.load_system_host_keys()
logger.debug("Setting %s for missing host keys", self.policy.__name__)
self._ssh.set_missing_host_key_policy(self.policy)
logger.info(
"Connecting to %s%s%s",
"{}@".format(username) or "",
host,
":{}".format(basepath) or "",
)
self._ssh.connect(host, username=username, compress="true")
if basepath:
logger.info("Use basepath %s", basepath)
self._basepath = basepath
def _collect_stream(self, channel, recv_fn):
content = ""
buffer = None
while True:
buffer = recv_fn(self.recv_size)
content += buffer.decode("utf-8")
if len(buffer) == 0:
break
return content
def _exec(self, cmd, log_error=True):
channel = self._ssh.get_transport().open_session()
channel.exec_command(cmd)
stdout = self._collect_stream(channel, channel.recv)
stderr = self._collect_stream(channel, channel.recv_stderr)
exit_code = channel.recv_exit_status()
logger.debug(stdout)
if exit_code != 0 and log_error:
logger.error(stderr)
logger.error("Command %s failed with exit code %s", cmd, exit_code)
return exit_code, stdout, stderr
def touch(self, filename):
logger.debug("Touching file %s", filename)
self._exec('touch "{}"'.format(filename))
def rename(self, remote_src, remote_dest):
logger.debug("Renaming %s into %s", remote_src, remote_dest)
exit_code, _, _ = self._exec('mv "{}" "{}"'.format(remote_src, remote_dest))
if exit_code != 0:
raise SSHError("Failed renaming {} into {}".format(remote_src, remote_dest))
def exists(self, path):
logger.debug("Checking if %s exists...", path)
exit_code, _, _ = self._exec('test -f "{}"'.format(path), log_error=False)
exists = exit_code == 0
logger.debug("File %s does%s exists.", path, "" if exists else " not")
return exists
def copy(self, remote_src, local_dst):
exit_code, remote_content, _ = self._exec('cat "{}"'.format(remote_src))
if exit_code != 0:
raise SSHError("Cannot read from %s", remote_src)
with open(local_dst, "w") as f:
f.write(remote_content)
# def remove(self, path):
# return os.remove(path)
def symlink(self, remote_src, remote_dest, relative_to=None):
if relative_to:
remote_src = remote_src.replace(relative_to, ".")
logger.debug("Symlinking %s to %s", remote_src, remote_dest)
exit_code, _, _ = self._exec('ln -sf "{}" "{}"'.format(remote_src, remote_dest))
if exit_code != 0:
raise SSHError("Failed symlinking {} to {}".format(remote_src, remote_dest))
def glob(self, remote_path, regex_str):
logger.debug("Globbing %s for %s", remote_path, regex_str)
exit_code, remote_content, _ = self._exec('ls "{}"'.format(remote_path))
if exit_code != 0:
raise SSHError("Failed listing {}".format(remote_path))
regex = re.compile(regex_str)
entries = (e for e in remote_content.split() if regex.match(e))
entries = (os.path.join(remote_path, e) for e in entries)
return entries
def rmtree(self, remote_path):
logger.debug("Removing tree %s", remote_path)
exit_code, _, _ = self._exec('rm -rf "{}"'.format(remote_path))
if exit_code != 0:
raise SSHError("Failed removing {}".format(remote_path))
def close(self):
logger.debug("Closing SSH connection...")
self._ssh.close()
logger.debug("Connection closed.")
