def restore(path):
dbinfo = choicedb(path+'/conf/db_info.yaml')
editData(dbinfo, [['reset master;']])
if sys.platform == 'win32':
t = Transport(dbinfo[0], 22)
t.connect(username=dbinfo[5], password=dbinfo[6])
sftp = SFTPClient.from_transport(t)
src = '/home/mysql/basedata/initdata.sql'
des = path + '/database/initdata.sql'
sftp.put(des,src )
t.close()
else:
child = spawn(
'scp '+path+'/database/initdata.sql '+ dbinfo[5]+'@'+dbinfo[0]+':/home/mysql/basedata/')
child.expect("password:"******"'"+dbinfo[3]+"'"' '+dbinfo[4]+' </home/mysql/basedata/initdata.sql')
return stderr.read().decode('gbk', errors='ignore')
def __enter__(self):
assert self.sftp_clients is None, 'the context is already started'
sftp_clients = {}
for server in self.servers:
try:
transport = Transport((server.host, server.port))
transport.connect(username=server.user, pkey=self.old_key)
except (OSError, SSHException) as e:
for t, _, __ in sftp_clients.values():
t.close()
l = logging.getLogger(__name__ + '.TwoPhaseRenewal.__enter__')
l.exception(
'An exception rise during master key renewal '
'(%s -> %s, server: %s@%s:%d): %s',
get_key_fingerprint(self.old_key),
get_key_fingerprint(self.new_key),
server.user, server.host, server.port, str(e)
)
raise
except socket.gaierror as e:
raise ConnectionError(
'failed to connect: {0!s}\n{1!s}'.format(server, e)
) from e
sftp_client = SFTPClient.from_transport(transport)
authorized_keys = AuthorizedKeyList(sftp_client)
sftp_clients[server] = transport, sftp_client, authorized_keys
authorized_keys.append(self.new_key)
self.sftp_clients = sftp_clients
return self.servers
def dbbackup(path):
dbinfo = choicedb(path + '/conf/db_info.yaml')
s = sshconnect(hostname=dbinfo[0], port=dbinfo[7], username=dbinfo[5], password=dbinfo[6])
st, std, stde = s.exec_command('cd /home/mysql/backup'+'\n' 'ls')
filelist = std.read().decode('gbk',errors='ignore').split('\n')
if 'databackup.sql' in filelist:
s.exec_command('cd /home/mysql/backup'+'\n' 'rm -rf databackup1.sql')
stdin, stdout, stderr = s.exec_command('mysqldump -t -u'+dbinfo[2]+' -p'+dbinfo[3]+' '+dbinfo[4]+' >/home/mysql/backup/databackup.sql')
if 'error' not in stderr.read().decode('gbk', errors='ignore'):
t1,t2,t3=s.exec_command('du -b /home/mysql/databackup.sql')
if sys.platform == 'win32':
t = Transport(dbinfo[0], 22)
t.connect(username=dbinfo[5], password=dbinfo[6])
sftp = SFTPClient.from_transport(t)
src = '/home/mysql/backup/databackup.sql'
des = path + '/database/databackup.sql'
sftp.get(src, des)
t.close()
else:
child = spawn(
'scp ' + dbinfo[5] + '@' + dbinfo[0] + ':/home/mysql/backup/databackup.sql ' + path + '/database/')
child.expect("password:")
child.sendline(dbinfo[6])
child.read()
with zipfile.ZipFile(path + '/database/backupdatabase.zip', 'w') as z:
z.write(path + '/database/databackup.sql')
return t2.read().decode('gbk',errors='ignore').split('\t')[0].strip()
else:
return False
s.close
def get_sftp_client():
global _sftp_client, _identity_file, _repository_server, _repository_port, _repository_user
try:
if not _sftp_client:
key = RSAKey.from_private_key_file(_identity_file)
t = Transport((_repository_server, _repository_port))
t.connect(username=_repository_user, pkey=key)
_sftp_client = DistSFTPClient.from_transport(t)
except Exception, e:
print e
def fx_authorized_sftp(fx_sftpd, fx_authorized_keys):
port, (thread, path, ev) = fx_sftpd.popitem()
thread.start()
key = RSAKey.generate(1024)
dot_ssh = path.mkdir('.ssh')
with dot_ssh.join('authorized_keys').open('w') as f:
print(format_openssh_pubkey(key), file=f)
for authorized_key in fx_authorized_keys:
print(format_openssh_pubkey(authorized_key), file=f)
transport = Transport(('127.0.0.1', port))
transport.connect(username='******', pkey=key)
sftp_client = SFTPClient.from_transport(transport)
yield sftp_client, path, [key] + fx_authorized_keys
sftp_client.close()
transport.close()
def __enter__(self):
assert self.sftp_clients is None, "the context is already started"
sftp_clients = {}
for server in self.servers:
try:
transport = Transport((server.host, server.port))
transport.connect(pkey=self.old_key)
except SSHException:
for t, _, __ in sftp_clients.values():
t.close()
raise
sftp_client = SFTPClient.from_transport(transport)
authorized_keys = AuthorizedKeyList(sftp_client)
sftp_clients[server] = transport, sftp_client, authorized_keys
authorized_keys.append(self.new_key)
self.sftp_clients = sftp_clients
return self.servers
def __enter__(self):
assert self.sftp_clients is None, 'the context is already started'
sftp_clients = {}
for server in self.servers:
try:
transport = Transport((server.host, server.port))
transport.connect(pkey=self.old_key)
except SSHException:
for t, _, __ in sftp_clients.values():
t.close()
raise
sftp_client = SFTPClient.from_transport(transport)
authorized_keys = AuthorizedKeyList(sftp_client)
sftp_clients[server] = transport, sftp_client, authorized_keys
authorized_keys.append(self.new_key)
self.sftp_clients = sftp_clients
return self.servers
def authorize(public_keys: collections.abc.Set,
master_key: PKey,
remote: Remote,
timeout: datetime.timedelta) -> datetime.datetime:
"""Make an one-time authorization to the ``remote``, and then revokes
it when ``timeout`` reaches soon.
:param public_keys: the set of public keys (:class:`paramiko.pkey.PKey`)
to authorize
:type public_keys: :class:`collections.abc.Set`
:param master_key: the master key (*not owner's key*)
:type master_key: :class:`paramiko.pkey.PKey`
:param remote: a remote to grant access permission
:type remote: :class:`~.remote.Remote`
:param timeout: the time an authorization keeps alive
:type timeout: :class:`datetime.timedelta`
:return: the expiration time
:rtype: :class:`datetime.datetime`
"""
transport = Transport((remote.host, remote.port))
transport.connect(username=remote.user, pkey=master_key)
try:
sftp_client = SFTPClient.from_transport(transport)
try:
authorized_keys = AuthorizedKeyList(sftp_client)
authorized_keys.extend(public_keys)
except:
sftp_client.close()
raise
except:
transport.close()
raise
def rollback():
time.sleep(timeout.total_seconds())
authorized_keys[:] = [master_key]
sftp_client.close()
transport.close()
timer = threading.Thread(target=rollback)
expires_at = datetime.datetime.now(datetime.timezone.utc) + timeout
timer.start()
return expires_at
def authenticate_sftp_user(user_name, password, otc, hostname, port):
def sftp_auth_handler(title, instructions, prompt_list):
if len(prompt_list) == 0:
return []
if 'Password' in prompt_list[0][0]:
return [password]
else:
return [otc]
transport = Transport((hostname, int(port)))
if otc != '':
transport.start_client()
transport.auth_interactive(user_name, sftp_auth_handler)
else:
transport.connect(None, user_name, password)
return transport
def sftp_push_file(host, user, password, to_, from_):
"""
上传文件，注意：不支持文件夹
:param host: 主机名
:param user: 用户名
:param password: 密码
:param from_: 远程路径，比如：/home/sdn/tmp.txt
:param to_: 本地路径，比如：D:/text.txt
:param timeout: 超时时间(默认)，必须是int类型
:return: bool
"""
logging.debug('SSH FILE PUSH')
try:
t = Transport((host, 22))
t.connect(username=user, password=password)
sftp = paramiko.SFTPClient.from_transport(t)
sftp.put(to_, from_)
t.close()
logging.info('SSH FILE PUSH SUCCESS')
return True
except Exception as e:
logging.error(str(type(e)) + ' ' + e.strerror)
logging.error('SSH FILE PUSH FAIL')
return False
class HetznerStorage:
def __init__(self):
self.__host = os.environ.get('HETZNER_STORAGEBOX')
self.__port = int(os.environ.get('HETZNER_PORT', '22'))
self.__username = os.environ.get('HETZNER_USER')
self.__password = os.environ.get('HETZNER_PASSWORD')
self.__path = os.environ.get('HETNZER_PATH', '/backups')
self.__hostKey = os.environ.get('HETZNER_HOSTKEY')
self.__transport = None
defaultSSHKeyFilePath = os.environ.get('HOME')+"/.ssh/id_rsa"
self.__keyFilePath = os.environ.get('HETZNER_SSHKEY', defaultSSHKeyFilePath)
self.__keyFileChoose = os.environ.get('HETZNER_SSHKEY_CHOOSE', 'first')
self._usePassword = False
self.uploadChunkSize = int(os.environ.get('UPLOAD_CHUNK_SIZE', '2097152'))
self.downloadChunkSize = int(os.environ.get('DOWLOAD_CHUNK_SIZE', '2097152'))
def load(self):
self.__connect()
pass
def __connect(self):
hostKey = self.__getHostKey()
if self.__transport:
self.__transport.close()
self.__transport = Transport( (self.__host, self.__port) )
if self._usePassword:
password = self.__password
if not password:
password = self.__promptPassword()
self.__transport.connect(hostKey, self.__username, password)
else:
privateKey = self.__loadKey()
self.__transport.connect(hostKey, self.__username, pkey=privateKey)
self.__sftp = SFTPClient.from_transport(self.__transport)
return self
def __promptPassword(self):
return getpass.getpass('Storagebox password:'******'any':
return None
hostKeyEntry = HostKeyEntry.from_line(self.__hostKey)
return hostKeyEntry.key
def createResort(self, resortName):
self.__sftp.mkdir( self.__path+'/'+resortName, 0o755)
def removeResort(self, resortName):
self.__rmdir(self.__path+'/'+resortName, True)
def __rmdir(self, directory, verbose=False):
print(directory)
for entry in self.__sftp.listdir_attr( directory ):
if stat.S_ISDIR(entry.st_mode):
subdir = directory+'/'+entry.filename
if verbose:
print("Recursing into "+subdir)
self.__rmdir(subdir)
else:
filePath = directory+'/'+entry.filename
if verbose:
print("Removing into "+filePath)
self.__sftp.remove(filePath)
if verbose:
print("Removing "+directory)
self.__sftp.rmdir(directory)
def findResort(self, resortName):
directories = self.__sftp.listdir( self.__path )
if resortName not in directories:
raise NoSuchResortError(resortName)
return self.buildResort(resortName)
def getResorts(self):
directories = self.__sftp.listdir( self.__path )
resorts = []
for directory in directories:
resorts.append( self.buildResort(directory) )
return resorts
def buildResort(self, resortName):
resort = Resort().name(resortName).storage(self)
return self.rebuildResort(resort)
def rebuildResort(self, resort):
resortPath = resort.appendName(self.__path+'/')
resortDirectories = self.__sftp.listdir( resortPath )
if 'mysql' in resortDirectories:
resort.withMySQL(
MySQL()
.resort(resort)
)
if 'postgres' in resortDirectories:
resort.withPostgres()
if 'files' in resortDirectories:
path = '/'.join([
resortPath,
'files'
])
resort.withBorg(
Borg()
.host(self.__host)
.port(23)
.path(path)
.user(self.__username)
.keyFilePath(self.__keyFilePath)
)
return resort
def resort(self, resortName):
self._currentResort = resortName
return self
def adapter(self, adapterName):
self.currentAdapter = adapterName
return self
def createAdapter(self, adapterName):
self.__sftp.mkdir( self.__path+'/'+self._currentResort+'/'+adapterName, 0o755)
return self
def usePassword(self):
self._usePassword = True
return self
def createFolder(self, folderName):
path = '/'.join([
self.__path,
self._currentResort,
self.currentAdapter,
folderName
])
self.__sftp.mkdir( path, 0o755)
def listFolder(self, path):
pathParts = [
self.__path,
self._currentResort,
self.currentAdapter,
]
if path is not None:
pathParts.append(path)
path = '/'.join(pathParts)
return self.__sftp.listdir( path )
def fileContent(self, path):
pathParts = [
self.__path,
self._currentResort,
self.currentAdapter,
path,
]
path = '/'.join(pathParts)
with self.__sftp.file( path ) as file:
return file.read()
def remove(self, remotePath, verbose=False):
fullRemotePath = '/'.join([
self.__path,
self._currentResort,
self.currentAdapter,
remotePath,
])
self.__rmdir(fullRemotePath, verbose)
def upload(self, localPath, remotePath, verbose=False):
fullRemotePath = '/'.join([
self.__path,
self._currentResort,
self.currentAdapter,
remotePath,
])
if os.path.isdir(localPath):
self.__uploadDirectory(localPath, fullRemotePath, verbose)
else:
self.__uploadFile(localPath, fullRemotePath, verbose)
def __uploadDirectory(self, localPath, remotePath, verbose=False):
mustMake = True
try:
remotePathStatus = self.__sftp.stat(remotePath)
if not stat.S_ISDIR(remotePathStatus.st_mode):
mustMake = False
except FileNotFoundError:
pass
if mustMake:
self.__sftp.mkdir(remotePath)
for localFileName in os.listdir( localPath ):
localFilePath = localPath + '/' + localFileName
remoteFilePath = remotePath + '/' + localFileName
if os.path.isdir(localFilePath):
if verbose:
print("Recursing into "+localFilePath+" as "+remoteFilePath)
self.__uploadDirectory(localFilePath, remoteFilePath, verbose)
else:
self.__uploadFile(localFilePath, remoteFilePath, verbose)
pass
def __uploadFile(self, localPath, remotePath, verbose=False):
if verbose:
print("Uploading "+localPath+" as "+remotePath)
with open(localPath, 'rb') as localFile:
with self.__sftp.file(remotePath, 'w') as remoteFile:
while True:
data = localFile.read(self.uploadChunkSize)
if not data:
break
remoteFile.write(data)
def download(self, remotePath, localPath, verbose=False):
fullRemotePath = '/'.join([
self.__path,
self._currentResort,
self.currentAdapter,
remotePath,
])
pathStat = self.__sftp.stat(fullRemotePath)
if stat.S_ISDIR(pathStat.st_mode):
self.__downloadDirectory(fullRemotePath, localPath, verbose)
else:
self.__downloadFile(fullRemotePath, localPath, verbose)
def __downloadDirectory(self, remotePath, localPath, verbose=False):
mustMake = True
try:
if os.path.isdir(localPath):
mustMake = False
except FileNotFoundError:
pass
if mustMake:
os.mkdir(localPath)
for remoteFileAttribute in self.__sftp.listdir_attr( remotePath ):
fileName = remoteFileAttribute.filename
localFilePath = localPath + '/' + fileName
remoteFilePath = remotePath + '/' + fileName
if stat.S_ISDIR(remoteFileAttribute.st_mode):
if verbose:
print("Recursing into "+localFilePath+" as "+remoteFilePath)
self.__downloadDirectory(remoteFilePath, localFilePath, verbose)
else:
self.__downloadFile(remoteFilePath, localFilePath, verbose)
pass
def __downloadFile(self, remotePath, localPath, verbose=False):
if verbose:
print("Downloading "+remotePath+" as "+localPath)
with self.__sftp.file(remotePath, 'rb') as remoteFile:
with open(localPath, 'wb') as localFile:
while True:
data = remoteFile.read(self.downloadChunkSize)
if not data:
break
localFile.write(data)
def generateId(self):
return RSAKey.generate(4096)
def copyId(self, privateKey = None):
if '.ssh' not in self.__sftp.listdir('/'):
self.__sftp.mkdir('/.ssh', 0o700)
if not privateKey:
privateKey = self.__loadKey()
with self.__sftp.open('/.ssh/authorized_keys', 'r') as f:
authorizedKeys = f.read()
fileContent = authorizedKeys.decode('utf-8')
if fileContent[ -1 ] != '\n':
print("File does not end in newline - adding")
with self.__sftp.open('/.ssh/authorized_keys', 'a+') as f:
f.write('\n')
rfcPublicKey = self.__publicRFC4716(privateKey)
publicKey = self.__publicKey(privateKey)
if rfcPublicKey not in fileContent:
print("public key in rfc format not in authorized keys - inserting")
with self.__sftp.open('/.ssh/authorized_keys', 'a+') as f:
f.write(rfcPublicKey+'\n')
else:
print('public key in rfc format already present')
if publicKey not in fileContent:
print("public key not in authorized keys - inserting")
with self.__sftp.open('/.ssh/authorized_keys', 'a+') as f:
f.write(publicKey+'\n')
else:
print('public key already present')
def __loadKey(self):
if self.__keyFilePath == 'agent':
return self.__loadFromAgent()
try:
return RSAKey.from_private_key_file(self.__keyFilePath)
except FileNotFoundError:
print("Failed to load "+self.__keyFilePath+" trying the ssh-agent.")
return self.__loadFromAgent()
def __loadFromAgent(self):
agent = Agent()
keys = agent.get_keys()
if len(keys) == 0:
raise KeyError("No keys found in the ssh agent. Did you add them with ssh-add?")
return self.__pick_key(keys)
def __pick_key(self,keys):
if len(keys) == 0:
raise KeyError("No ssh keys received from ssh agent")
if self.__keyFileChoose == 'first':
return keys[0]
for key in keys:
if key.get_base64() == self.__keyFileChoose:
return key
raise KeyError("No ssh key matching the given public key received from the ssh-agent")
def __publicRFC4716(self, privateKey):
# NOte: rfc4716 says 72 bytes at most but ssh-keygen into rfc4716
# creates 70 character lines
publicKeyPart = '\n'.join(list(self.chunkstring(privateKey.get_base64(), 70)))
key = '\n'.join([
'---- BEGIN SSH2 PUBLIC KEY ----',
publicKeyPart,
'---- END SSH2 PUBLIC KEY ----'
])
return key
def chunkstring(self, string, length):
return (string[0+i:length+i] for i in range(0, len(string), length))
def __publicKey(self, privateKey):
key = privateKey.get_name()
key += ' '
key += privateKey.get_base64()
return key
class Communicator(drm4g.communicators.Communicator):
"""
Create a SSH session to remote resources.
"""
_lock = __import__('threading').Lock()
_sem = __import__('threading').Semaphore(SFTP_CONNECTIONS)
_trans = None
def connect(self):
try:
with self._lock:
if not self._trans or not self._trans.is_authenticated():
logger.debug("Opening ssh connection ... ")
keys = None
logger.debug("Trying ssh-agent ... ")
drm4g_agent = drm4g.commands.Agent()
drm4g_agent.start()
drm4g_agent.update_agent_env()
# paramiko agent
agent = Agent()
keys = agent.get_keys()
if not keys:
logger.debug("Error trying to connect to '%s'" %
self.frontend)
logger.debug(
"Impossible to load '%s' key from the ssh-agent" %
self.private_key)
try:
status_ssh_agent = agent._conn
except Exception as err:
logger.warning(
"Probably you are using paramiko version <= 1.7.7.2 : %s "
% err)
status_ssh_agent = agent.conn
if not status_ssh_agent:
logger.warning("'ssh-agent' is not running")
else:
if agent.get_keys():
logger.warning(
"ssh-agent is running but none of the keys have been accepted"
"by remote frontend %s." % self.frontend)
else:
logger.debug(
"'ssh-agent' is running but without any keys"
)
if self.private_key:
logger.debug("Trying '%s' key ... " % self.private_key)
private_key_path = expanduser(self.private_key)
if (not exists(private_key_path)) and (
not 'PRIVATE KEY' in self.private_key):
output = "'%s'key does not exist" % private_key_path
raise ComException(output)
for pkey_class in (RSAKey, DSSKey):
try:
if 'PRIVATE KEY' in self.private_key:
import StringIO
key = pkey_class.from_private_key(
StringIO.StringIO(
self.private_key.strip("'")))
else:
key = pkey_class.from_private_key_file(
private_key_path)
keys = keys + (key, )
except Exception:
pass
if not keys:
output = "Impossible to load any keys"
logger.error(output)
raise ComException(output)
for key in keys:
try:
sock = socket.socket()
try:
sock.settimeout(SSH_CONNECT_TIMEOUT)
except:
output = "Timeout trying to connect to '%s'" % self.frontend
raise ComException(output)
logger.debug(
"Connecting to '%s' as user '%s' port '%s' ..."
% (self.frontend, self.username, self.port))
if ':' in self.frontend:
self.frontend, self.port = self.frontend.split(
':')
sock.connect((self.frontend, self.port))
self._trans = Transport(sock)
self._trans.connect(username=self.username,
pkey=key)
if self._trans.is_authenticated():
break
except socket.gaierror:
output = "Could not resolve hostname '%s' " % self.frontend
raise ComException(output)
except Exception as err:
logger.warning("Error connecting '%s': %s" %
(self.frontend, str(err)))
if not self._trans:
output = "Authentication failed for '%s'. Try to execute `ssh -vvv -p %d %s@%s` and see the response." % (
self.frontend, self.port, self.username, self.frontend)
raise ComException(output)
except ComException:
raise
except Exception as err:
if "No handlers could be found for logger" in str(err):
raise Exception(
"The connect function is the one causing problems : %s" %
str(err))
else:
raise
def execCommand(self, command, input=None):
self.connect()
with self._lock:
channel = self._trans.open_session()
channel.settimeout(SSH_CONNECT_TIMEOUT)
channel.exec_command(command)
if input:
for line in input.split():
channel.makefile('wb', -1).write('%s\n' % line)
channel.makefile('wb', -1).flush()
stdout = ''.join(channel.makefile('rb', -1).readlines())
stderr = ''.join(channel.makefile_stderr('rb', -1).readlines())
if channel:
channel.close()
return stdout, stderr
def mkDirectory(self, url):
to_dir = self._set_dir(urlparse(url).path)
stdout, stderr = self.execCommand("mkdir -p %s" % to_dir)
if stderr:
raise ComException("Could not create %s directory on '%s': %s" %
(to_dir, self.frontend, stderr))
def rmDirectory(self, url):
to_dir = self._set_dir(urlparse(url).path)
stdout, stderr = self.execCommand("rm -rf %s" % to_dir)
if stderr:
raise ComException("Could not remove %s directory on '%s': %s" %
(to_dir, self.frontend, stderr))
def copy(self, source_url, destination_url, execution_mode=''):
with self._sem:
self.connect()
scp = SCPClient(self._trans)
if 'file://' in source_url:
from_dir = urlparse(source_url).path
to_dir = self._set_dir(urlparse(destination_url).path)
scp.put(from_dir, to_dir)
if execution_mode == 'X':
stdout, stderr = self.execCommand("chmod +x %s" % to_dir)
else:
from_dir = self._set_dir(urlparse(source_url).path)
to_dir = urlparse(destination_url).path
logger.warning("%s , %s" % (from_dir, to_dir))
scp.get(from_dir, to_dir)
def close(self):
try:
if self._trans:
self._trans.close()
except Exception as err:
logger.warning("Could not close the SSH connection to '%s': %s" %
(self.frontend, str(err)))
def __del__(self):
"""
Attempt to clean up if not explicitly closed.
"""
self.close()
#internal
def _set_dir(self, path):
work_directory = re.compile(r'^~').sub(self.work_directory, path)
if work_directory[0] == r'~':
return ".%s" % work_directory[1:]
else:
return work_directory
## Let's make an ssh tunnel if we find it in profiles.config
from mr_utils.config import ProfileConfig
from paramiko.transport import Transport
import socket
if __name__ == '__main__':
profile = ProfileConfig()
ssh_host = profile.get_config_val('ssh.host')
ssh_user = profile.get_config_val('ssh.user')
host = profile.get_config_val('gadgetron.host')
port = profile.get_config_val('gadgetron.port')
print(ssh_host, ssh_user)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((ssh_host, 22))
transport = Transport(sock)
transport.connect(username=ssh_user, password=None)
chan = transport.open_channel('session', dest_addr=(host, port))
print(chan)
chan.close()
transport.close()
sock.close()
