def scp_upload(src_blob='Preproc.tar.gz',
dst_blob="~",
options={
'hostname': 'lecun',
'username': '******'
},
progress=simple_callback):
# from https://gist.github.com/acdha/6064215
#--- Make the Paramiko SSH thing use my .ssh/config file (b/c I like ProxyCommand!)
client = SSHClient()
client.load_system_host_keys()
client._policy = WarningPolicy()
client.set_missing_host_key_policy(
WarningPolicy()) # hmm. WarningPolicy? Most people use AutoAddPolicy.
ssh_config = SSHConfig()
user_config_file = os.path.expanduser("~/.ssh/config")
if os.path.exists(user_config_file):
with open(user_config_file) as f:
ssh_config.parse(f)
cfg = {'hostname': options['hostname'], 'username': options["username"]}
user_config = ssh_config.lookup(cfg['hostname'])
for k in ('hostname', 'username', 'port'):
if k in user_config:
cfg[k] = user_config[k]
if 'proxycommand' in user_config:
cfg['sock'] = ProxyCommand(user_config['proxycommand'])
client.connect(**cfg)
socket_timeout = None # number of seconds for timeout. None = never timeout. TODO: None means program may hang. But timeouts are annoying!
# SCPCLient takes a paramiko transport and progress callback as its arguments.
scp = SCPClient(client.get_transport(),
progress=progress,
socket_timeout=socket_timeout)
# NOW we can finally upload! (in a separate process)
#scp.put(src_blob, dst_blob) # now in scp_thread
# we want this to be non-blocking so we stick it in a thread
thread = threading.Thread(target=scp_thread,
args=(scp, src_blob, dst_blob))
thread.start()
def modifyPHGBA(masterNode, accessNode):
connected = False
attemptCount = 0
while not connected:
try:
attemptCount += 1
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(WarningPolicy())
ssh.connect(masterNode["externalIP"],
22,
"gpadmin",
str(os.environ.get("GPADMIN_PW")),
timeout=120)
(stdin, stdout, stderr) = ssh.exec_command(
"echo 'host all gpadmin " + accessNode['internalIP'] +
"/0 md5' >> /data/master/gpseg-1/pg_hba.conf")
stdout.readlines()
stderr.readlines()
(stdin, stdout, stderr) = ssh.exec_command("gpstop -a -r")
stdout.readlines()
stderr.readlines()
connected = True
except Exception as e:
print masterNode["nodeName"] + ": Attempting SSH Connection"
time.sleep(3)
if attemptCount > 1:
print "Failing Process"
exit()
finally:
ssh.close()
def create(clusterInfo):
print "Creating "+ str(NUM_USERS) + " Users on all Cluster Nodes"
warnings.simplefilter("ignore")
for node in clusterInfo["clusterNodes"]:
print node["nodeName"]+": Creating Users"
connected = False
attemptCount = 0
while not connected:
try:
attemptCount += 1
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(WarningPolicy())
ssh.connect(node["externalIP"], 22, SSH_USERNAME, None, pkey=None, key_filename=SSH_KEY_PATH, timeout=120)
for userNum in range(1,NUM_USERS+1):
userName = BASE_USERNAME + str(userNum).zfill(2)
homeDir = BASE_HOME + "/home"
pw = BASE_PASSWORD + str(userNum).zfill(2)
(stdin, stdout, stderr) = ssh.exec_command("sudo mkdir -p " + homeDir+";sudo useradd -b "+homeDir+" -s "+ "/bin/bash -m "+userName)
(stdin, stdout, stderr) = ssh.exec_command("sudo sh -c 'echo "+ pw + " | passwd --stdin " + userName+"'")
stderr.readlines()
stdout.readlines()
connected = True
except Exception as e:
# print e
print node["nodeName"] + ": Attempting SSH Connection"
time.sleep(3)
if attemptCount > 40:
print "Failing Process"
exit()
finally:
ssh.close()
def setGPADMINPW(masterNode):
connected = False
attemptCount = 0
while not connected:
try:
attemptCount += 1
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(WarningPolicy())
ssh.connect(masterNode["externalIP"],
22,
"gpadmin",
str(os.environ.get("GPADMIN_PW")),
timeout=120)
(stdin, stdout, stderr) = ssh.exec_command(
"psql -c \"alter user gpadmin with password '" +
str(os.environ.get("GPADMIN_PW")) + "';\"")
# (stdin, stdout, stderr) = ssh.exec_command("alter user gpadmin with password '"+str(os.environ.get("GPADMIN_PW"))+ "';")
stdout.readlines()
stderr.readlines()
connected = True
except Exception as e:
print masterNode["nodeName"] + ": Attempting SSH Connection"
time.sleep(3)
if attemptCount > 1:
print "Failing Process"
exit()
finally:
ssh.close()
def gpControl(clusterInfo,action):
warnings.simplefilter("ignore")
print "GPDB Controller Initiated: "+action
for node in clusterInfo["clusterNodes"]:
connected = False
attemptCount = 0
if "master" in node["role"]:
while not connected:
try:
attemptCount += 1
gpssh = paramiko.SSHClient()
gpssh.set_missing_host_key_policy(WarningPolicy())
gpssh.connect(node["externalIP"], 22, "gpadmin", GPADMIN_PW, pkey=None, key_filename=None,
timeout=120)
(stdin, stdout, stderr) = gpssh.exec_command("gp"+action+ " -a")
stderr.readlines()
#pprint.pprint(stdout.readlines())
print stdout.read()
connected = True
except Exception as e:
print node["nodeName"] + ": Attempting SSH Connection"
time.sleep(3)
if attemptCount > 40:
print "Failing Process"
exit()
finally:
gpssh.close()
def setup(clusterInfo):
warnings.simplefilter("ignore")
print "Setup Data Loading for Student Accounts"
for node in clusterInfo["clusterNodes"]:
connected = False
attemptCount = 0
if "etl" in node["role"]:
while not connected:
try:
print "Pulling Data to ETL Node"
attemptCount += 1
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(WarningPolicy())
labsDir=BASE_HOME+"/labs"
ssh.connect(node["externalIP"], 22, SSH_USERNAME, None, pkey=None, key_filename=SSH_KEY_PATH,
timeout=120)
(stdin, stdout, stderr) = ssh.exec_command("sudo mkdir -p "+labsDir+";sudo curl "+LABS +" | sudo tar -C "+labsDir+" -xjvf -")
stderr.readlines()
# pprint.pprint(stdout.readlines())
print stdout.read()
connected = True
except Exception as e:
print node["nodeName"] + ": Attempting SSH Connection"
time.sleep(3)
if attemptCount > 40:
print "Failing Process"
exit()
finally:
ssh.close()
def setPaths(clusterNode):
connected = False
attemptCount = 0
while not connected:
try:
attemptCount += 1
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(WarningPolicy())
ssh.connect(clusterNode["externalIP"],
22,
"gpadmin",
str(os.environ.get("GPADMIN_PW")),
timeout=120)
(stdin, stdout, stderr) = ssh.exec_command(
"echo 'source /usr/local/greenplum-db/greenplum_path.sh\n' >> ~/.bashrc"
)
stdout.readlines()
stderr.readlines()
(stdin, stdout, stderr) = ssh.exec_command(
"echo 'export MASTER_DATA_DIRECTORY=/data/disk1/master/gpseg-1\n' >> ~/.bashrc"
)
stdout.readlines()
stderr.readlines()
connected = True
except Exception as e:
print clusterNode["nodeName"] + ": Attempting SSH Connection"
time.sleep(3)
if attemptCount > 1:
print "Failing Process"
exit()
finally:
ssh.close()
def installAnalytics(clusterDictionary):
warnings.simplefilter("ignore")
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(WarningPolicy())
for node in clusterDictionary["clusterNodes"]:
if ("master" in node["role"]):
print node["nodeName"] + ": Installing MADlib"
ssh.connect(node["externalIP"], 22, "gpadmin", password=GPADMIN_PW, timeout=120)
(stdin, stdout, stderr) =ssh.exec_command("gppkg -i /tmp/"+MADLIB)
stderr.readlines()
stdout.readlines()
ssh.exec_command("$GPHOME/madlib/bin/madpack install -s madlib -p greenplum -c gpadmin@"+node["nodeName"]+"/template1")
ssh.exec_command("$GPHOME/madlib/bin/madpack install -s madlib -p greenplum -c gpadmin@"+node["nodeName"]+"/gpadmin")
print " - Installing R on all the Nodes. This might take awhile."
(stdin, stdout, stderr) =ssh.exec_command("gpssh -f /tmp/allhosts 'sudo yum -y install R'")
stderr.readlines()
stdout.readlines()
ssh.exec_command("createlang plpythonu -d template1")
ssh.exec_command("createlang plpythonu -d gpadmin")
(stdin, stdout, stderr) = ssh.exec_command("gppkg -i /tmp/"+PLR)
stderr.readlines()
stdout.readlines()
ssh.exec_command("createlang plr -d template1")
ssh.exec_command("createlang plr -d gpadmin")
print " - MADLib, PL/Python, PL/R Enabled"
def ssh_cmd(ip, remote_cmd, user='******', password=None):
"""
@param server_ip
@param user
@param password
@param remote_cmd
@return A map based on pass / fail run info
"""
output = StringIO()
error = StringIO()
ssh = SSHClient()
ssh.set_missing_host_key_policy(WarningPolicy())
ssh.connect(ip, username=user, password=password, allow_agent=False)
stdin, stdout, stderr = ssh.exec_command(remote_cmd)
stdin.close()
for line in stdout:
if util.logger < 10:
sys.stdout.write(line)
util.logger.info(line.strip())
output.write(line)
for line in stderr:
util.logger.error(line.strip())
error.write(line)
exit_status = stdout.channel.recv_exit_status()
ret = {
'success': True if exit_status == 0 else False,
'return': output.getvalue(),
'exit_status': exit_status,
'error': error.getvalue()
}
return ret
def _exec(self):
exitcode = 0
client = SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(WarningPolicy())
try:
client.connect(self.config.get('hostname'),
int(self.config.get('port', 22)),
key_filename=self.identity_file,
username=self.config.get('user'))
stdout, stderr, channel = self.exec_command(client)
Printer(self.host, stdout, stderr).loop()
if channel:
exitcode = channel.recv_exit_status()
except IOError as e:
print(colored('{0}: {1}'.format(self.host, str(e)), 'red'))
exitcode = 1
except (BadHostKeyException, AuthException, SSHException) as e:
print(colored('{0}: {1}'.format(self.host, e.message)), 'red')
exitcode = 1
except Exception as e:
print(colored('{0}: {1}'.format(self.host, e.message)), 'red')
exitcode = 1
finally:
client.close()
return exitcode
def get_proxmox_ssh(proxmox):
proxmox_ssh = SSHClient()
proxmox_ssh.set_missing_host_key_policy(WarningPolicy())
proxmox_ssh.connect(proxmox['host'],
username=proxmox['user'].split('@')[0])
return proxmox_ssh
def install(self):
filename = SyncGatewayInstaller._generate_filename(
self.__version, self.__build)
if not Path(filename).exists():
raise Exception(
"Unable to find installer, please call download first")
print("Installing Sync Gateway to {}...".format(self.__url))
ssh_client = SSHClient()
ssh_client.load_system_host_keys()
ssh_client.set_missing_host_key_policy(WarningPolicy())
ssh_connect(ssh_client, self.__url, self.__ssh_keyfile,
str(self.__ssh_keypass))
(_, stdout, _) = ssh_client.exec_command("test -f {}".format(filename))
if stdout.channel.recv_exit_status() == 0:
print(
"Install file already present on remote host, skipping upload..."
)
else:
print("Uploading file to remote host...")
sftp = ssh_client.open_sftp()
sftp_upload(sftp, filename, filename)
sftp.close()
ssh_command(ssh_client, self.__url,
"sudo yum install -y {}".format(filename))
print("Install finished!")
def __init__(self, sourceURI, password):
# Split the source URI into components
components = urlsplit(sourceURI)
hostname = components.hostname
username = '******' if components.username is None else components.username
port = 22 if components.port is None else components.port
remoteDirectory = components.path
# Normalize path
remoteDirectory = normpath(remoteDirectory)
if not remoteDirectory.startswith('/'):
raise RuntimeError('Remote directory %s is not an absolute path' %
remoteDirectory)
self.remoteDirectory = remoteDirectory
# Connect using SSH
self.client = SSHClient()
self.client.load_system_host_keys()
self.client.set_missing_host_key_policy(WarningPolicy())
self.client.connect(hostname=hostname,
port=port,
username=username,
password=password,
compress=True)
# Open SFTP channel over SSH
self.sftp = self.client.open_sftp()
# Keep track of the last opened file
self.lofFP = None
self.lofPath = None
def uncompressFiles(clusterNode, downloads):
logging.info('uncompressFiles Started on: ' + str(clusterNode["nodeName"]))
connected = False
attemptCount = 0
while not connected:
try:
attemptCount += 1
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(WarningPolicy())
logging.debug('Connecting to Node: ' + clusterNode["nodeName"])
logging.debug('SSH IP: ' + clusterNode["externalIP"] + ' User: '******' Key: ' +
str(os.environ["CONFIGS_PATH"]) +
str(os.environ["SSH_KEY"]))
ssh.connect(clusterNode["externalIP"],
22,
str(os.environ["SSH_USERNAME"]),
None,
pkey=None,
key_filename=str(os.environ["CONFIGS_PATH"]) +
str(os.environ["SSH_KEY"]),
timeout=120)
logging.info('Unzipping files')
for file in downloads:
if ".zip" in file["NAME"]:
(stdin, stdout,
stderr) = ssh.exec_command("cd /tmp;unzip ./" +
file["NAME"])
logging.debug(stdout.readlines())
logging.debug(stderr.readlines())
elif ".gz" in file["NAME"]:
(stdin, stdout,
stderr) = ssh.exec_command("cd /tmp;tar xvfz ./" +
file["NAME"])
logging.debug(stdout.readlines())
logging.debug(stderr.readlines())
if os.environ["GPDB_BUILD"]:
logging.info('Unzipping pre-release build')
(stdin, stdout, stderr) = ssh.exec_command(
"cd /tmp;unzip ./" +
os.path.basename(str(os.environ["GPDB_BUILD"])))
logging.debug(stdout.readlines())
logging.debug(stderr.readlines())
connected = True
except Exception as e:
print clusterNode["nodeName"] + ": Attempting SSH Connection"
time.sleep(3)
if attemptCount > 1:
logging.debug('Exception: ' + str(e))
logging.debug(traceback.print_exc())
logging.debug('Failed')
print "Failing Process"
exit()
finally:
ssh.close()
logging.info('uncompressFiles Completed on: ' +
str(clusterNode["nodeName"]))
def _uninstall_worker(instance: AWSInstance, ssh_keyfile: str):
ssh_client = SSHClient()
ssh_client.load_system_host_keys()
ssh_client.set_missing_host_key_policy(WarningPolicy())
ssh_connect(ssh_client, instance.address, ssh_keyfile)
exit_code = ssh_command(ssh_client, instance.address, "sudo yum erase -y couchbase-sync-gateway.x86_64")
ssh_client.close()
return exit_code
def prepFiles(clusterNode):
logging.info('prepFiles Started on: ' + str(clusterNode["nodeName"]))
connected = False
attemptCount = 0
while not connected:
try:
attemptCount += 1
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(WarningPolicy())
logging.debug('Connecting to Node: ' + clusterNode["nodeName"])
logging.debug('SSH IP: ' + clusterNode["externalIP"] + ' User: '******' Key: ' +
str(os.environ["CONFIGS_PATH"]) +
str(os.environ["SSH_KEY"]))
ssh.connect(clusterNode["externalIP"],
22,
str(os.environ["SSH_USERNAME"]),
None,
pkey=None,
key_filename=str(os.environ["CONFIGS_PATH"]) +
str(os.environ["SSH_KEY"]),
timeout=120)
logging.info('Preparing GPDB Install Binary')
(stdin, stdout, stderr) = ssh.exec_command(
"sudo sed -i 's/more <</cat <</g' /tmp/greenplum-db*.bin")
logging.debug(stdout.readlines())
logging.debug(stderr.readlines())
(stdin, stdout, stderr) = ssh.exec_command(
"sudo sed -i 's/agreed=/agreed=1/' /tmp/greenplum-db*.bin")
logging.debug(stdout.readlines())
logging.debug(stderr.readlines())
(stdin, stdout, stderr) = ssh.exec_command(
"sudo sed -i 's/pathVerification=/pathVerification=1/' /tmp/greenplum-db*.bin"
)
logging.debug(stdout.readlines())
logging.debug(stderr.readlines())
(stdin, stdout, stderr) = ssh.exec_command(
"sudo sed -i 's/user_specified_installPath=/user_specified_installPath=${installPath}/' /tmp/greenplum-db*.bin"
)
logging.debug(stdout.readlines())
logging.debug(stderr.readlines())
connected = True
except Exception as e:
print clusterNode["nodeName"] + ": Attempting SSH Connection"
time.sleep(3)
if attemptCount > 1:
logging.debug('Exception: ' + str(e))
logging.debug(traceback.print_exc())
logging.debug('Failed')
print "Failing Process"
exit()
finally:
ssh.close()
logging.info('prepFiles Completed on: ' +
str(clusterNode["nodeName"]))
def connect(self):
client = SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(WarningPolicy())
client.connect(hostname=self.remote_host, username=self.remote_user)
LOG.debug(client)
self.client = client
self.sftp = client.open_sftp()
def hostFileUpload(clusterNode):
logging.debug('hostFileUpload Started')
warnings.simplefilter("ignore")
logging.debug('SimpleFilter for Warnings: ignore')
paramiko.util.log_to_file("/tmp/paramiko.log")
connected = False
attemptCount = 0
while not connected:
try:
attemptCount += 1
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(WarningPolicy())
logging.debug('Current Dir: ' + os.getcwd())
logging.debug('Connecting to Node: ' + clusterNode["nodeName"])
logging.debug('SSH IP: ' + clusterNode["externalIP"] + ' User: '******' Key: ' +
str(os.environ["CONFIGS_PATH"]) +
str(os.environ["SSH_KEY"]))
ssh.connect(clusterNode["externalIP"],
22,
os.environ["SSH_USERNAME"],
None,
pkey=None,
key_filename=(str(os.environ["CONFIGS_PATH"]) +
str(os.environ["SSH_KEY"])),
timeout=120)
sftp = ssh.open_sftp()
sftp.put("hosts", "/tmp/hosts", confirm=True)
logging.debug('Put hosts file')
sftp.put("allhosts", "/tmp/allhosts", confirm=True)
logging.debug('Put allhosts file')
sftp.put("workers", "/tmp/workers", confirm=True)
logging.debug('Put Workers File')
(stdin, stdout, stderr
) = ssh.exec_command("sudo sh -c 'cat /tmp/hosts >> /etc/hosts'")
logging.debug(stdout.readlines())
logging.debug(stderr.readlines())
connected = True
except Exception as e:
# print e
print " " + clusterNode[
"nodeName"] + ": Attempting SSH Connection"
time.sleep(3)
if attemptCount > 40:
logging.debug('Exception: ' + str(e))
logging.debug(traceback.print_exc())
logging.debug('Failed')
print "Failing Process"
exit()
finally:
ssh.close()
logging.debug('hostFileUpload Completed')
def installComponents(masterNode, downloads):
logging.info('installComponents Started on: ' +
str(masterNode["nodeName"]))
connected = False
attemptCount = 0
while not connected:
try:
attemptCount += 1
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(WarningPolicy())
logging.debug('Connecting to Node: ' + str(masterNode["nodeName"]))
logging.debug('SSH IP: ' + masterNode["externalIP"] +
' User: gpadmin')
ssh.connect(masterNode["externalIP"],
22,
"gpadmin",
str(os.environ["GPADMIN_PW"]),
timeout=120)
(stdin, stdout,
stderr) = ssh.exec_command("createlang plpythonu -d template1")
logging.debug(stdout.readlines())
logging.debug(stderr.readlines())
(stdin, stdout,
stderr) = ssh.exec_command("createlang plpythonu -d gpadmin")
logging.debug(stdout.readlines())
logging.debug(stderr.readlines())
(stdin, stdout,
stderr) = ssh.exec_command("gppkg -i /tmp/madlib*.gppkg")
logging.debug(stdout.readlines())
logging.debug(stderr.readlines())
ssh.exec_command(
"$GPHOME/madlib/bin/madpack install -s madlib -p greenplum -c gpadmin@"
+ masterNode["nodeName"] + "/template1")
logging.debug(stdout.readlines())
logging.debug(stderr.readlines())
ssh.exec_command(
"$GPHOME/madlib/bin/madpack install -s madlib -p greenplum -c gpadmin@"
+ masterNode["nodeName"] + "/gpadmin")
logging.debug(stdout.readlines())
logging.debug(stderr.readlines())
connected = True
except Exception as e:
print e
print masterNode["nodeName"] + ": Waiting on Database Connection"
time.sleep(3)
if attemptCount > 10:
logging.debug('Exception: ' + str(e))
logging.debug(traceback.print_exc())
logging.debug('Failed')
print "Failing Process: Please Verify Database Manually."
exit()
logging.info('installComponents Completed on: ' +
str(masterNode["nodeName"]))
def scp_to(ip, local_path, user='******', password=None, remote_path=""):
"""
Send a file to a server
"""
ssh = SSHClient()
ssh.set_missing_host_key_policy(WarningPolicy())
ssh.connect(ip, username=user, password=password, allow_agent=False)
sftp = ssh.open_sftp()
sftp.put(local_path, remote_path)
def __init__(self, hostname, username, password, test_password=None):
self._hostname = hostname
self._username = username
self._password = password
self._test_pasword = test_password
self.client = SSHClient()
self.client.set_missing_host_key_policy(WarningPolicy())
self._session = None
self._test_commands_enabled = False
super().__init__()
def deploy_sg_config(instance: AWSInstance, cb_node: AWSInstance,
ssh_keyfile: str, keypass: Credential):
template = {
"logging": {
"log_file_path": "/var/tmp/sglogs",
"console": {
"enabled": False
},
"debug": {
"enabled": True
}
},
"databases": {
"db": {
"server": "couchbase://{}".format(cb_node.internal_address),
"username": "******",
"password": "******",
"bucket": "device-farm-data",
"users": {
"GUEST": {
"disabled": False,
"admin_channels": ["*"]
}
},
"allow_conflicts": False,
"revs_limit": 20,
"enable_shared_bucket_access": True,
"import_docs": "continuous"
}
},
"interface": "0.0.0.0:4984",
"adminInterface": "{}:4985".format(instance.private_ip)
}
config_filename = "{}_config.json".format(instance.name)
with open(config_filename, "w") as fout:
json.dump(template, fout)
ssh_client = SSHClient()
ssh_client.load_system_host_keys()
ssh_client.set_missing_host_key_policy(WarningPolicy())
ssh_connect(ssh_client, instance.address, ssh_keyfile, str(keypass))
ssh_command(ssh_client, instance.name, "sudo systemctl stop sync_gateway")
sftp = ssh_client.open_sftp()
sftp_upload(sftp, config_filename, config_filename)
sftp.close()
command = """
sudo chown sync_gateway {0};
sudo mv {0} /home/sync_gateway/sync_gateway.json;
sudo systemctl start sync_gateway
""".format(config_filename)
ssh_command(ssh_client, instance.name, command)
ssh_client.close()
def setPaths(clusterNode):
logging.info('setPaths Started on: ' + str(clusterNode["nodeName"]))
connected = False
attemptCount = 0
while not connected:
try:
attemptCount += 1
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(WarningPolicy())
logging.debug('Connecting to Node: ' +
str(clusterNode["nodeName"]))
logging.debug('SSH IP: ' + clusterNode["externalIP"] +
' User: gpadmin')
ssh.connect(clusterNode["externalIP"],
22,
"gpadmin",
str(os.environ["GPADMIN_PW"]),
timeout=120)
logging.info('Setting up bashrc')
(stdin, stdout, stderr) = ssh.exec_command(
"echo 'source /usr/local/greenplum-db/greenplum_path.sh\n' >> ~/.bashrc"
)
logging.debug(stdout.readlines())
logging.debug(stderr.readlines())
if 'yes' in os.environ["RAID0"]:
logging.info(
'Set MASTER_DATA_DIRECTORY to /data1/master/gpseg-1')
(stdin, stdout, stderr) = ssh.exec_command(
"echo 'export MASTER_DATA_DIRECTORY=/data1/master/gpseg-1\n' >> ~/.bashrc"
)
logging.debug(stdout.readlines())
logging.debug(stderr.readlines())
else:
logging.info(
'Set MASTER_DATA_DIRECTORY to /data/disk1/master/gpseg-1')
(stdin, stdout, stderr) = ssh.exec_command(
"echo 'export MASTER_DATA_DIRECTORY=/data/disk1/master/gpseg-1\n' >> ~/.bashrc"
)
logging.debug(stdout.readlines())
logging.debug(stderr.readlines())
connected = True
except Exception as e:
print clusterNode["nodeName"] + ": Attempting SSH Connection"
time.sleep(3)
if attemptCount > 1:
logging.debug('Exception: ' + str(e))
logging.debug(traceback.print_exc())
logging.debug('Failed')
print "Failing Process"
exit()
finally:
ssh.close()
logging.info('setPaths Completed on: ' +
str(clusterNode["nodeName"]))
def _get_ssh_clients(self):
"""
Returns an SSHClient and SFTPClient configured for use.
"""
ssh_client = SSHClient()
ssh_client.set_missing_host_key_policy(WarningPolicy())
ssh_client.connect(self.archive.host,
username=self.archive.username,
pkey=self.archive.ssh_credentials.get_pkey())
sftp_client = SFTPClient.from_transport(ssh_client.get_transport())
return ssh_client, sftp_client
def is_local_sshd_available():
with contextlib.closing(SSHClient()) as c:
c.set_missing_host_key_policy(WarningPolicy())
#c.load_system_host_keys()
try:
c.connect(local_ipv4_ssh_addr(), password=util.get_val(os.environ, 'SSH_PWD'))
except:
logging.exception('No usable sshd on this machine: ')
return False
else:
return True
def scp_from(ip, remote_path, user='******', password=None, local_path=""):
"""
@param path_to_file: file to copy
@param copy_location: place on localhost to place file
"""
ssh = SSHClient()
ssh.set_missing_host_key_policy(WarningPolicy())
ssh.connect(ip, username=user, password=password, allow_agent=False)
sftp = ssh.open_sftp()
sftp.get(remote_path, local_path)
def hostsFiles(clusterDictionary):
print "Running hostsFile to Build all needed Hosts files."
clusterPath = "./" + clusterDictionary["clusterName"]
with open ("hosts","w") as hostsFile:
hostsFile.write("###### GPDB-GCE ENTRIES #######\n")
for node in clusterDictionary["clusterNodes"]:
hostsFile.write(node["internalIP"]+" "+node["nodeName"]+"\n")
with open ("workers","w") as workersFile:
with open("allhosts", "w") as allhostsFile:
for node in clusterDictionary["clusterNodes"]:
if "master" in node["role"] :
allhostsFile.write(node["nodeName"] + "\n")
elif "standby" in node["role"]:
allhostsFile.write(node["nodeName"] + "\n")
elif "etl" in node["role"]:
allhostsFile.write(node["nodeName"] + "\n")
else:
workersFile.write(node["nodeName"] + "\n")
allhostsFile.write(node["nodeName"] + "\n")
for node in clusterDictionary["clusterNodes"]:
connected = False
attemptCount = 0
while not connected:
try:
attemptCount += 1
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(WarningPolicy())
ssh.connect(node["externalIP"], 22, SSH_USERNAME, None, pkey=None, key_filename=SSH_KEY_PATH,timeout=120)
sftp = ssh.open_sftp()
sftp.put("hosts", "/tmp/hosts")
sftp.put("allhosts", "/tmp/allhosts")
sftp.put("workers", "/tmp/workers")
ssh.exec_command("sudo sh -c 'cat /tmp/hosts >> /etc/hosts'")
connected = True
except Exception as e:
#print e
print node["nodeName"]+": Attempting SSH Connection"
time.sleep(3)
if attemptCount > 40:
print "Failing Process"
exit()
finally:
ssh.close()
def installBits(clusterNode):
logging.info('installBits Started on: ' + str(clusterNode["nodeName"]))
connected = False
attemptCount = 0
while not connected:
try:
attemptCount += 1
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(WarningPolicy())
logging.debug('Connecting to Node: ' + clusterNode["nodeName"])
logging.debug('SSH IP: ' + clusterNode["externalIP"] + ' User: '******' Key: ' +
str(os.environ["CONFIGS_PATH"]) +
str(os.environ["SSH_KEY"]))
ssh.connect(clusterNode["externalIP"],
22,
str(os.environ["SSH_USERNAME"]),
None,
pkey=None,
key_filename=str(os.environ["CONFIGS_PATH"]) +
str(os.environ["SSH_KEY"]),
timeout=120)
logging.info('Installing GPDB')
(stdin, stdout,
stderr) = ssh.exec_command("sudo /tmp/greenplum-db*.bin")
logging.debug(stdout.readlines())
logging.debug(stderr.readlines())
(stdin, stdout, stderr) = ssh.exec_command(
"sudo chown -R gpadmin: /usr/local/greenplum-db*")
logging.debug(stdout.readlines())
logging.debug(stderr.readlines())
# Go ahead and change owner on Data Disk(s). Only One now, but change this if more disks are added.
# (stdin, stdout, stderr) = ssh.exec_command("sudo mkdir -p /data/master;sudo chown -R gpadmin: /data")
# I believe these need to be commented as the command got commented
# stdout.readlines()
# stderr.readlines()
connected = True
except Exception as e:
print clusterNode["nodeName"] + ": Attempting SSH Connection"
time.sleep(3)
if attemptCount > 1:
logging.debug('Exception: ' + str(e))
logging.debug(traceback.print_exc())
logging.debug('Failed')
print "Failing Process"
exit()
finally:
ssh.close()
logging.info('installBits Completed on: ' +
str(clusterNode["nodeName"]))
def initGPDB(clusterDictionary):
warnings.simplefilter("ignore")
client = paramiko.SSHClient()
client.set_missing_host_key_policy(WarningPolicy())
for node in clusterDictionary["clusterNodes"]:
if ("master" in node["role"]):
client.connect(node["externalIP"], 22, "gpadmin", GPADMIN_PW,timeout=120)
print "Initializing Greenplum Database"
(stdin, stdout, stderr) = client.exec_command("gpinitsystem -c /tmp/gpinitsystem_config -a")
output = stdout.readlines()
(stdin, stdout, stderr) = client.exec_command("echo git clonegpinitsystem -c /tmp/gpinitsystem_config -a")
def __init__(self):
super().__init__()
self.set_missing_host_key_policy(WarningPolicy())
# username for Zynq, pswd for Zynq, the Zynq's IP, and the port that it's SSH server is listening on
self.username = "******"
self.password = "******"
self.ip = "169.254.27.144"
self.port = 22
self.persistpath = "/run/media/mmcblk0p1/save/"
self.connect(self.ip,
port=self.port,
username=self.username,
password=self.password)