def postgres_active(port, host=None): ret = 0 pg_lock_file = '/tmp/.s.PGSQL.%d.lock' % port pg_lock_netstat = 0 pid = 0 # ping host if present if host: ok = ping_host(host) if not ok: return (ok, pid) # netstat to check the port number (ok, out) = run('%s %s -an 2> /dev/null | %s ".s.PGSQL.%d" | %s \'{print $NF}\'| %s -F"." \'{print $NF}\' | %s -u' % (host and gplib.ssh_prefix(host=host) or '', ENV.NETSTAT, ENV.GREP, port, ENV.AWK, ENV.AWK, ENV.SORT)) if not ok: return (ok, pid) for p_chk in out: p = int(p_chk) if p == port: pg_lock_netstat = port pg_lock_tmp = False if file_exists(file=pg_lock_file, host=host): pg_lock_tmp = True if pg_lock_netstat == 0 and pg_lock_tmp == False: ret = 1 pid = 0 log_info('No socket connection or lock file in /tmp found for %s port=%d' % (host and host or '', port)) else: if not pg_lock_tmp and pg_lock_netstat != 0: log_warn('No lock file %s but process running on %s port %d' % (pg_lock_file, host and host or '', port)) ret = 1 if pg_lock_tmp and pg_lock_netstat == 0: if file_exists(file=pg_lock_file, host=host): (ok, out) = run('%s %s %s | %s -1 | %s \'{print $1}\'' % (host and gplib.ssh_prefix(host=host) or '', ENV.CAT, pg_lock_file, ENV.HEAD, ENV.AWK)) if not ok: return (ok, pid) pid = int(out[0]) else: log_warn('Unable to access %s' % pg_lock_file) log_warn('Have lock file %s but no process running on %s port %d' % (pg_lock_file, host and host or '', port)) ret = 1 if pg_lock_tmp and pg_lock_netstat != 0: if file_exists(file=pg_lock_file, host=host): (ok, out) = run('%s %s %s | %s -1 | %s \'{print $1}\'' % (host and gplib.ssh_prefix(host=host) or '', ENV.CAT, pg_lock_file, ENV.HEAD, ENV.AWK)) if not ok: return (ok, pid) pid = int(out[0]) else: log_warn('Unable to access %s' % pg_lock_file) ret = 1 #log_info('Have lock file %s and a process running on %s port %d' % (pg_lock_file, host and host or '', port)) return (not ret, pid)
def append_pg_hba(host, datadir, new_addr): pg_hba = os.path.join(datadir, 'pg_hba.conf') for addr in new_addr: (ok, out) = run_warn('%s "echo host all all %s/32 trust >> %s"' % (gplib.ssh_prefix(host=host), addr, pg_hba)) if not ok: fatal('update %s:%s failed' % (host, pg_hba)) return True
def chk_multi_home(hostlist, full=True): hosts = full and hostlist or hostlist[0:2] hostnames=[] for host in hosts: (ok, out) = run('%s %s' % (gplib.ssh_prefix(host=host), ENV.HOSTNAME)) if not ok: error('failed to run hostname on remote host') hostnames.append(out[0].strip()) hostnames_uniq = unique(hostnames) return len(hostnames_uniq) != len(hostnames)
def get_mirror_type(db, multi_home): if not chk_mirrors_configured(db): mir_type = 'No Mirror' mir_type_num = ENV.MIRROR_NULL_TYPE else: db.execute("select count(distinct hostname)/(select count(distinct hostname) from gp_segment_configuration where preferred_role='p' and content<>-1) from gp_segment_configuration where content<>-1") r = db.read_tuple() sep_count = r[0] if sep_count == 2: sep_text = '[Separate array]' else: sep_text = '[Shared array]' # get number of primary hosts db.execute("select count(distinct(hostname)) from gp_segment_configuration where content<>-1") r = db.read_tuple() num_seg_hosts = r[0] # get the primary and mirror hostnames for the first segment instance host db.execute("select hostname, content, preferred_role='p' as definedprimary from gp_segment_configuration where content>-1 and content<(select max(port)-min(port)+1 from gp_segment_configuration where content<>-1 and preferred_role='p') order by content,dbid") first_pri_mir_array = [ x for x in db.iterate_dict() ] first_pri_array = filter(lambda (x): x['definedprimary'] == True, first_pri_mir_array) first_mir_array = filter(lambda (x): x['definedprimary'] == False, first_pri_mir_array) seg_per_host = len(first_pri_mir_array) / 2 if not multi_home: hosts = [ x['hostname'] for x in first_pri_mir_array ] if len(unique(hosts)) == 2 or num_seg_hosts == 1: mir_type = 'Group [Single-home] %s' % sep_text mir_type_num = ENV.MIRROR_SINGLE_HOME_GROUP_TYPE else: mir_type = 'Spread [Single-home] %s' % sep_text mir_type_num = ENV.MIRROR_SINGLE_HOME_SPREAD_TYPE else: mir_array = [] for i in range(seg_per_host): pri_host = first_pri_array[i]['hostname'] db.execute("select hostname from gp_segment_configuration where preferred_role = 'm' and content = %d" % (i)) r = db.read_tuple() mir_host = r[0] (ok, out) = run('%s "hostname"' % (gplib.ssh_prefix(host=mir_host))) if not ok: error('hostname on %s failed' % mir_host) mir_array.append(out[0]) uniq_cnt = len(unique(mir_array)) if uniq_cnt == 1: mir_type = 'Group [Multi-home] %s' % sep_text mir_type_num = ENV.MIRROR_MULTI_HOME_GROUP_TYPE else: mir_type = 'Spread [Mutli-home] %s' % sep_text mir_type_num = ENV.MIRROR_MULTI_HOME_SPREAD_TYPE return (mir_type_num, mir_type)
def get_ipaddr(host): (ok, out) = run('%s "%s %s | %s \\"inet\\"| %s -v \\"127.0.0\\"| %s -v \\"inet6\\""' % (gplib.ssh_prefix(host=host), ENV.IFCONFIG, ENV.IFCONFIG_TXT, ENV.GREP, ENV.GREP, ENV.GREP)) if not ok: return None addr = [] for l in out: x = l.strip().split() ip = x[1].split(':') addr.append(len(ip) == 2 and ip[1] or ip[0]) return addr
def edit_file(host, file, search_txt, sub_txt, append=False): if host != 'localhost': gphome = os.environ.get('GPHOME') cmd = makeCommand('''python -c \\"import sys; sys.path.extend(['%s', '%s']); import gpmlib; gpmlib.edit_file('localhost', '%s', '%s', '%s', %s)\\"''' % (os.path.join(gphome, 'bin', 'lib'), os.path.join(gphome, 'lib', 'python'), file, search_txt, sub_txt, str(append))) (ok, out) = run2('''%s "%s"''' % (gplib.ssh_prefix(host=host), cmd), True) return ok else: f = None tmpf = None tmpfile = '%s.tmp' % file try: try: f = open(file, 'r') tmpf = open(tmpfile, 'w') for line in f: if line.find(search_txt) != -1: tmpf.write(sub_txt) if append: tmpf.write(' # ') tmpf.write(line) else: tmpf.write('\n') else: tmpf.write(line) except IOError, e: log_error(str(e)) return False finally: if f: f.close() if tmpf: tmpf.close() try: os.rename(tmpfile, file) except OSError, e: log_error('rename file from %s to %s failed' % (tmpfile, file)) return False return True
def directory_writable(dir, host=None): f = None file = os.path.join(dir, 'tmp_file_test') if not host or host == 'localhost': try: try: f = open(file, 'w') f.close() except IOError, e: fatal('write file %s error' % file) finally: f.close() os.remove(file) else: gphome = os.environ.get('GPHOME') cmd = makeCommand('''python -c \\"import sys, os; sys.path.extend(['%s', '%s']); import gpmlib; gpmlib.directory_writable('%s')\\"''' % (os.path.join(gphome, 'bin', 'lib'), os.path.join(gphome, 'lib', 'python'), dir)) (ok, out) = run('''%s "%s"''' % (gplib.ssh_prefix(host=host), cmd)) if not ok: fatal('write file %s error' % file) return True
def sync_segment_datadir(src_host, src_dir, dst_cfg, logfile, force_stop=True, setpid_callback=None, err_is_fatal=True, verbose=False, progressBytes=None, progressTime=None): if setpid_callback: log_warn('Use of sync_segment_datadir setpid_callback keyword is deprecated') if directory_exists(dst_cfg['datadir'], dst_cfg['hostname']): log_info('Data Directory %s exists' % dst_cfg['datadir']) else: # make mirror data directory log_info('Make directory %s' % dst_cfg['datadir']) (ok, out) = run('%s "%s %s && chmod 700 %s"' % (gplib.ssh_prefix(host=dst_cfg['hostname']), ENV.MKDIR, dst_cfg['datadir'], dst_cfg['datadir'])) if not ok: fatal('mkdir directory %s host %s failed' % (dst_cfg['datadir'], dst_cfg['hostname'])) # Call PysyncProxy to initiate the sync operation with progress feedback pysyncOptions = ["--delete", "-x", "db_dumps", "-x", "pg_log"] sync = pysync.PysyncProxy(src_host, src_dir, dst_cfg['hostname'], dst_cfg['datadir'], pysyncOptions, verbose=verbose, progressBytes=progressBytes, progressTime=progressTime, recordProgressCallback=_SyncProgress(log_info, dst_cfg).mark_sync_progress) code = sync.run() ok = (code == 0) if not ok: log_warn('-----------------------------------------------------') log_warn('Command Failed: %s' % sync.cmd) log_warn('Return code: %d' % code) log_warn('Exit status: %d' % os.WEXITSTATUS(sync.returncode)) if sync.stdout: log_warn('Standard output:') for l in sync.stdout: log_warn('\t %s' % l.strip()) else: log_warn('Standard output: None') if sync.stderr: log_warn('Standard error:') for l in sync.stderr: log_warn('\t %s' % l.strip()) else: log_warn('Standard error: None') log_warn('-----------------------------------------------------') if err_is_fatal: fatal('failed to synchronize data from primary segment %s:%s to mirror segment %s:%s' % (src_host, src_dir, dst_cfg['hostname'], dst_cfg['datadir'])) else: log_error('failed to synchronize data from primary segment %s:%s to mirror segment %s:%s' % (src_host, src_dir, dst_cfg['hostname'], dst_cfg['datadir'])) return False # delete postmaster.pid in case active segment is still running postmaster_pid = os.path.join(dst_cfg['datadir'], 'postmaster.pid') (ok, out) = run('%s "if [ -f %s ] ; then rm -rf %s; fi"' % (gplib.ssh_prefix(host=dst_cfg['hostname']), postmaster_pid, postmaster_pid)) # delete postmaster.opts in case active segment is still running postmaster_opts = os.path.join(dst_cfg['datadir'], 'postmaster.opts') (ok, out) = run('%s "if [ -f %s ] ; then rm -rf %s; fi"' % (gplib.ssh_prefix(host=dst_cfg['hostname']), postmaster_opts, postmaster_opts)) # change port number in postgresql.conf sub_txt = 'port=%d' % dst_cfg['port'] ok = edit_file(dst_cfg['hostname'], os.path.join(dst_cfg['datadir'], 'postgresql.conf'), 'port=', sub_txt, True) if not ok: fatal('failed to edit postgresql.conf') # start mirror segment in admin mode log_info('create pg_log directory') cmd = makeCommand('mkdir %s/pg_log' % dst_cfg['datadir']) (ok, out) = run_warn('%s "%s"' % (gplib.ssh_prefix(host=dst_cfg['hostname']),cmd)) log_info('sync segment datadir=%s port=%d on host %s succeeded' % (dst_cfg['datadir'], dst_cfg['port'], dst_cfg['hostname'])) return True log_info('start segment datadir=%s port=%d on %s' % (dst_cfg['datadir'], dst_cfg['port'], dst_cfg['hostname'])) cmd = makeCommand('env PGOPTIONS=\\"-c gp_session_role=utility\\" %s -w -D %s -l %s/pg_log/startup.log -o \\"-i -p %d \\" start 2>&1' % (os.path.join(ENV.GPHOME, 'bin/pg_ctl'), dst_cfg['datadir'], dst_cfg['datadir'], dst_cfg['port'])) (ok, out) = run_warn('%s "%s"' % (gplib.ssh_prefix(host=dst_cfg['hostname']), cmd)) if not ok: fatal('failed to start mirror segment %s:%s in admin mode' % (dst_cfg['hostname'], dst_cfg['datadir'])) if force_stop: # stop mirror segment log_info('stop segment datadir=%s port=%d on %s' % (dst_cfg['datadir'], dst_cfg['port'], dst_cfg['hostname'])) cmd = makeCommand('env PGOPTIONS=\\"-c gp_session_role=utility\\" %s -w stop -D %s -m smart -l %s.log' % (os.path.join(ENV.GPHOME, 'bin/pg_ctl'), dst_cfg['datadir'], dst_cfg['datadir'])) (ok, out) = run_warn('%s "%s" >> %s 2>&1' % (gplib.ssh_prefix(host=dst_cfg['hostname']), cmd, logfile)) if not ok: fatal('failed to stop mirror segment %s:%s' % (dst_cfg['hostname'], dst_cfg['datadir'])) log_info('sync segment datadir=%s port=%d on host %s succeeded' % (dst_cfg['datadir'], dst_cfg['port'], dst_cfg['hostname'])) return True
def directory_exists(dir, host=None): if not host or host == 'localhost': return os.path.isdir(dir) else: (ok, out) = run('%s test -d %s' % (gplib.ssh_prefix(host=host), dir)) return ok
def file_exists(file, host=None): if not host or host == 'localhost': return os.path.isfile(file) else: (ok, out) = run('%s test -f %s && test -r %s' % (gplib.ssh_prefix(host=host), file, file)) return ok