Example #1
0
def postgres_active(port, host=None):
    ret = 0
    pg_lock_file = '/tmp/.s.PGSQL.%d.lock' % port
    pg_lock_netstat = 0
    pid = 0

    # ping host if present
    if host:
        ok = ping_host(host)
        if not ok:
            return (ok, pid)

    # netstat to check the port number
    (ok, out) = run('%s %s -an 2> /dev/null | %s ".s.PGSQL.%d" | %s \'{print $NF}\'| %s -F"." \'{print $NF}\' | %s -u' % 
                    (host and gplib.ssh_prefix(host=host) or '', ENV.NETSTAT, ENV.GREP, port, ENV.AWK, ENV.AWK, ENV.SORT))
    if not ok:
        return (ok, pid)

    for p_chk in out:
        p = int(p_chk)
        if p == port:
            pg_lock_netstat = port
    pg_lock_tmp = False
    if file_exists(file=pg_lock_file, host=host):
        pg_lock_tmp = True
            
    if pg_lock_netstat == 0 and pg_lock_tmp == False:
        ret = 1
        pid = 0
        log_info('No socket connection or lock file in /tmp found for %s port=%d' % (host and host or '', port))
    else:
        if not pg_lock_tmp and pg_lock_netstat != 0:
            log_warn('No lock file %s but process running on %s port %d' % (pg_lock_file, host and host or '', port))
            ret = 1

        if pg_lock_tmp and pg_lock_netstat == 0:
            if file_exists(file=pg_lock_file, host=host):
                (ok, out) = run('%s %s %s | %s -1 | %s \'{print $1}\'' % (host and gplib.ssh_prefix(host=host) or '', ENV.CAT, pg_lock_file, ENV.HEAD, ENV.AWK))
                if not ok:
                    return (ok, pid)
                pid = int(out[0])
            else:
                log_warn('Unable to access %s' % pg_lock_file)
            log_warn('Have lock file %s but no process running on %s port %d' % (pg_lock_file, host and host or '', port))
            ret = 1
        if pg_lock_tmp and pg_lock_netstat != 0:
            if file_exists(file=pg_lock_file, host=host):
                (ok, out) = run('%s %s %s | %s -1 | %s \'{print $1}\'' % (host and gplib.ssh_prefix(host=host) or '', ENV.CAT, pg_lock_file, ENV.HEAD, ENV.AWK))
                if not ok:
                    return (ok, pid)
                pid = int(out[0])
            else:
                log_warn('Unable to access %s' % pg_lock_file)
                ret = 1
            #log_info('Have lock file %s and a process running on %s port %d' % (pg_lock_file, host and host or '', port))

    return (not ret, pid)
Example #2
0
def append_pg_hba(host, datadir, new_addr):
    pg_hba = os.path.join(datadir, 'pg_hba.conf')
    for addr in new_addr:
        (ok, out) = run_warn('%s "echo host all all %s/32 trust >> %s"' % 
                        (gplib.ssh_prefix(host=host), 
                         addr, pg_hba))
        if not ok:
            fatal('update %s:%s failed' % (host, pg_hba))

    return True
Example #3
0
def chk_multi_home(hostlist, full=True):
    hosts = full and hostlist or hostlist[0:2]
    hostnames=[]
    for host in hosts:
        (ok, out) = run('%s %s' % (gplib.ssh_prefix(host=host), ENV.HOSTNAME))
        if not ok:
            error('failed to run hostname on remote host')
        hostnames.append(out[0].strip())

    hostnames_uniq = unique(hostnames)
    return len(hostnames_uniq) != len(hostnames)
Example #4
0
def get_mirror_type(db, multi_home):
    if not chk_mirrors_configured(db):
        mir_type = 'No Mirror'
        mir_type_num = ENV.MIRROR_NULL_TYPE
    else:
        db.execute("select count(distinct hostname)/(select count(distinct hostname) from gp_segment_configuration where preferred_role='p' and content<>-1) from gp_segment_configuration where content<>-1")
        r = db.read_tuple()
        sep_count = r[0]
        if sep_count == 2:
            sep_text = '[Separate array]'
        else:
            sep_text = '[Shared array]'

        # get number of primary hosts
        db.execute("select count(distinct(hostname)) from gp_segment_configuration where content<>-1")
        r = db.read_tuple()
        num_seg_hosts = r[0]
        # get the primary and mirror hostnames for the first segment instance host
        db.execute("select hostname, content, preferred_role='p' as definedprimary from gp_segment_configuration where content>-1 and content<(select max(port)-min(port)+1 from gp_segment_configuration where content<>-1 and preferred_role='p') order by content,dbid")
        first_pri_mir_array = [ x for x in db.iterate_dict() ]
        first_pri_array = filter(lambda (x): x['definedprimary'] == True, first_pri_mir_array)
        first_mir_array = filter(lambda (x): x['definedprimary'] == False, first_pri_mir_array)
        seg_per_host = len(first_pri_mir_array) / 2
        if not multi_home:
            hosts = [ x['hostname'] for x in first_pri_mir_array ]
            if len(unique(hosts)) == 2 or num_seg_hosts == 1:
                mir_type = 'Group [Single-home] %s' % sep_text
                mir_type_num = ENV.MIRROR_SINGLE_HOME_GROUP_TYPE
            else:
                mir_type = 'Spread [Single-home] %s' % sep_text
                mir_type_num = ENV.MIRROR_SINGLE_HOME_SPREAD_TYPE
        else:
            mir_array = []
            for i in range(seg_per_host):
                pri_host = first_pri_array[i]['hostname']
                db.execute("select hostname from gp_segment_configuration where preferred_role = 'm' and content = %d" % (i))
                r = db.read_tuple()
                mir_host = r[0]
                (ok, out) = run('%s "hostname"' % (gplib.ssh_prefix(host=mir_host)))
                if not ok:
                    error('hostname on %s failed' % mir_host)
                mir_array.append(out[0])

            uniq_cnt = len(unique(mir_array))
            if uniq_cnt == 1:
                mir_type = 'Group [Multi-home] %s' % sep_text
                mir_type_num = ENV.MIRROR_MULTI_HOME_GROUP_TYPE
            else:
                mir_type = 'Spread [Mutli-home] %s' % sep_text
                mir_type_num = ENV.MIRROR_MULTI_HOME_SPREAD_TYPE
                
    return (mir_type_num, mir_type)
Example #5
0
def get_ipaddr(host):
    (ok, out) = run('%s "%s %s | %s \\"inet\\"| %s -v \\"127.0.0\\"| %s -v \\"inet6\\""' %
                    (gplib.ssh_prefix(host=host),  ENV.IFCONFIG, 
                     ENV.IFCONFIG_TXT, ENV.GREP, ENV.GREP, ENV.GREP))

    if not ok:
        return None

    addr = []
    for l in out:
        x = l.strip().split()
        ip = x[1].split(':')
        addr.append(len(ip) == 2 and ip[1] or ip[0])
        
    return addr
Example #6
0
def edit_file(host, file, search_txt, sub_txt, append=False):
    if host != 'localhost':
        gphome = os.environ.get('GPHOME')
        cmd = makeCommand('''python -c \\"import sys; sys.path.extend(['%s', '%s']); import gpmlib; gpmlib.edit_file('localhost', '%s', '%s', '%s', %s)\\"''' % 
                          (os.path.join(gphome, 'bin', 'lib'),
                           os.path.join(gphome, 'lib', 'python'),
                           file,
                           search_txt,
                           sub_txt,
                           str(append)))

        (ok, out) = run2('''%s "%s"''' % (gplib.ssh_prefix(host=host), cmd), True)
        return ok
    else:

        f = None
        tmpf = None
        tmpfile = '%s.tmp' % file
        try:
            try:
                f = open(file, 'r')
                tmpf = open(tmpfile, 'w')
                for line in f:
                    if line.find(search_txt) != -1:
                        tmpf.write(sub_txt)
                        if append:
                            tmpf.write(' # ')
                            tmpf.write(line)
                        else:
                            tmpf.write('\n')
                    else:
                        tmpf.write(line)
            except IOError, e:
                log_error(str(e))
                return False
        finally:
            if f: f.close()
            if tmpf: tmpf.close()

        try:
            os.rename(tmpfile, file)
        except OSError, e:
            log_error('rename file from %s to %s failed' % (tmpfile, file))
            return False


        return True
Example #7
0
def directory_writable(dir, host=None):
    f = None
    file = os.path.join(dir, 'tmp_file_test')
    if not host or host == 'localhost':
        try:
            try:
                f = open(file, 'w')
                f.close()
            except IOError, e:
                fatal('write file %s error' % file)
        finally:
            f.close()
            os.remove(file)
        
    else:
        gphome = os.environ.get('GPHOME')
        cmd = makeCommand('''python -c \\"import sys, os; sys.path.extend(['%s', '%s']); import gpmlib; gpmlib.directory_writable('%s')\\"''' %
                          (os.path.join(gphome, 'bin', 'lib'), os.path.join(gphome, 'lib', 'python'), dir))
        (ok, out) = run('''%s "%s"''' % (gplib.ssh_prefix(host=host), cmd))
        if not ok:
            fatal('write file %s error' % file)
                        
    return True
Example #8
0
def sync_segment_datadir(src_host, src_dir, dst_cfg, logfile, 
                         force_stop=True, setpid_callback=None, err_is_fatal=True, 
                         verbose=False, progressBytes=None, progressTime=None):
    
    if setpid_callback:
        log_warn('Use of sync_segment_datadir setpid_callback keyword is deprecated')

    if directory_exists(dst_cfg['datadir'], dst_cfg['hostname']):
        log_info('Data Directory %s exists' % dst_cfg['datadir'])
    else:
        # make mirror data directory
        log_info('Make directory %s' % dst_cfg['datadir'])
        (ok, out) = run('%s "%s %s && chmod 700 %s"' % 
                        (gplib.ssh_prefix(host=dst_cfg['hostname']),
                         ENV.MKDIR,
                         dst_cfg['datadir'],
                         dst_cfg['datadir']))
        if not ok:
            fatal('mkdir directory %s host %s failed' % (dst_cfg['datadir'], dst_cfg['hostname']))


    # Call PysyncProxy to initiate the sync operation with progress feedback
    pysyncOptions = ["--delete", "-x", "db_dumps", "-x", "pg_log"]
    sync = pysync.PysyncProxy(src_host, src_dir, dst_cfg['hostname'], dst_cfg['datadir'], 
                              pysyncOptions, verbose=verbose, 
                              progressBytes=progressBytes, progressTime=progressTime, 
                              recordProgressCallback=_SyncProgress(log_info, dst_cfg).mark_sync_progress)
    code = sync.run()
    ok = (code == 0)

    if not ok:
        log_warn('-----------------------------------------------------')
        log_warn('Command Failed: %s' % sync.cmd)
        log_warn('Return code: %d' % code)
        log_warn('Exit status: %d' % os.WEXITSTATUS(sync.returncode))
        if sync.stdout:
            log_warn('Standard output:')
            for l in sync.stdout:
                log_warn('\t %s' % l.strip())
        else:
            log_warn('Standard output:  None')

        if sync.stderr:
            log_warn('Standard error:')
            for l in sync.stderr:
                log_warn('\t %s' % l.strip())
        else:
            log_warn('Standard error:   None')
        log_warn('-----------------------------------------------------')

        if err_is_fatal:
            fatal('failed to synchronize data from primary segment %s:%s to mirror segment %s:%s' % (src_host, src_dir, 
                                                                                                 dst_cfg['hostname'], dst_cfg['datadir']))
        else:
            log_error('failed to synchronize data from primary segment %s:%s to mirror segment %s:%s' % (src_host, src_dir, 
                                                                                                 dst_cfg['hostname'], dst_cfg['datadir']))
            return False

    # delete postmaster.pid in case active segment is still running
    postmaster_pid = os.path.join(dst_cfg['datadir'], 'postmaster.pid')
    (ok, out) = run('%s "if [ -f %s ] ; then rm -rf %s; fi"' %
                    (gplib.ssh_prefix(host=dst_cfg['hostname']),
                     postmaster_pid,
                     postmaster_pid))

    # delete postmaster.opts in case active segment is still running
    postmaster_opts = os.path.join(dst_cfg['datadir'], 'postmaster.opts')
    (ok, out) = run('%s "if [ -f %s ] ; then rm -rf %s; fi"' %
                    (gplib.ssh_prefix(host=dst_cfg['hostname']),
                     postmaster_opts,
                     postmaster_opts))
                     

    # change port number in postgresql.conf
    sub_txt = 'port=%d' % dst_cfg['port']
    ok = edit_file(dst_cfg['hostname'], os.path.join(dst_cfg['datadir'], 'postgresql.conf'), 
                   'port=', sub_txt, True)
    if not ok:
        fatal('failed to edit postgresql.conf')

    # start mirror segment in admin mode
    log_info('create pg_log directory')
    cmd = makeCommand('mkdir %s/pg_log' % dst_cfg['datadir'])
    (ok, out) = run_warn('%s "%s"' % (gplib.ssh_prefix(host=dst_cfg['hostname']),cmd))


    log_info('sync segment datadir=%s port=%d on host %s succeeded' % (dst_cfg['datadir'], dst_cfg['port'], dst_cfg['hostname']))
    return True    
    
    log_info('start segment datadir=%s port=%d on %s' % (dst_cfg['datadir'], dst_cfg['port'], dst_cfg['hostname']))
    cmd = makeCommand('env PGOPTIONS=\\"-c gp_session_role=utility\\" %s -w -D %s -l %s/pg_log/startup.log -o \\"-i -p %d \\" start 2>&1' %
                      (os.path.join(ENV.GPHOME, 'bin/pg_ctl'), 
                      dst_cfg['datadir'], dst_cfg['datadir'], dst_cfg['port']))    
    (ok, out) = run_warn('%s "%s"' % 
                         (gplib.ssh_prefix(host=dst_cfg['hostname']),
                          cmd))
    if not ok:
        fatal('failed to start mirror segment %s:%s in admin mode' % (dst_cfg['hostname'], dst_cfg['datadir']))

    if force_stop:
        # stop mirror segment
        log_info('stop segment datadir=%s port=%d on %s' % (dst_cfg['datadir'], dst_cfg['port'], dst_cfg['hostname'])) 
        cmd = makeCommand('env PGOPTIONS=\\"-c gp_session_role=utility\\" %s -w stop -D %s -m smart -l %s.log' % 
                          (os.path.join(ENV.GPHOME, 'bin/pg_ctl'), 
                           dst_cfg['datadir'], dst_cfg['datadir']))
        (ok, out) = run_warn('%s "%s" >> %s 2>&1' % 
                             (gplib.ssh_prefix(host=dst_cfg['hostname']),
                              cmd,
                              logfile))        
        if not ok:
            fatal('failed to stop mirror segment %s:%s' % (dst_cfg['hostname'], dst_cfg['datadir']))
    

    log_info('sync segment datadir=%s port=%d on host %s succeeded' % (dst_cfg['datadir'], dst_cfg['port'], dst_cfg['hostname']))
    return True
Example #9
0
def directory_exists(dir, host=None):
    if not host or host == 'localhost':
        return os.path.isdir(dir)
    else:
        (ok, out) = run('%s test -d %s' % (gplib.ssh_prefix(host=host), dir))
        return ok
Example #10
0
def file_exists(file, host=None):
    if not host or host == 'localhost':
        return os.path.isfile(file)
    else:
        (ok, out) = run('%s test -f %s && test -r %s' % (gplib.ssh_prefix(host=host), file, file))
        return ok