def execute(self): needed_space = 0 dburl = dbconn.DbURL(dbname=self.dump_database, port=self.segport) conn = None try: conn = dbconn.connect(dburl, utility=True) if self.include_dump_tables: for dump_table in self.include_dump_tables: schema, table = dump_table.split('.') needed_space += execSQLForSingleton( conn, "select sotdsize/1024 from gp_toolkit.gp_size_of_table_disk where sotdschemaname='%s' and sotdtablename='%s';" % (schema, table)) else: needed_space = execSQLForSingleton( conn, "select sodddatsize/1024 from gp_toolkit.gp_size_of_database where sodddatname='%s';" % self.dump_database) except UnexpectedRowsError, e: logger.exception( "Disk space queries have failed. Cannot estimate disk space needed for dump." ) raise ExceptionNoStackTraceNeeded( "Cannot estimate disk space needed for dump. Use -b to override this check." )
def execute(self): schema, table = UpdateHistoryTable.HISTORY_TABLE.split('.') exists = CheckTableExists(database=self.dump_database, schema=schema, table=table, master_port=self.master_port).run() if not exists: conn = None CREATE_HISTORY_TABLE = """ create table %s (rec_date timestamp, start_time char(8), end_time char(8), options text, dump_key varchar(20), dump_exit_status smallint, script_exit_status smallint, exit_text varchar(10)) distributed by (rec_date); """ % UpdateHistoryTable.HISTORY_TABLE try: dburl = dbconn.DbURL(port=self.master_port, dbname=self.dump_database) conn = dbconn.connect(dburl) execSQL(conn, CREATE_HISTORY_TABLE) conn.commit() except Exception, e: logger.exception( "Unable to create %s in %s database" % (UpdateHistoryTable.HISTORY_TABLE, self.dump_database)) return else: logger.info( "Created %s in %s database" % (UpdateHistoryTable.HISTORY_TABLE, self.dump_database)) finally:
def execute(self): dburl = dbconn.DbURL() query = self.UPDATE_VERIFICATION_ENTRY % (self.state, self.done, self.mismatch, self.token) with dbconn.connect(dburl, allowSystemTableMods='dml') as conn: dbconn.execSQL(conn, query) conn.commit()
def execute(self): dburl = dbconn.DbURL() query = self.INSERT_VERIFICATION_ENTRY % ( self.token, self.type, self.content, VerificationState.RUNNING) with dbconn.connect(dburl, allowSystemTableMods='dml') as conn: dbconn.execSQL(conn, query) conn.commit()
def execute(self): dburl = dbconn.DbURL() query = self.SELECT_VERIFICATION_ENTRY % self.token with dbconn.connect(dburl) as conn: try: tuple = dbconn.execSQLForSingletonRow(conn, query) except UnexpectedRowsError, e: if e.actual == 0: raise TokenNotFound(self.token) raise
def __init__(self, masterDataDir, readFromMasterCatalog, timeout=None, retries=None): """ masterDataDir: if None then we try to find it from the system environment readFromMasterCatalog: if True then we will connect to the master in utility mode and fetch some more data from there (like collation settings) """ if masterDataDir is None: self.__masterDataDir = gp.get_masterdatadir() else: self.__masterDataDir = masterDataDir logger.debug("Obtaining master's port from master data directory") pgconf_dict = pgconf.readfile(self.__masterDataDir + "/postgresql.conf") self.__masterPort = pgconf_dict.int('port') logger.debug("Read from postgresql.conf port=%s" % self.__masterPort) self.__masterMaxConnections = pgconf_dict.int('max_connections') logger.debug("Read from postgresql.conf max_connections=%s" % self.__masterMaxConnections) self.__gpHome = gp.get_gphome() self.__gpVersion = gp.GpVersion.local( 'local GP software version check', self.__gpHome) logger.info("local Greenplum Version: '%s'" % self.__gpVersion) # read collation settings from master if readFromMasterCatalog: dbUrl = dbconn.DbURL(port=self.__masterPort, dbname='template1', timeout=timeout, retries=retries) conn = dbconn.connect(dbUrl, utility=True) (self.__lcCollate, self.__lcMonetary, self.__lcNumeric) = catalog.getCollationSettings(conn) # MPP-13807, read/show the master's database version too self.__pgVersion = dbconn.execSQLForSingletonRow( conn, "select version();")[0] logger.info("master Greenplum Version: '%s'" % self.__pgVersion) conn.close() checkNotNone("lc_collate", self.__lcCollate) checkNotNone("lc_monetary", self.__lcMonetary) checkNotNone("lc_numeric", self.__lcNumeric) else: self.__lcCollate = None self.__lcMonetary = None self.__lcNumeric = None self.__pgVersion = None
def _analyze(self, restore_db, master_port): conn = None logger.info('Commencing analyze of %s database, please wait' % restore_db) try: dburl = dbconn.DbURL(port=master_port, dbname=restore_db) conn = dbconn.connect(dburl) execSQL(conn, 'analyze') conn.commit() except Exception, e: logger.warn('Issue with analyze of %s database' % restore_db)
def execute(self): try: dburl = dbconn.DbURL(port=self.master_port, dbname=self.database) conn = dbconn.connect(dburl) count = execSQLForSingleton( conn, "select count(*) from pg_class, pg_namespace where pg_class.relname = '%s' and pg_class.relnamespace = pg_namespace.oid and pg_namespace.nspname = '%s'" % (self.table, self.schema)) return count > 0 finally: if conn is not None: conn.close()
def pauseFaultProber(self): assert not self.__isPaused assert self.__masterDbUrl is not None # must be initialized assert self.__conn is None logger.debug("Pausing fault prober") self.__conn = dbconn.connect( self.__masterDbUrl, True ) # use utility mode so we don't do any segment connection stuff dbconn.execSQL(self.__conn, "set gp_fts_probe_pause = on") self.__isPaused = True
def execute(self): conn = None try: dburl = dbconn.DbURL(port=self.master_port) conn = dbconn.connect(dburl) count = execSQLForSingleton( conn, "select count(*) from pg_database where datname='%s';" % self.database) if count == 0: raise ExceptionNoStackTraceNeeded( "Database %s does not exist." % self.database) finally: if conn is not None: conn.close()
def execute(self): conn = None logger.info('Commencing vacuum of %s database, please wait' % self.database) try: dburl = dbconn.DbURL(port=self.master_port, dbname=self.database) conn = dbconn.connect(dburl) cursor = conn.cursor() cursor.execute( "commit") # hack to move drop stmt out of implied transaction cursor.execute("vacuum") cursor.close() except Exception, e: logger.exception('Error encountered with vacuum of %s database' % self.database)
def execute(self): existing_tables = [] table_counts = [] conn = None try: dburl = dbconn.DbURL(port=self.master_port, dbname=self.restore_db) conn = dbconn.connect(dburl) for restore_table in self.restore_tables: if '.' not in restore_table: logger.warn( "No schema name supplied for %s, removing from list of tables to restore" % restore_table) continue schema, table = restore_table.split('.') count = execSQLForSingleton( conn, "select count(*) from pg_class, pg_namespace where pg_class.relname = '%s' and pg_class.relnamespace = pg_namespace.oid and pg_namespace.nspname = '%s'" % (table, schema)) if count == 0: logger.warn( "Table %s does not exist in database %s, removing from list of tables to restore" % (table, self.restore_db)) continue count = execSQLForSingleton( conn, "select count(*) from %s.%s" % (schema, table)) if count > 0: logger.warn('Table %s has %d records %s' % (restore_table, count, WARN_MARK)) existing_tables.append(restore_table) table_counts.append((restore_table, count)) finally: if conn is not None: conn.close() if len(existing_tables) == 0: raise ExceptionNoStackTraceNeeded("Have no tables to restore") logger.info("Have %d tables to restore, will continue" % len(existing_tables)) return (existing_tables, table_counts)
def execute(self): dburl = dbconn.DbURL(dbname=self.database, port=self.master_port) conn = None try: conn = dbconn.connect(dburl) count = execSQLForSingleton( conn, "select count(*) from pg_class, pg_namespace where pg_namespace.nspname = 'gp_toolkit' and pg_class.relnamespace = pg_namespace.oid" ) finally: if conn is not None: conn.close() if count > 0: logger.debug("gp_toolkit exists within database %s." % self.database) return logger.info("gp_toolkit not found. Installing...") Psql('Installing gp_toolkit', filename='$GPHOME/share/postgresql/gp_toolkit.sql', database=self.database, port=self.master_port).run(validateAfter=True)
def _analyze(self, restore_db, restore_tables, master_port): conn = None try: dburl = dbconn.DbURL(port=master_port, dbname=restore_db) conn = dbconn.connect(dburl) for table in restore_tables: logger.info( 'Commencing analyze of %s in %s database, please wait...' % (table, restore_db)) try: execSQL(conn, 'analyze %s' % table) conn.commit() except Exception, e: logger.warn( 'Issue with analyze of %s table, check log file for details' % table) else: logger.info('Analyze of %s table completed without error' % table) finally: if conn is not None: conn.close()
def execute(self): ret = [] dburl = dbconn.DbURL() with dbconn.connect(dburl) as conn: # TODO: improve execSQL APIs to avoid need to use cursor here for such a simple task cursor = conn.cursor() cursor.execute(self.SELECT_ALL_VERIFICATIONS) res = cursor.fetchall() cursor.close() for tuple in res: # TODO: execSQL or pygresql should be able to do this for us ret.append({ 'vertoken': tuple[0], 'vertype': tuple[1], 'vercontent': tuple[2], 'verstarttime': tuple[3], 'verstate': tuple[4], 'verdone': tuple[5], 'verendtime': tuple[6], 'vermismatch': tuple[7] }) return ret
def getFaultProberInterval(self): probe_interval_re = re.compile(r'(?P<val>\d+)(?P<unit>[a-zA-Z]*)') probe_interval_secs = 60 conn = None try: conn = dbconn.connect(self.__masterDbUrl, True) fts_probe_interval_value = catalog.getSessionGUC( conn, 'gp_fts_probe_interval') m = probe_interval_re.match(fts_probe_interval_value) if m.group('unit') == 'min': probe_interval_secs = int(m.group('val')) * 60 else: probe_interval_secs = int(m.group('val')) except: raise finally: if conn: conn.close() return probe_interval_secs
def _process_createdb(self, restore_timestamp, restore_db, master_datadir, master_port): conn = None try: dburl = dbconn.DbURL(port=master_port) conn = dbconn.connect(dburl) count = execSQLForSingleton( conn, "select count(*) from pg_database where datname='%s';" % restore_db) if count == 1: logger.info("Dropping database %s" % restore_db) try: cursor = conn.cursor() cursor.execute( "commit" ) # hack to move drop stmt out of implied transaction cursor.execute("drop database %s" % restore_db) cursor.close() except Exception, e: logger.exception("Could not create database %s" % restore_db) raise ExceptionNoStackTraceNeeded( 'Failed to drop database %s' % restore_db) else: logger.info('Dropped database %s' % restore_db) finally: if conn is not None: conn.close() createdb_file = os.path.join( master_datadir, DUMP_DIR, restore_timestamp[0:8], "%s%s" % (CREATEDB_PREFIX, restore_timestamp)) logger.info('Invoking %s' % createdb_file) Psql('Invoking schema dump', filename=createdb_file).run(validateAfter=True)
def execute(self): dburl = dbconn.DbURL() query = self.REMOVE_VERIFICATION_ENTRY % self.token with dbconn.connect(dburl, allowSystemTableMods='dml') as conn: dbconn.execSQL(conn, query) conn.commit()
def rebalance(self): # Get the unbalanced primary segments grouped by hostname # These segments are what we will shutdown. logger.info("Getting unbalanced segments") unbalanced_primary_segs = GpArray.getSegmentsByHostName( self.gpArray.get_unbalanced_primary_segdbs()) pool = WorkerPool() count = 0 try: # Disable ctrl-c signal.signal(signal.SIGINT, signal.SIG_IGN) logger.info("Stopping unbalanced primary segments...") for hostname in unbalanced_primary_segs.keys(): cmd = GpSegStopCmd("stop unbalanced primary segs", self.gpEnv.getGpHome(), self.gpEnv.getGpVersion(), 'fast', unbalanced_primary_segs[hostname], ctxt=REMOTE, remoteHost=hostname, timeout=600) pool.addCommand(cmd) count += 1 pool.wait_and_printdots(count, False) failed_count = 0 completed = pool.getCompletedItems() for res in completed: if not res.get_results().wasSuccessful(): failed_count += 1 if failed_count > 0: logger.warn( "%d segments failed to stop. A full rebalance of the") logger.warn( "system is not possible at this time. Please check the") logger.warn( "log files, correct the problem, and run gprecoverseg -r") logger.warn("again.") logger.info( "gprecoverseg will continue with a partial rebalance.") pool.empty_completed_items() # issue a distributed query to make sure we pick up the fault # that we just caused by shutting down segments conn = None try: logger.info("Triggering segment reconfiguration") dburl = dbconn.DbURL() conn = dbconn.connect(dburl) cmd = ReconfigDetectionSQLQueryCommand(conn) pool.addCommand(cmd) pool.wait_and_printdots(1, False) except Exception: # This exception is expected pass finally: if conn: conn.close() # Final step is to issue a recoverseg operation to resync segments logger.info("Starting segment synchronization") cmd = GpRecoverseg("rebalance recoverseg") pool.addCommand(cmd) pool.wait_and_printdots(1, False) except Exception, ex: raise ex
def setUp(self): self.dburl = dbconn.DbURL() self.conn = dbconn.connect(self.dburl)
def skipIfDatabaseDown(): try: dbconn = connect(DbURL()) except: return unittest.skip("database must be up") return lambda o: o
(UpdateHistoryTable.HISTORY_TABLE, self.dump_database)) finally: if conn is not None: conn.close() translate_rc_to_msg = {0: "COMPLETED", 1: "WARNING", 2: "FATAL"} exit_msg = translate_rc_to_msg[self.pseudo_exit_status] APPEND_HISTORY_TABLE = """ insert into %s values (now(), '%s', '%s', '%s', '%s', %d, %d, '%s'); """ % ( UpdateHistoryTable.HISTORY_TABLE, self.time_start, self.time_end, self.options_list, self.timestamp, self.dump_exit_status, self.pseudo_exit_status, exit_msg) conn = None try: dburl = dbconn.DbURL(port=self.master_port, dbname=self.dump_database) conn = dbconn.connect(dburl) execSQL(conn, APPEND_HISTORY_TABLE) conn.commit() except Exception, e: logger.exception( "Failed to insert record into %s in %s database" % (UpdateHistoryTable.HISTORY_TABLE, self.dump_database)) else: logger.info("Inserted dump record into %s in %s database" % (UpdateHistoryTable.HISTORY_TABLE, self.dump_database)) finally: if conn is not None: conn.close() class DumpGlobal(Operation):