def add_table(self, src_db, dst_db, tbl, create_flags): src_curs = src_db.cursor() dst_curs = dst_db.cursor() tbl_exists = skytools.exists_table(dst_curs, tbl) if create_flags: if tbl_exists: self.log.info('Table %s already exist, not touching' % tbl) else: if not skytools.exists_table(src_curs, tbl): # table not present on provider - nowhere to get the DDL from self.log.warning('Table "%s" missing on provider, skipping' % tbl) return schema = skytools.fq_name_parts(tbl)[0] if not skytools.exists_schema(dst_curs, schema): q = "create schema %s" % skytools.quote_ident(schema) dst_curs.execute(q) s = skytools.TableStruct(src_curs, tbl) src_db.commit() s.create(dst_curs, create_flags, log = self.log) elif not tbl_exists: self.log.warning('Table "%s" missing on subscriber, use --create if necessary' % tbl) return q = "select * from londiste.local_add_table(%s, %s)" self.exec_cmd(dst_curs, q, [self.set_name, tbl]) dst_db.commit()
def add_table(self, src_db, dst_db, tbl, create_flags): src_curs = src_db.cursor() dst_curs = dst_db.cursor() tbl_exists = skytools.exists_table(dst_curs, tbl) if create_flags: if tbl_exists: self.log.info('Table %s already exist, not touching' % tbl) else: if not skytools.exists_table(src_curs, tbl): # table not present on provider - nowhere to get the DDL from self.log.warning('Table "%s" missing on provider, skipping' % tbl) return schema = skytools.fq_name_parts(tbl)[0] if not skytools.exists_schema(dst_curs, schema): q = "create schema %s" % skytools.quote_ident(schema) dst_curs.execute(q) s = skytools.TableStruct(src_curs, tbl) src_db.commit() s.create(dst_curs, create_flags, log = self.log) elif not tbl_exists: self.log.warning('Table "%s" missing on subscriber, use --create if necessary' % tbl) return # actual table registration q = "select * from londiste.local_add_table(%s, %s)" self.exec_cmd(dst_curs, q, [self.set_name, tbl]) if self.options.expect_sync: q = "select * from londiste.local_set_table_state(%s, %s, NULL, 'ok')" self.exec_cmd(dst_curs, q, [self.set_name, tbl]) if self.options.copy_condition: q = "select * from londiste.local_set_table_attrs(%s, %s, %s)" attrs = {'copy_condition': self.options.copy_condition} enc_attrs = skytools.db_urlencode(attrs) self.exec_cmd(dst_curs, q, [self.set_name, tbl, enc_attrs]) dst_db.commit()
def add_table(self, src_db, dst_db, tbl, create_flags): src_curs = src_db.cursor() dst_curs = dst_db.cursor() tbl_exists = skytools.exists_table(dst_curs, tbl) if create_flags: if tbl_exists: self.log.info('Table %s already exist, not touching' % tbl) else: if not skytools.exists_table(src_curs, tbl): # table not present on provider - nowhere to get the DDL from self.log.warning('Table "%s" missing on provider, skipping' % tbl) return schema = skytools.fq_name_parts(tbl)[0] if not skytools.exists_schema(dst_curs, schema): q = "create schema %s" % skytools.quote_ident(schema) dst_curs.execute(q) s = skytools.TableStruct(src_curs, tbl) src_db.commit() s.create(dst_curs, create_flags, log = self.log) elif not tbl_exists: self.log.warning('Table "%s" missing on subscriber, use --create if necessary' % tbl) return tgargs = [] if self.options.trigger_arg: tgargs = self.options.trigger_arg tgflags = self.options.trigger_flags if tgflags: tgargs.append('tgflags='+tgflags) if self.options.no_triggers: tgargs.append('no_triggers') if self.options.merge_all: tgargs.append('merge_all') attrs = {} if self.options.handler: hstr = londiste.handler.create_handler_string( self.options.handler, self.options.handler_arg) p = londiste.handler.build_handler(tbl, hstr, self.log) attrs['handler'] = hstr p.add(tgargs) # actual table registration q = "select * from londiste.local_add_table(%s, %s, %s)" self.exec_cmd(dst_curs, q, [self.set_name, tbl, tgargs]) if self.options.expect_sync: q = "select * from londiste.local_set_table_state(%s, %s, NULL, 'ok')" self.exec_cmd(dst_curs, q, [self.set_name, tbl]) else: if self.options.skip_truncate: attrs['skip_truncate'] = 1 if self.options.copy_condition: attrs['copy_condition'] = self.options.copy_condition if attrs: enc_attrs = skytools.db_urlencode(attrs) q = "select * from londiste.local_set_table_attrs(%s, %s, %s)" self.exec_cmd(dst_curs, q, [self.set_name, tbl, enc_attrs]) dst_db.commit()
def load_cols(self, curs, tbl): schema, table = skytools.fq_name_parts(tbl) q = "select column_name from information_schema.columns"\ " where table_schema = %s and table_name = %s" curs.execute(q, [schema, table]) cols = [] for row in curs.fetchall(): cols.append(row[0]) return cols
def add_table(self, src_db, dst_db, tbl, create_flags): src_curs = src_db.cursor() dst_curs = dst_db.cursor() tbl_exists = skytools.exists_table(dst_curs, tbl) if create_flags: if tbl_exists: self.log.info('Table %s already exist, not touching' % tbl) else: if not skytools.exists_table(src_curs, tbl): # table not present on provider - nowhere to get the DDL from self.log.warning('Table "%s" missing on provider, skipping' % tbl) return schema = skytools.fq_name_parts(tbl)[0] if not skytools.exists_schema(dst_curs, schema): q = "create schema %s" % skytools.quote_ident(schema) dst_curs.execute(q) s = skytools.TableStruct(src_curs, tbl) src_db.commit() s.create(dst_curs, create_flags, log = self.log) elif not tbl_exists: self.log.warning('Table "%s" missing on subscriber, use --create if necessary' % tbl) return tgargs = [] if self.options.trigger_arg: tgargs = self.options.trigger_arg tgflags = self.options.trigger_flags if tgflags: tgargs.append('tgflags='+tgflags) if self.options.no_triggers: tgargs.append('no_triggers') attrs = {} if self.options.handler: hstr = londiste.handler.create_handler_string( self.options.handler, self.options.handler_arg) p = londiste.handler.build_handler(tbl, hstr, self.log) attrs['handler'] = hstr p.add(tgargs) # actual table registration q = "select * from londiste.local_add_table(%s, %s, %s)" self.exec_cmd(dst_curs, q, [self.set_name, tbl, tgargs]) if self.options.expect_sync: q = "select * from londiste.local_set_table_state(%s, %s, NULL, 'ok')" self.exec_cmd(dst_curs, q, [self.set_name, tbl]) else: if self.options.skip_truncate: attrs['skip_truncate'] = 1 if self.options.copy_condition: attrs['copy_condition'] = self.options.copy_condition if attrs: enc_attrs = skytools.db_urlencode(attrs) q = "select * from londiste.local_set_table_attrs(%s, %s, %s)" self.exec_cmd(dst_curs, q, [self.set_name, tbl, enc_attrs]) dst_db.commit()
def __init__(self, ctx, table, create_log = True ): """ Table name is used to construct insert update and delete statements Table must have primary key field whose name is in format id_<table> Tablename should be in format schema.tablename """ self._ctx = ctx self._table = skytools.quote_fqident(table) self._id = "id_" + skytools.fq_name_parts(table)[1] self._where = skytools.quote_ident(self._id) + " = {" + self._id + "}" self._logging = create_log
def __init__(self, curs, table_name): """Initializes class by loading info about table_name from database.""" super(TableStruct, self).__init__(curs, table_name) self.table_name = table_name # fill args schema, name = skytools.fq_name_parts(table_name) args = { 'schema': schema, 'table': name, 'fqname': self.fqname, 'fq2name': skytools.quote_literal(self.fqname), 'oid': skytools.get_table_oid(curs, table_name), 'pg_class_oid': skytools.get_table_oid(curs, 'pg_catalog.pg_class'), } # load table struct self.col_list = self._load_elem(curs, self.name, args, TColumn) # if db is GP then read also table distribution keys if skytools.exists_table(curs, "pg_catalog.gp_distribution_policy"): self.dist_key_list = self._load_elem(curs, self.name, args, TGPDistKey) else: self.dist_key_list = None self.object_list = [ TTable(table_name, self.col_list, self.dist_key_list) ] self.seq_list = [] # load seqs for col in self.col_list: if col.seqname: fqname = quote_fqident(col.seqname) owner = self.fqname + '.' + quote_ident(col.name) seq_args = { 'fqname': fqname, 'fq2name': skytools.quote_literal(fqname), 'owner': skytools.quote_literal(owner), } self.seq_list += self._load_elem(curs, col.seqname, seq_args, TSeq) self.object_list += self.seq_list # load additional objects to_load = [ TColumnDefault, TConstraint, TIndex, TTrigger, TRule, TGrant, TOwner, TParent ] for eclass in to_load: self.object_list += self._load_elem(curs, self.name, args, eclass)
def find_dist_fields(self, curs): if not skytools.exists_table(curs, "pg_catalog.gp_distribution_policy"): return [] schema, name = skytools.fq_name_parts(self.table_name) q = "select a.attname"\ " from pg_class t, pg_namespace n, pg_attribute a,"\ " gp_distribution_policy p"\ " where n.oid = t.relnamespace"\ " and p.localoid = t.oid"\ " and a.attrelid = t.oid"\ " and a.attnum = any(p.attrnums)"\ " and n.nspname = %s and t.relname = %s" curs.execute(q, [schema, name]) res = [] for row in curs.fetchall(): res.append(row[0]) return res
def find_dist_fields(self, curs): if not skytools.exists_table(curs, "pg_catalog.mpp_distribution_policy"): return [] schema, name = skytools.fq_name_parts(self.table_name) q = "select a.attname"\ " from pg_class t, pg_namespace n, pg_attribute a,"\ " mpp_distribution_policy p"\ " where n.oid = t.relnamespace"\ " and p.localoid = t.oid"\ " and a.attrelid = t.oid"\ " and a.attnum = any(p.attrnums)"\ " and n.nspname = %s and t.relname = %s" curs.execute(q, [schema, name]) res = [] for row in curs.fetchall(): res.append(row[0]) return res
def __init__(self, curs, table_name): """Initializes class by loading info about table_name from database.""" BaseStruct.__init__(self, curs, table_name) self.table_name = table_name # fill args schema, name = skytools.fq_name_parts(table_name) args = { 'schema': schema, 'table': name, 'fqname': self.fqname, 'fq2name': skytools.quote_literal(self.fqname), 'oid': skytools.get_table_oid(curs, table_name), 'pg_class_oid': skytools.get_table_oid(curs, 'pg_catalog.pg_class'), } # load table struct self.col_list = self._load_elem(curs, self.name, args, TColumn) # if db is GP then read also table distribution keys if skytools.exists_table(curs, "pg_catalog.gp_distribution_policy"): self.dist_key_list = self._load_elem(curs, self.name, args, TGPDistKey) else: self.dist_key_list = None self.object_list = [ TTable(table_name, self.col_list, self.dist_key_list) ] self.seq_list = [] # load seqs for col in self.col_list: if col.seqname: fqname = quote_fqident(col.seqname) owner = self.fqname + '.' + quote_ident(col.name) seq_args = { 'fqname': fqname, 'owner': skytools.quote_literal(owner) } self.seq_list += self._load_elem(curs, col.seqname, seq_args, TSeq) self.object_list += self.seq_list # load additional objects to_load = [TColumnDefault, TConstraint, TIndex, TTrigger, TRule, TGrant, TOwner, TParent] for eclass in to_load: self.object_list += self._load_elem(curs, self.name, args, eclass)
def add_table(self, src_db, dst_db, tbl, create_flags, src_tbls): # use full names tbl = skytools.fq_name(tbl) dest_table = self.options.dest_table or tbl dest_table = skytools.fq_name(dest_table) src_curs = src_db.cursor() dst_curs = dst_db.cursor() tbl_exists = skytools.exists_table(dst_curs, dest_table) if dest_table == tbl: desc = tbl else: desc = "%s(%s)" % (tbl, dest_table) if create_flags: if tbl_exists: self.log.info('Table %s already exist, not touching' % desc) else: src_dest_table = src_tbls[tbl]['dest_table'] if not skytools.exists_table(src_curs, src_dest_table): # table not present on provider - nowhere to get the DDL from self.log.warning('Table %s missing on provider, cannot create, skipping' % desc) return schema = skytools.fq_name_parts(dest_table)[0] if not skytools.exists_schema(dst_curs, schema): q = "create schema %s" % skytools.quote_ident(schema) dst_curs.execute(q) s = skytools.TableStruct(src_curs, src_dest_table) src_db.commit() # create, using rename logic only when necessary newname = None if src_dest_table != dest_table: newname = dest_table s.create(dst_curs, create_flags, log = self.log, new_table_name = newname) tgargs = [] if self.options.trigger_arg: tgargs = self.options.trigger_arg tgflags = self.options.trigger_flags if tgflags: tgargs.append('tgflags='+tgflags) if self.options.no_triggers: tgargs.append('no_triggers') if self.options.merge_all: tgargs.append('merge_all') if self.options.no_merge: tgargs.append('no_merge') attrs = {} if self.options.handler: hstr = londiste.handler.create_handler_string( self.options.handler, self.options.handler_arg) p = londiste.handler.build_handler(tbl, hstr, self.options.dest_table) attrs['handler'] = hstr p.add(tgargs) if self.options.find_copy_node: attrs['copy_node'] = '?' elif self.options.copy_node: attrs['copy_node'] = self.options.copy_node if self.options.expect_sync: tgargs.append('expect_sync') if not self.options.expect_sync: if self.options.skip_truncate: attrs['skip_truncate'] = 1 if self.options.max_parallel_copy: attrs['max_parallel_copy'] = self.options.max_parallel_copy # actual table registration args = [self.set_name, tbl, tgargs, None, None] if attrs: args[3] = skytools.db_urlencode(attrs) if dest_table != tbl: args[4] = dest_table q = "select * from londiste.local_add_table(%s, %s, %s, %s, %s)" self.exec_cmd(dst_curs, q, args) dst_db.commit()
def add_table(self, src_db, dst_db, tbl, create_flags, src_tbls): # use full names tbl = skytools.fq_name(tbl) dest_table = self.options.dest_table or tbl dest_table = skytools.fq_name(dest_table) src_curs = src_db.cursor() dst_curs = dst_db.cursor() tbl_exists = skytools.exists_table(dst_curs, dest_table) dst_db.commit() self.set_lock_timeout(dst_curs) if dest_table == tbl: desc = tbl else: desc = "%s(%s)" % (tbl, dest_table) if create_flags: if tbl_exists: self.log.info('Table %s already exist, not touching', desc) else: src_dest_table = src_tbls[tbl]['dest_table'] if not skytools.exists_table(src_curs, src_dest_table): # table not present on provider - nowhere to get the DDL from self.log.warning( 'Table %s missing on provider, cannot create, skipping', desc) return schema = skytools.fq_name_parts(dest_table)[0] if not skytools.exists_schema(dst_curs, schema): q = "create schema %s" % skytools.quote_ident(schema) dst_curs.execute(q) s = skytools.TableStruct(src_curs, src_dest_table) src_db.commit() # create, using rename logic only when necessary newname = None if src_dest_table != dest_table: newname = dest_table s.create(dst_curs, create_flags, log=self.log, new_table_name=newname) elif not tbl_exists and self.options.skip_non_existing: self.log.warning('Table %s does not exist on local node, skipping', desc) return tgargs = self.build_tgargs() attrs = {} if self.options.handler: attrs['handler'] = self.build_handler(tbl, tgargs, self.options.dest_table) if self.options.find_copy_node: attrs['copy_node'] = '?' elif self.options.copy_node: attrs['copy_node'] = self.options.copy_node if not self.options.expect_sync: if self.options.skip_truncate: attrs['skip_truncate'] = 1 if self.options.max_parallel_copy: attrs['max_parallel_copy'] = self.options.max_parallel_copy # actual table registration args = [self.set_name, tbl, tgargs, None, None] if attrs: args[3] = skytools.db_urlencode(attrs) if dest_table != tbl: args[4] = dest_table q = "select * from londiste.local_add_table(%s, %s, %s, %s, %s)" self.exec_cmd(dst_curs, q, args) dst_db.commit()
def add_table(self, src_db, dst_db, tbl, create_flags): src_curs = src_db.cursor() dst_curs = dst_db.cursor() tbl_exists = skytools.exists_table(dst_curs, tbl) if create_flags: if tbl_exists: self.log.info('Table %s already exist, not touching' % tbl) else: if not skytools.exists_table(src_curs, tbl): # table not present on provider - nowhere to get the DDL from self.log.warning('Table "%s" missing on provider, skipping' % tbl) return schema = skytools.fq_name_parts(tbl)[0] if not skytools.exists_schema(dst_curs, schema): q = "create schema %s" % skytools.quote_ident(schema) dst_curs.execute(q) s = skytools.TableStruct(src_curs, tbl) src_db.commit() s.create(dst_curs, create_flags, log = self.log) tgargs = [] if self.options.trigger_arg: tgargs = self.options.trigger_arg tgflags = self.options.trigger_flags if tgflags: tgargs.append('tgflags='+tgflags) if self.options.no_triggers: tgargs.append('no_triggers') if self.options.merge_all: tgargs.append('merge_all') if self.options.no_merge: tgargs.append('no_merge') attrs = {} if self.options.handler: hstr = londiste.handler.create_handler_string( self.options.handler, self.options.handler_arg) p = londiste.handler.build_handler(tbl, hstr, self.log) attrs['handler'] = hstr p.add(tgargs) if self.options.expect_sync: tgargs.append('expect_sync') if not self.options.expect_sync: if self.options.skip_truncate: attrs['skip_truncate'] = 1 if self.options.copy_condition: attrs['copy_condition'] = self.options.copy_condition if self.options.max_parallel_copy: attrs['max_parallel_copy'] = self.options.max_parallel_copy args = [self.set_name, tbl, tgargs] if attrs: args.append(skytools.db_urlencode(attrs)) q = "select * from londiste.local_add_table(%s)" %\ ','.join(['%s']*len(args)) # actual table registration self.exec_cmd(dst_curs, q, args) dst_db.commit()
def add_table(self, src_db, dst_db, tbl, create_flags, src_tbls): # use full names tbl = skytools.fq_name(tbl) dest_table = self.options.dest_table or tbl dest_table = skytools.fq_name(dest_table) src_curs = src_db.cursor() dst_curs = dst_db.cursor() tbl_exists = skytools.exists_table(dst_curs, dest_table) if dest_table == tbl: desc = tbl else: desc = "%s(%s)" % (tbl, dest_table) if create_flags: if tbl_exists: self.log.info('Table %s already exist, not touching' % desc) else: src_dest_table = src_tbls[tbl]['dest_table'] if not skytools.exists_table(src_curs, src_dest_table): # table not present on provider - nowhere to get the DDL from self.log.warning('Table %s missing on provider, cannot create, skipping' % desc) return schema = skytools.fq_name_parts(dest_table)[0] if not skytools.exists_schema(dst_curs, schema): q = "create schema %s" % skytools.quote_ident(schema) dst_curs.execute(q) s = skytools.TableStruct(src_curs, src_dest_table) src_db.commit() # create, using rename logic only when necessary newname = None if src_dest_table != dest_table: newname = dest_table s.create(dst_curs, create_flags, log = self.log, new_table_name = newname) tgargs = [] if self.options.trigger_arg: tgargs = self.options.trigger_arg tgflags = self.options.trigger_flags if tgflags: tgargs.append('tgflags='+tgflags) if self.options.no_triggers: tgargs.append('no_triggers') if self.options.merge_all: tgargs.append('merge_all') if self.options.no_merge: tgargs.append('no_merge') attrs = {} if self.options.handler: hstr = londiste.handler.create_handler_string( self.options.handler, self.options.handler_arg) p = londiste.handler.build_handler(tbl, hstr, self.options.dest_table) attrs['handler'] = hstr p.add(tgargs) if self.options.copy_node: attrs['copy_node'] = self.options.copy_node if self.options.expect_sync: tgargs.append('expect_sync') if not self.options.expect_sync: if self.options.skip_truncate: attrs['skip_truncate'] = 1 if self.options.max_parallel_copy: attrs['max_parallel_copy'] = self.options.max_parallel_copy # actual table registration args = [self.set_name, tbl, tgargs, None, None] if attrs: args[3] = skytools.db_urlencode(attrs) if dest_table != tbl: args[4] = dest_table q = "select * from londiste.local_add_table(%s, %s, %s, %s, %s)" self.exec_cmd(dst_curs, q, args) dst_db.commit()
def add_table(self, src_db, dst_db, tbl, create_flags): src_curs = src_db.cursor() dst_curs = dst_db.cursor() tbl_exists = skytools.exists_table(dst_curs, tbl) if create_flags: if tbl_exists: self.log.info("Table %s already exist, not touching" % tbl) else: if not skytools.exists_table(src_curs, tbl): # table not present on provider - nowhere to get the DDL from self.log.warning('Table "%s" missing on provider, skipping' % tbl) return schema = skytools.fq_name_parts(tbl)[0] if not skytools.exists_schema(dst_curs, schema): q = "create schema %s" % skytools.quote_ident(schema) dst_curs.execute(q) s = skytools.TableStruct(src_curs, tbl) src_db.commit() s.create(dst_curs, create_flags, log=self.log) tgargs = [] if self.options.trigger_arg: tgargs = self.options.trigger_arg tgflags = self.options.trigger_flags if tgflags: tgargs.append("tgflags=" + tgflags) if self.options.no_triggers: tgargs.append("no_triggers") if self.options.merge_all: tgargs.append("merge_all") if self.options.no_merge: tgargs.append("no_merge") attrs = {} if self.options.handler: hstr = londiste.handler.create_handler_string(self.options.handler, self.options.handler_arg) p = londiste.handler.build_handler(tbl, hstr, self.log) attrs["handler"] = hstr p.add(tgargs) if self.options.expect_sync: tgargs.append("expect_sync") if not self.options.expect_sync: if self.options.skip_truncate: attrs["skip_truncate"] = 1 if self.options.copy_condition: attrs["copy_condition"] = self.options.copy_condition if self.options.max_parallel_copy: attrs["max_parallel_copy"] = self.options.max_parallel_copy args = [self.set_name, tbl, tgargs] if attrs: args.append(skytools.db_urlencode(attrs)) q = "select * from londiste.local_add_table(%s)" % ",".join(["%s"] * len(args)) # actual table registration self.exec_cmd(dst_curs, q, args) dst_db.commit()