def save_stats (self): """ Dump client stats to database. Scheduled to be called periodically. """ # do not send stats if stats is missing or stats queue is missing if not self.client_stats or not self.stat_queue_name: return now = time.time() time_passed = now - self.last_stat_dump self.log.info ("Sending usage stats to repository [%i]", len(self.client_stats)) # post role usage usage = [] for client in self.client_stats.values(): self.log.trace ("client: %s", client) usage.append (client.to_dict()) params = skytools.db_urlencode(dict( hostname = self.hostname, sample_length = '%d seconds' % time_passed, snap_time = datetime.datetime.now().isoformat())) confdb_funcargs = ('username=discovery', params, skytools.make_record_array(usage)) funcargs = [None, self.stat_queue_name, 'dba.set_role_usage', skytools.db_urlencode(dict(enumerate(confdb_funcargs)))] msg = DatabaseMessage (function = 'pgq.insert_event', params = cc.json.dumps(funcargs)) if self.msg_suffix: msg.req += '.' + self.msg_suffix self.main.ccpublish(msg) self.client_stats = {} self.last_stat_dump = now
def _make_record_convert(row): """Converts complex values.""" d = row.copy() for k, v in d.items(): if isinstance(v, list): d[k] = skytools.make_pgarray(v) return skytools.db_urlencode(d)
def create_handler_string(name, arglist): handler = name if arglist: args = _parse_arglist(arglist) astr = skytools.db_urlencode(args) handler = '%s(%s)' % (handler, astr) return handler
def add_table(self, src_db, dst_db, tbl, create_flags): src_curs = src_db.cursor() dst_curs = dst_db.cursor() tbl_exists = skytools.exists_table(dst_curs, tbl) if create_flags: if tbl_exists: self.log.info('Table %s already exist, not touching' % tbl) else: if not skytools.exists_table(src_curs, tbl): # table not present on provider - nowhere to get the DDL from self.log.warning('Table "%s" missing on provider, skipping' % tbl) return schema = skytools.fq_name_parts(tbl)[0] if not skytools.exists_schema(dst_curs, schema): q = "create schema %s" % skytools.quote_ident(schema) dst_curs.execute(q) s = skytools.TableStruct(src_curs, tbl) src_db.commit() s.create(dst_curs, create_flags, log = self.log) elif not tbl_exists: self.log.warning('Table "%s" missing on subscriber, use --create if necessary' % tbl) return # actual table registration q = "select * from londiste.local_add_table(%s, %s)" self.exec_cmd(dst_curs, q, [self.set_name, tbl]) if self.options.expect_sync: q = "select * from londiste.local_set_table_state(%s, %s, NULL, 'ok')" self.exec_cmd(dst_curs, q, [self.set_name, tbl]) if self.options.copy_condition: q = "select * from londiste.local_set_table_attrs(%s, %s, %s)" attrs = {'copy_condition': self.options.copy_condition} enc_attrs = skytools.db_urlencode(attrs) self.exec_cmd(dst_curs, q, [self.set_name, tbl, enc_attrs]) dst_db.commit()
def add_table(self, src_db, dst_db, tbl, create_flags): src_curs = src_db.cursor() dst_curs = dst_db.cursor() tbl_exists = skytools.exists_table(dst_curs, tbl) if create_flags: if tbl_exists: self.log.info('Table %s already exist, not touching' % tbl) else: if not skytools.exists_table(src_curs, tbl): # table not present on provider - nowhere to get the DDL from self.log.warning('Table "%s" missing on provider, skipping' % tbl) return schema = skytools.fq_name_parts(tbl)[0] if not skytools.exists_schema(dst_curs, schema): q = "create schema %s" % skytools.quote_ident(schema) dst_curs.execute(q) s = skytools.TableStruct(src_curs, tbl) src_db.commit() s.create(dst_curs, create_flags, log = self.log) elif not tbl_exists: self.log.warning('Table "%s" missing on subscriber, use --create if necessary' % tbl) return tgargs = [] if self.options.trigger_arg: tgargs = self.options.trigger_arg tgflags = self.options.trigger_flags if tgflags: tgargs.append('tgflags='+tgflags) if self.options.no_triggers: tgargs.append('no_triggers') if self.options.merge_all: tgargs.append('merge_all') attrs = {} if self.options.handler: hstr = londiste.handler.create_handler_string( self.options.handler, self.options.handler_arg) p = londiste.handler.build_handler(tbl, hstr, self.log) attrs['handler'] = hstr p.add(tgargs) # actual table registration q = "select * from londiste.local_add_table(%s, %s, %s)" self.exec_cmd(dst_curs, q, [self.set_name, tbl, tgargs]) if self.options.expect_sync: q = "select * from londiste.local_set_table_state(%s, %s, NULL, 'ok')" self.exec_cmd(dst_curs, q, [self.set_name, tbl]) else: if self.options.skip_truncate: attrs['skip_truncate'] = 1 if self.options.copy_condition: attrs['copy_condition'] = self.options.copy_condition if attrs: enc_attrs = skytools.db_urlencode(attrs) q = "select * from londiste.local_set_table_attrs(%s, %s, %s)" self.exec_cmd(dst_curs, q, [self.set_name, tbl, enc_attrs]) dst_db.commit()
def make_record(row): """ Takes record as dict and returns it as urlencoded string. Used to send data out of db service layer.or to fake incoming calls """ for v in row.values(): if isinstance(v, list): return _make_record_convert(row) return skytools.db_urlencode(row)
def create_handler_string(name, arglist): handler = name if name.find('(') >= 0: raise Exception('invalid handler name: %s' % name) if arglist: args = _parse_arglist(arglist) astr = skytools.db_urlencode(args) handler = '%s(%s)' % (handler, astr) return handler
def add_table(self, src_db, dst_db, tbl, create_flags): src_curs = src_db.cursor() dst_curs = dst_db.cursor() tbl_exists = skytools.exists_table(dst_curs, tbl) if create_flags: if tbl_exists: self.log.info('Table %s already exist, not touching' % tbl) else: if not skytools.exists_table(src_curs, tbl): # table not present on provider - nowhere to get the DDL from self.log.warning('Table "%s" missing on provider, skipping' % tbl) return schema = skytools.fq_name_parts(tbl)[0] if not skytools.exists_schema(dst_curs, schema): q = "create schema %s" % skytools.quote_ident(schema) dst_curs.execute(q) s = skytools.TableStruct(src_curs, tbl) src_db.commit() s.create(dst_curs, create_flags, log = self.log) elif not tbl_exists: self.log.warning('Table "%s" missing on subscriber, use --create if necessary' % tbl) return tgargs = [] if self.options.trigger_arg: tgargs = self.options.trigger_arg tgflags = self.options.trigger_flags if tgflags: tgargs.append('tgflags='+tgflags) if self.options.no_triggers: tgargs.append('no_triggers') attrs = {} if self.options.handler: hstr = londiste.handler.create_handler_string( self.options.handler, self.options.handler_arg) p = londiste.handler.build_handler(tbl, hstr, self.log) attrs['handler'] = hstr p.add(tgargs) # actual table registration q = "select * from londiste.local_add_table(%s, %s, %s)" self.exec_cmd(dst_curs, q, [self.set_name, tbl, tgargs]) if self.options.expect_sync: q = "select * from londiste.local_set_table_state(%s, %s, NULL, 'ok')" self.exec_cmd(dst_curs, q, [self.set_name, tbl]) else: if self.options.skip_truncate: attrs['skip_truncate'] = 1 if self.options.copy_condition: attrs['copy_condition'] = self.options.copy_condition if attrs: enc_attrs = skytools.db_urlencode(attrs) q = "select * from londiste.local_set_table_attrs(%s, %s, %s)" self.exec_cmd(dst_curs, q, [self.set_name, tbl, enc_attrs]) dst_db.commit()
def __init__(self, table_name, args, log): """Init per-batch table data cache.""" conf = args.copy() # remove Multimaster args from conf for name in ['func_name','func_conf']: if name in conf: conf.pop(name) conf = skytools.db_urlencode(conf) args = update(args, {'func_name': 'merge_on_time', 'func_conf': conf}) ApplyFuncHandler.__init__(self, table_name, args, log)
def __init__(self, table_name, args, dest_table): """Init per-batch table data cache.""" conf = args.copy() # remove Multimaster args from conf for name in ['func_name', 'func_conf']: if name in conf: conf.pop(name) conf = skytools.db_urlencode(conf) args = update(args, {'func_name': 'merge_on_time', 'func_conf': conf}) ApplyFuncHandler.__init__(self, table_name, args, dest_table)
def save_stats(self): """ Dump client stats to database. Scheduled to be called periodically. """ # do not send stats if stats is missing or stats queue is missing if not self.client_stats or not self.stat_queue_name: return now = time.time() time_passed = now - self.last_stat_dump self.log.info("Sending usage stats to repository [%i]", len(self.client_stats)) # post role usage usage = [] for client in self.client_stats.values(): self.log.trace("client: %s", client) usage.append(client.to_dict()) params = skytools.db_urlencode( dict(hostname=self.hostname, sample_length='%d seconds' % time_passed, snap_time=datetime.datetime.now().isoformat())) confdb_funcargs = ('username=discovery', params, skytools.make_record_array(usage)) funcargs = [ None, self.stat_queue_name, 'dba.set_role_usage', skytools.db_urlencode(dict(enumerate(confdb_funcargs))) ] msg = DatabaseMessage(function='pgq.insert_event', params=cc.json.dumps(funcargs)) if self.msg_suffix: msg.req += '.' + self.msg_suffix self.main.ccpublish(msg) self.client_stats = {} self.last_stat_dump = now
def cmd_resync(self, *args): """Reload data from provider node.""" db = self.get_database('db') args = self.expand_arg_list(db, 'r', True, args) if not self.options.find_copy_node: self.load_local_info() src_db = self.get_provider_db() src_curs = src_db.cursor() src_tbls = self.fetch_set_tables(src_curs) src_db.commit() problems = 0 for tbl in args: tbl = skytools.fq_name(tbl) if tbl not in src_tbls or not src_tbls[tbl]['local']: self.log.error( "Table %s does not exist on provider, need to switch to different provider", tbl) problems += 1 if problems > 0: self.log.error("Problems, cancelling operation") sys.exit(1) if self.options.find_copy_node or self.options.copy_node: q = "select table_name, table_attrs from londiste.get_table_list(%s) where local" cur = db.cursor() cur.execute(q, [self.set_name]) for row in cur.fetchall(): if row['table_name'] not in args: continue attrs = skytools.db_urldecode(row['table_attrs'] or '') if self.options.find_copy_node: attrs['copy_node'] = '?' elif self.options.copy_node: attrs['copy_node'] = self.options.copy_node s_attrs = skytools.db_urlencode(attrs) q = "select * from londiste.local_set_table_attrs(%s, %s, %s)" self.exec_cmd(db, q, [self.set_name, row['table_name'], s_attrs]) q = "select * from londiste.local_set_table_state(%s, %s, null, null)" self.exec_cmd_many(db, q, [self.set_name], args)
def cmd_change_handler(self, tbl): """Change handler (table_attrs) of the replicated table.""" self.load_local_info() tbl = skytools.fq_name(tbl) db = self.get_database('db') curs = db.cursor() q = "select table_attrs, dest_table "\ " from londiste.get_table_list(%s) "\ " where table_name = %s and local" curs.execute(q, [self.set_name, tbl]) if curs.rowcount == 0: self.log.error("Table %s not found on this node", tbl) sys.exit(1) attrs, dest_table = curs.fetchone() attrs = skytools.db_urldecode(attrs or '') old_handler = attrs.get('handler') tgargs = self.build_tgargs() if self.options.handler: new_handler = self.build_handler(tbl, tgargs, dest_table) else: new_handler = None if old_handler == new_handler: self.log.info( "Handler is already set to desired value, nothing done") sys.exit(0) if new_handler: attrs['handler'] = new_handler elif 'handler' in attrs: del attrs['handler'] args = [self.set_name, tbl, tgargs, None] if attrs: args[3] = skytools.db_urlencode(attrs) q = "select * from londiste.local_change_handler(%s, %s, %s, %s)" self.exec_cmd(curs, q, args) db.commit()
def cmd_change_handler(self, tbl): """Change handler (table_attrs) of the replicated table.""" self.load_local_info() tbl = skytools.fq_name(tbl) db = self.get_database('db') curs = db.cursor() q = "select table_attrs, dest_table "\ " from londiste.get_table_list(%s) "\ " where table_name = %s and local" curs.execute(q, [self.set_name, tbl]) if curs.rowcount == 0: self.log.error("Table %s not found on this node", tbl) sys.exit(1) attrs, dest_table = curs.fetchone() attrs = skytools.db_urldecode(attrs or '') old_handler = attrs.get('handler') tgargs = self.build_tgargs() if self.options.handler: new_handler = self.build_handler(tbl, tgargs, dest_table) else: new_handler = None if old_handler == new_handler: self.log.info("Handler is already set to desired value, nothing done") sys.exit(0) if new_handler: attrs['handler'] = new_handler elif 'handler' in attrs: del attrs['handler'] args = [self.set_name, tbl, tgargs, None] if attrs: args[3] = skytools.db_urlencode(attrs) q = "select * from londiste.local_change_handler(%s, %s, %s, %s)" self.exec_cmd(curs, q, args) db.commit()
def cmd_resync(self, *args): """Reload data from provider node.""" db = self.get_database('db') args = self.expand_arg_list(db, 'r', True, args) if not self.options.find_copy_node: self.load_local_info() src_db = self.get_provider_db() src_curs = src_db.cursor() src_tbls = self.fetch_set_tables(src_curs) src_db.commit() problems = 0 for tbl in args: tbl = skytools.fq_name(tbl) if tbl not in src_tbls or not src_tbls[tbl]['local']: self.log.error("Table %s does not exist on provider, need to switch to different provider", tbl) problems += 1 if problems > 0: self.log.error("Problems, cancelling operation") sys.exit(1) if self.options.find_copy_node or self.options.copy_node: q = "select table_name, table_attrs from londiste.get_table_list(%s) where local" cur = db.cursor() cur.execute(q, [self.set_name]) for row in cur.fetchall(): if row['table_name'] not in args: continue attrs = skytools.db_urldecode (row['table_attrs'] or '') if self.options.find_copy_node: attrs['copy_node'] = '?' elif self.options.copy_node: attrs['copy_node'] = self.options.copy_node attrs = skytools.db_urlencode (attrs) q = "select * from londiste.local_set_table_attrs (%s, %s, %s)" self.exec_cmd(db, q, [self.set_name, row['table_name'], attrs]) q = "select * from londiste.local_set_table_state(%s, %s, null, null)" self.exec_cmd_many(db, q, [self.set_name], args)
def make_record(row): """ Takes record as dict and returns it as urlencoded string. Used to send data out of db service layer.or to fake incoming calls """ return skytools.db_urlencode(row)
def add_table(self, src_db, dst_db, tbl, create_flags): src_curs = src_db.cursor() dst_curs = dst_db.cursor() tbl_exists = skytools.exists_table(dst_curs, tbl) if create_flags: if tbl_exists: self.log.info('Table %s already exist, not touching' % tbl) else: if not skytools.exists_table(src_curs, tbl): # table not present on provider - nowhere to get the DDL from self.log.warning('Table "%s" missing on provider, skipping' % tbl) return schema = skytools.fq_name_parts(tbl)[0] if not skytools.exists_schema(dst_curs, schema): q = "create schema %s" % skytools.quote_ident(schema) dst_curs.execute(q) s = skytools.TableStruct(src_curs, tbl) src_db.commit() s.create(dst_curs, create_flags, log = self.log) tgargs = [] if self.options.trigger_arg: tgargs = self.options.trigger_arg tgflags = self.options.trigger_flags if tgflags: tgargs.append('tgflags='+tgflags) if self.options.no_triggers: tgargs.append('no_triggers') if self.options.merge_all: tgargs.append('merge_all') if self.options.no_merge: tgargs.append('no_merge') attrs = {} if self.options.handler: hstr = londiste.handler.create_handler_string( self.options.handler, self.options.handler_arg) p = londiste.handler.build_handler(tbl, hstr, self.log) attrs['handler'] = hstr p.add(tgargs) if self.options.expect_sync: tgargs.append('expect_sync') if not self.options.expect_sync: if self.options.skip_truncate: attrs['skip_truncate'] = 1 if self.options.copy_condition: attrs['copy_condition'] = self.options.copy_condition if self.options.max_parallel_copy: attrs['max_parallel_copy'] = self.options.max_parallel_copy args = [self.set_name, tbl, tgargs] if attrs: args.append(skytools.db_urlencode(attrs)) q = "select * from londiste.local_add_table(%s)" %\ ','.join(['%s']*len(args)) # actual table registration self.exec_cmd(dst_curs, q, args) dst_db.commit()
def create_node(self, node_type, node_name, node_location): """Generic node init.""" provider_loc = self.options.provider if node_type not in ('root', 'branch', 'leaf'): raise Exception('unknown node type') # connect to database db = self.get_database("new_node", connstr=node_location) # check if code is installed self.install_code(db) # query current status res = self.exec_query(db, "select * from pgq_node.get_node_info(%s)", [self.queue_name]) info = res[0] if info['node_type'] is not None: self.log.info("Node is already initialized as %s" % info['node_type']) return self.log.info("Initializing node") node_attrs = {} worker_name = self.options.worker if not worker_name: raise Exception('--worker required') combined_queue = self.options.merge if combined_queue and node_type != 'leaf': raise Exception('--merge can be used only for leafs') if self.options.sync_watermark: if node_type != 'branch': raise UsageError( '--sync-watermark can be used only for branch nodes') node_attrs['sync_watermark'] = self.options.sync_watermark # register member if node_type == 'root': global_watermark = None combined_queue = None provider_name = None self.exec_cmd( db, "select * from pgq_node.register_location(%s, %s, %s, false)", [self.queue_name, node_name, node_location]) self.exec_cmd( db, "select * from pgq_node.create_node(%s, %s, %s, %s, %s, %s, %s)", [ self.queue_name, node_type, node_name, worker_name, provider_name, global_watermark, combined_queue ]) provider_db = None else: if not provider_loc: raise Exception('Please specify --provider') root_db = self.find_root_db(provider_loc) queue_info = self.load_queue_info(root_db) # check if member already exists if queue_info.get_member(node_name) is not None: self.log.error("Node '%s' already exists" % node_name) sys.exit(1) combined_set = None provider_db = self.get_database('provider_db', connstr=provider_loc) q = "select node_type, node_name from pgq_node.get_node_info(%s)" res = self.exec_query(provider_db, q, [self.queue_name]) row = res[0] if not row['node_name']: raise Exception("provider node not found") provider_name = row['node_name'] # register member on root self.exec_cmd( root_db, "select * from pgq_node.register_location(%s, %s, %s, false)", [self.queue_name, node_name, node_location]) # lookup provider provider = queue_info.get_member(provider_name) if not provider: self.log.error("Node %s does not exist" % provider_name) sys.exit(1) # register on provider self.exec_cmd( provider_db, "select * from pgq_node.register_location(%s, %s, %s, false)", [self.queue_name, node_name, node_location]) rows = self.exec_cmd( provider_db, "select * from pgq_node.register_subscriber(%s, %s, %s, null)", [self.queue_name, node_name, worker_name]) global_watermark = rows[0]['global_watermark'] # initialize node itself # insert members self.exec_cmd( db, "select * from pgq_node.register_location(%s, %s, %s, false)", [self.queue_name, node_name, node_location]) for m in queue_info.member_map.values(): self.exec_cmd( db, "select * from pgq_node.register_location(%s, %s, %s, %s)", [self.queue_name, m.name, m.location, m.dead]) # real init self.exec_cmd( db, "select * from pgq_node.create_node(%s, %s, %s, %s, %s, %s, %s)", [ self.queue_name, node_type, node_name, worker_name, provider_name, global_watermark, combined_queue ]) self.extra_init(node_type, db, provider_db) if node_attrs: s_attrs = skytools.db_urlencode(node_attrs) self.exec_cmd(db, "select * from pgq_node.set_node_attrs(%s, %s)", [self.queue_name, s_attrs]) self.log.info("Done")
def to_urlenc(self): """Convert container to urlencoded string.""" sdict = {} for k, v in self.attrs.items(): sdict[k] = ','.join(v) return skytools.db_urlencode(sdict)
def add_table(self, src_db, dst_db, tbl, create_flags, src_tbls): # use full names tbl = skytools.fq_name(tbl) dest_table = self.options.dest_table or tbl dest_table = skytools.fq_name(dest_table) src_curs = src_db.cursor() dst_curs = dst_db.cursor() tbl_exists = skytools.exists_table(dst_curs, dest_table) if dest_table == tbl: desc = tbl else: desc = "%s(%s)" % (tbl, dest_table) if create_flags: if tbl_exists: self.log.info('Table %s already exist, not touching' % desc) else: src_dest_table = src_tbls[tbl]['dest_table'] if not skytools.exists_table(src_curs, src_dest_table): # table not present on provider - nowhere to get the DDL from self.log.warning('Table %s missing on provider, cannot create, skipping' % desc) return schema = skytools.fq_name_parts(dest_table)[0] if not skytools.exists_schema(dst_curs, schema): q = "create schema %s" % skytools.quote_ident(schema) dst_curs.execute(q) s = skytools.TableStruct(src_curs, src_dest_table) src_db.commit() # create, using rename logic only when necessary newname = None if src_dest_table != dest_table: newname = dest_table s.create(dst_curs, create_flags, log = self.log, new_table_name = newname) tgargs = [] if self.options.trigger_arg: tgargs = self.options.trigger_arg tgflags = self.options.trigger_flags if tgflags: tgargs.append('tgflags='+tgflags) if self.options.no_triggers: tgargs.append('no_triggers') if self.options.merge_all: tgargs.append('merge_all') if self.options.no_merge: tgargs.append('no_merge') attrs = {} if self.options.handler: hstr = londiste.handler.create_handler_string( self.options.handler, self.options.handler_arg) p = londiste.handler.build_handler(tbl, hstr, self.options.dest_table) attrs['handler'] = hstr p.add(tgargs) if self.options.copy_node: attrs['copy_node'] = self.options.copy_node if self.options.expect_sync: tgargs.append('expect_sync') if not self.options.expect_sync: if self.options.skip_truncate: attrs['skip_truncate'] = 1 if self.options.max_parallel_copy: attrs['max_parallel_copy'] = self.options.max_parallel_copy # actual table registration args = [self.set_name, tbl, tgargs, None, None] if attrs: args[3] = skytools.db_urlencode(attrs) if dest_table != tbl: args[4] = dest_table q = "select * from londiste.local_add_table(%s, %s, %s, %s, %s)" self.exec_cmd(dst_curs, q, args) dst_db.commit()
def add_table(self, src_db, dst_db, tbl, create_flags, src_tbls): # use full names tbl = skytools.fq_name(tbl) dest_table = self.options.dest_table or tbl dest_table = skytools.fq_name(dest_table) src_curs = src_db.cursor() dst_curs = dst_db.cursor() tbl_exists = skytools.exists_table(dst_curs, dest_table) if dest_table == tbl: desc = tbl else: desc = "%s(%s)" % (tbl, dest_table) if create_flags: if tbl_exists: self.log.info('Table %s already exist, not touching' % desc) else: src_dest_table = src_tbls[tbl]['dest_table'] if not skytools.exists_table(src_curs, src_dest_table): # table not present on provider - nowhere to get the DDL from self.log.warning('Table %s missing on provider, cannot create, skipping' % desc) return schema = skytools.fq_name_parts(dest_table)[0] if not skytools.exists_schema(dst_curs, schema): q = "create schema %s" % skytools.quote_ident(schema) dst_curs.execute(q) s = skytools.TableStruct(src_curs, src_dest_table) src_db.commit() # create, using rename logic only when necessary newname = None if src_dest_table != dest_table: newname = dest_table s.create(dst_curs, create_flags, log = self.log, new_table_name = newname) tgargs = [] if self.options.trigger_arg: tgargs = self.options.trigger_arg tgflags = self.options.trigger_flags if tgflags: tgargs.append('tgflags='+tgflags) if self.options.no_triggers: tgargs.append('no_triggers') if self.options.merge_all: tgargs.append('merge_all') if self.options.no_merge: tgargs.append('no_merge') attrs = {} if self.options.handler: hstr = londiste.handler.create_handler_string( self.options.handler, self.options.handler_arg) p = londiste.handler.build_handler(tbl, hstr, self.options.dest_table) attrs['handler'] = hstr p.add(tgargs) if self.options.find_copy_node: attrs['copy_node'] = '?' elif self.options.copy_node: attrs['copy_node'] = self.options.copy_node if self.options.expect_sync: tgargs.append('expect_sync') if not self.options.expect_sync: if self.options.skip_truncate: attrs['skip_truncate'] = 1 if self.options.max_parallel_copy: attrs['max_parallel_copy'] = self.options.max_parallel_copy # actual table registration args = [self.set_name, tbl, tgargs, None, None] if attrs: args[3] = skytools.db_urlencode(attrs) if dest_table != tbl: args[4] = dest_table q = "select * from londiste.local_add_table(%s, %s, %s, %s, %s)" self.exec_cmd(dst_curs, q, args) dst_db.commit()
def add_table(self, src_db, dst_db, tbl, create_flags, src_tbls): # use full names tbl = skytools.fq_name(tbl) dest_table = self.options.dest_table or tbl dest_table = skytools.fq_name(dest_table) src_curs = src_db.cursor() dst_curs = dst_db.cursor() tbl_exists = skytools.exists_table(dst_curs, dest_table) dst_db.commit() self.set_lock_timeout(dst_curs) if dest_table == tbl: desc = tbl else: desc = "%s(%s)" % (tbl, dest_table) if create_flags: if tbl_exists: self.log.info('Table %s already exist, not touching', desc) else: src_dest_table = src_tbls[tbl]['dest_table'] if not skytools.exists_table(src_curs, src_dest_table): # table not present on provider - nowhere to get the DDL from self.log.warning( 'Table %s missing on provider, cannot create, skipping', desc) return schema = skytools.fq_name_parts(dest_table)[0] if not skytools.exists_schema(dst_curs, schema): q = "create schema %s" % skytools.quote_ident(schema) dst_curs.execute(q) s = skytools.TableStruct(src_curs, src_dest_table) src_db.commit() # create, using rename logic only when necessary newname = None if src_dest_table != dest_table: newname = dest_table s.create(dst_curs, create_flags, log=self.log, new_table_name=newname) elif not tbl_exists and self.options.skip_non_existing: self.log.warning('Table %s does not exist on local node, skipping', desc) return tgargs = self.build_tgargs() attrs = {} if self.options.handler: attrs['handler'] = self.build_handler(tbl, tgargs, self.options.dest_table) if self.options.find_copy_node: attrs['copy_node'] = '?' elif self.options.copy_node: attrs['copy_node'] = self.options.copy_node if not self.options.expect_sync: if self.options.skip_truncate: attrs['skip_truncate'] = 1 if self.options.max_parallel_copy: attrs['max_parallel_copy'] = self.options.max_parallel_copy # actual table registration args = [self.set_name, tbl, tgargs, None, None] if attrs: args[3] = skytools.db_urlencode(attrs) if dest_table != tbl: args[4] = dest_table q = "select * from londiste.local_add_table(%s, %s, %s, %s, %s)" self.exec_cmd(dst_curs, q, args) dst_db.commit()
def add_table(self, src_db, dst_db, tbl, create_flags): src_curs = src_db.cursor() dst_curs = dst_db.cursor() tbl_exists = skytools.exists_table(dst_curs, tbl) if create_flags: if tbl_exists: self.log.info("Table %s already exist, not touching" % tbl) else: if not skytools.exists_table(src_curs, tbl): # table not present on provider - nowhere to get the DDL from self.log.warning('Table "%s" missing on provider, skipping' % tbl) return schema = skytools.fq_name_parts(tbl)[0] if not skytools.exists_schema(dst_curs, schema): q = "create schema %s" % skytools.quote_ident(schema) dst_curs.execute(q) s = skytools.TableStruct(src_curs, tbl) src_db.commit() s.create(dst_curs, create_flags, log=self.log) tgargs = [] if self.options.trigger_arg: tgargs = self.options.trigger_arg tgflags = self.options.trigger_flags if tgflags: tgargs.append("tgflags=" + tgflags) if self.options.no_triggers: tgargs.append("no_triggers") if self.options.merge_all: tgargs.append("merge_all") if self.options.no_merge: tgargs.append("no_merge") attrs = {} if self.options.handler: hstr = londiste.handler.create_handler_string(self.options.handler, self.options.handler_arg) p = londiste.handler.build_handler(tbl, hstr, self.log) attrs["handler"] = hstr p.add(tgargs) if self.options.expect_sync: tgargs.append("expect_sync") if not self.options.expect_sync: if self.options.skip_truncate: attrs["skip_truncate"] = 1 if self.options.copy_condition: attrs["copy_condition"] = self.options.copy_condition if self.options.max_parallel_copy: attrs["max_parallel_copy"] = self.options.max_parallel_copy args = [self.set_name, tbl, tgargs] if attrs: args.append(skytools.db_urlencode(attrs)) q = "select * from londiste.local_add_table(%s)" % ",".join(["%s"] * len(args)) # actual table registration self.exec_cmd(dst_curs, q, args) dst_db.commit()
def create_node(self, node_type, node_name, node_location): """Generic node init.""" provider_loc = self.options.provider if node_type not in ('root', 'branch', 'leaf'): raise Exception('unknown node type') # connect to database db = self.get_database("new_node", connstr = node_location) # check if code is installed self.install_code(db) # query current status res = self.exec_query(db, "select * from pgq_node.get_node_info(%s)", [self.queue_name]) info = res[0] if info['node_type'] is not None: self.log.info("Node is already initialized as %s" % info['node_type']) return self.log.info("Initializing node") node_attrs = {} worker_name = self.options.worker if not worker_name: raise Exception('--worker required') combined_queue = self.options.merge if combined_queue and node_type != 'leaf': raise Exception('--merge can be used only for leafs') if self.options.sync_watermark: if node_type != 'branch': raise UsageError('--sync-watermark can be used only for branch nodes') node_attrs['sync_watermark'] = self.options.sync_watermark # register member if node_type == 'root': global_watermark = None combined_queue = None provider_name = None self.exec_cmd(db, "select * from pgq_node.register_location(%s, %s, %s, false)", [self.queue_name, node_name, node_location]) self.exec_cmd(db, "select * from pgq_node.create_node(%s, %s, %s, %s, %s, %s, %s)", [self.queue_name, node_type, node_name, worker_name, provider_name, global_watermark, combined_queue]) provider_db = None else: if not provider_loc: raise Exception('Please specify --provider') root_db = self.find_root_db(provider_loc) queue_info = self.load_queue_info(root_db) # check if member already exists if queue_info.get_member(node_name) is not None: self.log.error("Node '%s' already exists" % node_name) sys.exit(1) combined_set = None provider_db = self.get_database('provider_db', connstr = provider_loc) q = "select node_type, node_name from pgq_node.get_node_info(%s)" res = self.exec_query(provider_db, q, [self.queue_name]) row = res[0] if not row['node_name']: raise Exception("provider node not found") provider_name = row['node_name'] # register member on root self.exec_cmd(root_db, "select * from pgq_node.register_location(%s, %s, %s, false)", [self.queue_name, node_name, node_location]) # lookup provider provider = queue_info.get_member(provider_name) if not provider: self.log.error("Node %s does not exist" % provider_name) sys.exit(1) # register on provider self.exec_cmd(provider_db, "select * from pgq_node.register_location(%s, %s, %s, false)", [self.queue_name, node_name, node_location]) rows = self.exec_cmd(provider_db, "select * from pgq_node.register_subscriber(%s, %s, %s, null)", [self.queue_name, node_name, worker_name]) global_watermark = rows[0]['global_watermark'] # initialize node itself # insert members self.exec_cmd(db, "select * from pgq_node.register_location(%s, %s, %s, false)", [self.queue_name, node_name, node_location]) for m in queue_info.member_map.values(): self.exec_cmd(db, "select * from pgq_node.register_location(%s, %s, %s, %s)", [self.queue_name, m.name, m.location, m.dead]) # real init self.exec_cmd(db, "select * from pgq_node.create_node(%s, %s, %s, %s, %s, %s, %s)", [ self.queue_name, node_type, node_name, worker_name, provider_name, global_watermark, combined_queue ]) self.extra_init(node_type, db, provider_db) if node_attrs: s_attrs = skytools.db_urlencode(node_attrs) self.exec_cmd(db, "select * from pgq_node.set_node_attrs(%s, %s)", [self.queue_name, s_attrs]) self.log.info("Done")