def work(self): """Syncer main function.""" # 'SELECT 1' and COPY must use same snapshot, so change isolation level. dst_db = self.get_database('db', isolation_level = skytools.I_REPEATABLE_READ) pnode, ploc = self.get_provider_location(dst_db) dst_tables, names = self.get_tables(dst_db) if len(self.args) > 2: tlist = self.args[2:] else: tlist = names for tbl in tlist: tbl = skytools.fq_name(tbl) if not tbl in dst_tables: self.log.warning('Table not subscribed: %s' % tbl) continue t2 = dst_tables[tbl] if t2.merge_state != 'ok': self.log.warning('Table %s not synced yet, no point' % tbl) continue pnode, ploc, wname = find_copy_source(self, self.queue_name, tbl, pnode, ploc) self.log.info('%s: Using node %s as provider', tbl, pnode) if wname is None: wname = self.consumer_name self.downstream_worker_name = wname self.process_one_table(tbl, t2, dst_db, pnode, ploc) # signal caller about bad tables sys.exit(self.bad_tables)
def register_copy_consumer(self): dst_db = self.get_database('db') dst_curs = dst_db.cursor() # fetch table attrs q = "select * from londiste.get_table_list(%s) where table_name = %s" dst_curs.execute(q, [self.queue_name, self.copy_table_name]) rows = dst_curs.fetchall() attrs = {} if len(rows) > 0: v_attrs = rows[0]['table_attrs'] if v_attrs: attrs = skytools.db_urldecode(v_attrs) # fetch parent consumer state q = "select * from pgq_node.get_consumer_state(%s, %s)" rows = self.exec_cmd(dst_db, q, [self.queue_name, self.old_consumer_name]) state = rows[0] source_node = state['provider_node'] source_location = state['provider_location'] # do we have node here? if 'copy_node' in attrs: if attrs['copy_node'] == '?': source_node, source_location, wname = find_copy_source( self, self.queue_name, self.copy_table_name, source_node, source_location) else: # take node from attrs source_node = attrs['copy_node'] q = "select * from pgq_node.get_queue_locations(%s) where node_name = %s" dst_curs.execute(q, [self.queue_name, source_node]) rows = dst_curs.fetchall() if len(rows): source_location = rows[0]['node_location'] self.log.info("Using '%s' as source node", source_node) self.register_consumer(source_location)
def register_copy_consumer(self): dst_db = self.get_database('db') dst_curs = dst_db.cursor() # fetch table attrs q = "select * from londiste.get_table_list(%s) where table_name = %s" dst_curs.execute(q, [self.queue_name, self.copy_table_name]) rows = dst_curs.fetchall() attrs = {} if len(rows) > 0: v_attrs = rows[0]['table_attrs'] if v_attrs: attrs = skytools.db_urldecode(v_attrs) # fetch parent consumer state q = "select * from pgq_node.get_consumer_state(%s, %s)" rows = self.exec_cmd(dst_db, q, [self.queue_name, self.old_consumer_name]) state = rows[0] source_node = state['provider_node'] source_location = state['provider_location'] # do we have node here? if 'copy_node' in attrs: if attrs['copy_node'] == '?': source_node, source_location, ___wname = find_copy_source(self, self.queue_name, self.copy_table_name, source_node, source_location) else: # take node from attrs source_node = attrs['copy_node'] q = "select * from pgq_node.get_queue_locations(%s) where node_name = %s" dst_curs.execute(q, [self.queue_name, source_node]) rows = dst_curs.fetchall() if len(rows): source_location = rows[0]['node_location'] self.log.info("Using '%s' as source node", source_node) self.register_consumer(source_location)
def work(self): """Syncer main function.""" # 'SELECT 1' and COPY must use same snapshot, so change isolation level. dst_db = self.get_database('db', isolation_level=skytools.I_REPEATABLE_READ) pnode, ploc = self.get_provider_location(dst_db) dst_tables, names = self.get_tables(dst_db) if len(self.args) > 2: tlist = self.args[2:] else: tlist = names for tbl in tlist: tbl = skytools.fq_name(tbl) if not tbl in dst_tables: self.log.warning('Table not subscribed: %s', tbl) continue t2 = dst_tables[tbl] if t2.merge_state != 'ok': self.log.warning('Table %s not synced yet, no point', tbl) continue pnode, ploc, wname = find_copy_source(self, self.queue_name, tbl, pnode, ploc) self.log.info('%s: Using node %s as provider', tbl, pnode) if wname is None: wname = self.consumer_name self.downstream_worker_name = wname self.process_one_table(tbl, t2, dst_db, pnode, ploc) # signal caller about bad tables sys.exit(self.bad_tables)
def cmd_add_table(self, *args): """Attach table(s) to local node.""" self.load_local_info() src_db = self.get_provider_db() if not self.is_root(): src_curs = src_db.cursor() src_tbls = self.fetch_set_tables(src_curs) src_db.commit() dst_db = self.get_database('db') dst_curs = dst_db.cursor() dst_tbls = self.fetch_set_tables(dst_curs) if self.is_root(): src_tbls = dst_tbls else: self.sync_table_list(dst_curs, src_tbls, dst_tbls) dst_db.commit() needs_tbl = self.handler_needs_table() args = self.expand_arg_list(dst_db, 'r', False, args, needs_tbl) # pick proper create flags if self.options.create_full: create_flags = skytools.T_ALL elif self.options.create: create_flags = skytools.T_TABLE | skytools.T_PKEY else: create_flags = 0 # search for usable copy node if requested & needed if (self.options.find_copy_node and create_flags != 0 and needs_tbl and not self.is_root()): src_name, src_loc, _ = find_copy_source(self, self.queue_name, args, None, self.provider_location) self.options.copy_node = src_name self.close_database('provider_db') src_db = self.get_provider_db() src_curs = src_db.cursor() src_tbls = self.fetch_set_tables(src_curs) src_db.commit() # dont check for exist/not here (root handling) if not self.is_root() and not self.options.expect_sync and not self.options.find_copy_node: problems = False for tbl in args: tbl = skytools.fq_name(tbl) if (tbl in src_tbls) and not src_tbls[tbl]['local']: if self.options.skip_non_existing: self.log.warning("Table %s does not exist on provider", tbl) else: self.log.error("Table %s does not exist on provider, need to switch to different provider", tbl) problems = True if problems: self.log.error("Problems, canceling operation") sys.exit(1) # sanity check if self.options.dest_table and len(args) > 1: self.log.error("--dest-table can be given only for single table") sys.exit(1) # seems ok for tbl in args: self.add_table(src_db, dst_db, tbl, create_flags, src_tbls) # wait if self.options.wait_sync: self.wait_for_sync(dst_db)
def cmd_add_table(self, *args): """Attach table(s) to local node.""" self.load_local_info() src_db = self.get_provider_db() if not self.is_root(): src_curs = src_db.cursor() src_tbls = self.fetch_set_tables(src_curs) src_db.commit() dst_db = self.get_database('db') dst_curs = dst_db.cursor() dst_tbls = self.fetch_set_tables(dst_curs) if self.is_root(): src_tbls = dst_tbls else: self.sync_table_list(dst_curs, src_tbls, dst_tbls) dst_db.commit() needs_tbl = self.handler_needs_table() args = self.expand_arg_list(dst_db, 'r', False, args, needs_tbl) # pick proper create flags if self.options.create_full: create_flags = skytools.T_ALL elif self.options.create: create_flags = skytools.T_TABLE | skytools.T_PKEY else: create_flags = 0 # search for usable copy node if requested & needed if (self.options.find_copy_node and create_flags != 0 and needs_tbl and not self.is_root()): src_name, _, _ = find_copy_source(self, self.queue_name, args, None, self.provider_location) self.options.copy_node = src_name self.close_database('provider_db') src_db = self.get_provider_db() src_curs = src_db.cursor() src_tbls = self.fetch_set_tables(src_curs) src_db.commit() # dont check for exist/not here (root handling) if not self.is_root( ) and not self.options.expect_sync and not self.options.find_copy_node: problems = False for tbl in args: tbl = skytools.fq_name(tbl) if (tbl in src_tbls) and not src_tbls[tbl]['local']: if self.options.skip_non_existing: self.log.warning("Table %s does not exist on provider", tbl) else: self.log.error( "Table %s does not exist on provider, need to switch to different provider", tbl) problems = True if problems: self.log.error("Problems, canceling operation") sys.exit(1) # sanity check if self.options.dest_table and len(args) > 1: self.log.error("--dest-table can be given only for single table") sys.exit(1) # seems ok for tbl in args: self.add_table(src_db, dst_db, tbl, create_flags, src_tbls) # wait if self.options.wait_sync: self.wait_for_sync(dst_db)