def match(self, objname, curs, tables, seqs): nargs = 0 pos1 = objname.find('(') if pos1 > 0: pos2 = objname.find(')') if pos2 > 0: s = objname[pos1+1 : pos2] objname = objname[:pos1] nargs = int(s) return skytools.exists_function(curs, objname, nargs)
def match(self, objname, curs, tables, seqs): nargs = 0 pos1 = objname.find('(') if pos1 > 0: pos2 = objname.find(')') if pos2 > 0: s = objname[pos1 + 1:pos2] objname = objname[:pos1] nargs = int(s) return skytools.exists_function(curs, objname, nargs)
def check_version(curs, schema, new_ver_str, recheck_func=None): funcname = "%s.version" % schema if not skytools.exists_function(curs, funcname, 0): if recheck_func is not None: return recheck_func(curs) else: return 0 q = "select %s()" % funcname curs.execute(q) old_ver_str = curs.fetchone()[0] return is_version_ge(old_ver_str, new_ver_str)
def check_version(curs, schema, new_ver_str, recheck_func=None): funcname = "%s.version" % schema if not skytools.exists_function(curs, funcname, 0): if recheck_func is not None: return recheck_func(curs), 'NULL' else: return 0, 'NULL' q = "select %s()" % funcname curs.execute(q) old_ver_str = curs.fetchone()[0] ok = is_version_ge(old_ver_str, new_ver_str) return ok, old_ver_str
def check_version(curs, schema, new_ver_str, recheck_func=None): funcname = "%s.version" % schema if not skytools.exists_function(curs, funcname, 0): if recheck_func is not None: return recheck_func(curs) else: return 0 q = "select %s()" % funcname curs.execute(q) old_ver_str = curs.fetchone()[0] new_ver = parse_ver(new_ver_str) old_ver = parse_ver(old_ver_str) return old_ver >= new_ver
def check_version(curs, schema, new_ver_str, recheck_func=None, force_gt=False): funcname = "%s.version" % schema if not skytools.exists_function(curs, funcname, 0): if recheck_func is not None: return recheck_func(curs), 'NULL' else: return 0, 'NULL' q = "select %s()" % funcname curs.execute(q) old_ver_str = curs.fetchone()[0] if force_gt: ok = is_version_gt(old_ver_str, new_ver_str) else: ok = is_version_ge(old_ver_str, new_ver_str) return ok, old_ver_str
def check_part(self, dst, part_time): """Create part table if not exists. It part_template present, execute it else if part function present in db, call it else clone master table""" curs = self.dst_curs if skytools.exists_table(curs, dst): return dst = quote_fqident(dst) vals = { 'dest': dst, 'part': dst, 'parent': self.quoted_name, 'pkeys': ",".join(self.pkeys), # quoting? # we do this to make sure that constraints for # tables who contain a schema will still work 'schema_table': dst.replace(".", "__"), 'part_field': self.conf.part_field, 'part_time': part_time, 'period': self.conf.period, } def exec_with_vals(tmpl): if tmpl: sql = tmpl % vals curs.execute(sql) return True return False exec_with_vals(self.conf.pre_part) if not exec_with_vals(self.conf.part_template): self.log.debug('part_template not provided, using part func') # if part func exists call it with val arguments if skytools.exists_function(curs, PART_FUNC, len(PART_FUNC_ARGS)): self.log.debug('check_part.exec: func:%s, args: %s' %\ (PART_FUNC_CALL, vals)) curs.execute(PART_FUNC_CALL, vals) else: self.log.debug('part func %s not found, cloning table' %\ PART_FUNC) struct = TableStruct(curs, self.table_name) struct.create(curs, T_ALL, dst) exec_with_vals(self.conf.post_part) self.log.info("Created table: %s" % dst)
def check_part(self, dst, part_time): """Create part table if not exists. It part_template present, execute it else if part function present in db, call it else clone master table""" curs = self.dst_curs if skytools.exists_table(curs, dst): return dst = quote_fqident(dst) vals = { "dest": dst, "part": dst, "parent": self.quoted_name, "pkeys": ",".join(self.pkeys), # quoting? # we do this to make sure that constraints for # tables who contain a schema will still work "schema_table": dst.replace(".", "__"), "part_field": self.conf.part_field, "part_time": part_time, "period": self.conf.period, } def exec_with_vals(tmpl): if tmpl: sql = tmpl % vals curs.execute(sql) return True return False exec_with_vals(self.conf.pre_part) if not exec_with_vals(self.conf.part_template): self.log.debug("part_template not provided, using part func") # if part func exists call it with val arguments if skytools.exists_function(curs, PART_FUNC, len(PART_FUNC_ARGS)): self.log.debug("check_part.exec: func:%s, args: %s" % (PART_FUNC_CALL, vals)) curs.execute(PART_FUNC_CALL, vals) else: self.log.debug("part func %s not found, cloning table" % PART_FUNC) struct = TableStruct(curs, self.table_name) struct.create(curs, T_ALL, dst) exec_with_vals(self.conf.post_part) self.log.info("Created table: %s" % dst)
def check_part(self, dst, part_time): """Create part table if not exists. It part_template present, execute it else if part function present in db, call it else clone master table""" curs = self.dst_curs if skytools.exists_table(curs, dst): return dst = quote_fqident(dst) vals = {'dest': dst, 'part': dst, 'parent': self.fq_dest_table, 'pkeys': ",".join(self.pkeys), # quoting? # we do this to make sure that constraints for # tables who contain a schema will still work 'schema_table': dst.replace(".", "__"), 'part_field': self.conf.part_field, 'part_time': part_time, 'period': self.conf.period, } def exec_with_vals(tmpl): if tmpl: sql = tmpl % vals curs.execute(sql) return True return False exec_with_vals(self.conf.pre_part) if not exec_with_vals(self.conf.part_template): self.log.debug('part_template not provided, using part func') # if part func exists call it with val arguments pfargs = ', '.join('%%(%s)s' % arg for arg in PART_FUNC_ARGS) pfcall = 'select %s(%s)' % (self.conf.part_func, pfargs) if skytools.exists_function(curs, self.conf.part_func, len(PART_FUNC_ARGS)): self.log.debug('check_part.exec: func:%s, args: %s' % (pfcall, vals)) curs.execute(pfcall, vals) else: self.log.debug('part func %s not found, cloning table' % self.conf.part_func) struct = TableStruct(curs, self.dest_table) struct.create(curs, T_ALL, dst) exec_with_vals(self.conf.post_part) self.log.info("Created table: %s" % dst)
def do_maintenance(self): """Helper function for running maintenance.""" db = self.get_database('db', autocommit=1) cx = db.cursor() if skytools.exists_function(cx, "pgq.maint_rotate_tables_step1", 1): # rotate each queue in own TX q = "select queue_name from pgq.get_queue_info()" cx.execute(q) for row in cx.fetchall(): cx.execute("select pgq.maint_rotate_tables_step1(%s)", [row[0]]) res = cx.fetchone()[0] if res: self.log.info('Rotating %s' % row[0]) else: cx.execute("select pgq.maint_rotate_tables_step1();") # finish rotation cx.execute("select pgq.maint_rotate_tables_step2();") # move retry events to main queue in small blocks rcount = 0 while 1: cx.execute('select pgq.maint_retry_events();') res = cx.fetchone()[0] rcount += res if res == 0: break if rcount: self.log.info('Got %d events for retry' % rcount) # vacuum tables that are needed cx.execute('set maintenance_work_mem = 32768') cx.execute('select * from pgq.maint_tables_to_vacuum()') for row in cx.fetchall(): cx.execute('vacuum %s;' % row[0])
def check_part(self, dst, part_time): """Create part table if not exists. It part_template present, execute it else if part function present in db, call it else clone master table""" curs = self.dst_curs if skytools.exists_table(curs, dst): return dst = quote_fqident(dst) vals = { 'dest': dst, 'part': dst, 'parent': self.fq_dest_table, 'pkeys': ",".join(self.pkeys), # quoting? # we do this to make sure that constraints for # tables who contain a schema will still work 'schema_table': dst.replace(".", "__"), 'part_field': self.conf.part_field, 'part_time': part_time, 'period': self.conf.period, } def exec_with_vals(tmpl): if tmpl: sql = tmpl % vals curs.execute(sql) return True return False exec_with_vals(self.conf.pre_part) if not exec_with_vals(self.conf.part_template): self.log.debug('part_template not provided, using part func') # if part func exists call it with val arguments pfargs = ', '.join('%%(%s)s' % arg for arg in PART_FUNC_ARGS) # set up configured function pfcall = 'select %s(%s)' % (self.conf.part_func, pfargs) have_func = skytools.exists_function(curs, self.conf.part_func, len(PART_FUNC_ARGS)) # backwards compat if not have_func and self.conf.part_func == PART_FUNC_NEW: pfcall = 'select %s(%s)' % (PART_FUNC_OLD, pfargs) have_func = skytools.exists_function(curs, PART_FUNC_OLD, len(PART_FUNC_ARGS)) if have_func: self.log.debug('check_part.exec: func:%s, args: %s' % (pfcall, vals)) curs.execute(pfcall, vals) else: # # Otherwise crete simple clone. # # FixMe: differences from create_partitions(): # - check constraints # - inheritance # self.log.debug('part func %s not found, cloning table' % self.conf.part_func) struct = TableStruct(curs, self.dest_table) struct.create(curs, T_ALL, dst) exec_with_vals(self.conf.post_part) self.log.info("Created table: %s" % dst)
def check_part(self, dst, part_time): """Create part table if not exists. It part_template present, execute it else if part function present in db, call it else clone master table""" curs = self.dst_curs if (self.conf.ignore_old_events and self.conf.retention_period and self.is_obsolete_partition(dst, self.conf.retention_period, self.conf.period)): self.ignored_tables.add(dst) return if skytools.exists_table(curs, dst): return dst = quote_fqident(dst) vals = {'dest': dst, 'part': dst, 'parent': self.fq_dest_table, 'pkeys': ",".join(self.pkeys), # quoting? # we do this to make sure that constraints for # tables who contain a schema will still work 'schema_table': dst.replace(".", "__"), 'part_field': self.conf.part_field, 'part_time': part_time, 'period': self.conf.period, } def exec_with_vals(tmpl): if tmpl: sql = tmpl % vals curs.execute(sql) return True return False exec_with_vals(self.conf.pre_part) if not exec_with_vals(self.conf.part_template): self.log.debug('part_template not provided, using part func') # if part func exists call it with val arguments pfargs = ', '.join('%%(%s)s' % arg for arg in PART_FUNC_ARGS) # set up configured function pfcall = 'select %s(%s)' % (self.conf.part_func, pfargs) have_func = skytools.exists_function(curs, self.conf.part_func, len(PART_FUNC_ARGS)) # backwards compat if not have_func and self.conf.part_func == PART_FUNC_NEW: pfcall = 'select %s(%s)' % (PART_FUNC_OLD, pfargs) have_func = skytools.exists_function(curs, PART_FUNC_OLD, len(PART_FUNC_ARGS)) if have_func: self.log.debug('check_part.exec: func: %s, args: %s', pfcall, vals) curs.execute(pfcall, vals) else: # # Otherwise create simple clone. # # FixMe: differences from create_partitions(): # - check constraints # - inheritance # self.log.debug('part func %s not found, cloning table', self.conf.part_func) struct = TableStruct(curs, self.dest_table) struct.create(curs, T_ALL, dst) exec_with_vals(self.conf.post_part) self.log.info("Created table: %s", dst) if self.conf.retention_period: dropped = self.drop_obsolete_partitions(self.dest_table, self.conf.retention_period, self.conf.period) if self.conf.ignore_old_events and dropped: for tbl in dropped: self.ignored_tables.add(tbl) if tbl in self.row_handler.table_map: del self.row_handler.table_map[tbl]