def match(self, objname, curs, tables, seqs):
nargs = 0
pos1 = objname.find('(')
if pos1 > 0:
pos2 = objname.find(')')
if pos2 > 0:
s = objname[pos1+1 : pos2]
objname = objname[:pos1]
nargs = int(s)
return skytools.exists_function(curs, objname, nargs)
def match(self, objname, curs, tables, seqs):
nargs = 0
pos1 = objname.find('(')
if pos1 > 0:
pos2 = objname.find(')')
if pos2 > 0:
s = objname[pos1 + 1:pos2]
objname = objname[:pos1]
nargs = int(s)
return skytools.exists_function(curs, objname, nargs)
def check_version(curs, schema, new_ver_str, recheck_func=None):
funcname = "%s.version" % schema
if not skytools.exists_function(curs, funcname, 0):
if recheck_func is not None:
return recheck_func(curs)
else:
return 0
q = "select %s()" % funcname
curs.execute(q)
old_ver_str = curs.fetchone()[0]
return is_version_ge(old_ver_str, new_ver_str)
def check_version(curs, schema, new_ver_str, recheck_func=None):
funcname = "%s.version" % schema
if not skytools.exists_function(curs, funcname, 0):
if recheck_func is not None:
return recheck_func(curs), 'NULL'
else:
return 0, 'NULL'
q = "select %s()" % funcname
curs.execute(q)
old_ver_str = curs.fetchone()[0]
ok = is_version_ge(old_ver_str, new_ver_str)
return ok, old_ver_str
def check_version(curs, schema, new_ver_str, recheck_func=None):
funcname = "%s.version" % schema
if not skytools.exists_function(curs, funcname, 0):
if recheck_func is not None:
return recheck_func(curs)
else:
return 0
q = "select %s()" % funcname
curs.execute(q)
old_ver_str = curs.fetchone()[0]
new_ver = parse_ver(new_ver_str)
old_ver = parse_ver(old_ver_str)
return old_ver >= new_ver
def check_version(curs, schema, new_ver_str, recheck_func=None, force_gt=False):
funcname = "%s.version" % schema
if not skytools.exists_function(curs, funcname, 0):
if recheck_func is not None:
return recheck_func(curs), 'NULL'
else:
return 0, 'NULL'
q = "select %s()" % funcname
curs.execute(q)
old_ver_str = curs.fetchone()[0]
if force_gt:
ok = is_version_gt(old_ver_str, new_ver_str)
else:
ok = is_version_ge(old_ver_str, new_ver_str)
return ok, old_ver_str
def check_part(self, dst, part_time):
"""Create part table if not exists.
It part_template present, execute it
else if part function present in db, call it
else clone master table"""
curs = self.dst_curs
if skytools.exists_table(curs, dst):
return
dst = quote_fqident(dst)
vals = {
'dest': dst,
'part': dst,
'parent': self.quoted_name,
'pkeys': ",".join(self.pkeys), # quoting?
# we do this to make sure that constraints for
# tables who contain a schema will still work
'schema_table': dst.replace(".", "__"),
'part_field': self.conf.part_field,
'part_time': part_time,
'period': self.conf.period,
}
def exec_with_vals(tmpl):
if tmpl:
sql = tmpl % vals
curs.execute(sql)
return True
return False
exec_with_vals(self.conf.pre_part)
if not exec_with_vals(self.conf.part_template):
self.log.debug('part_template not provided, using part func')
# if part func exists call it with val arguments
if skytools.exists_function(curs, PART_FUNC, len(PART_FUNC_ARGS)):
self.log.debug('check_part.exec: func:%s, args: %s' %\
(PART_FUNC_CALL, vals))
curs.execute(PART_FUNC_CALL, vals)
else:
self.log.debug('part func %s not found, cloning table' %\
PART_FUNC)
struct = TableStruct(curs, self.table_name)
struct.create(curs, T_ALL, dst)
exec_with_vals(self.conf.post_part)
self.log.info("Created table: %s" % dst)
def check_part(self, dst, part_time):
"""Create part table if not exists.
It part_template present, execute it
else if part function present in db, call it
else clone master table"""
curs = self.dst_curs
if skytools.exists_table(curs, dst):
return
dst = quote_fqident(dst)
vals = {
"dest": dst,
"part": dst,
"parent": self.quoted_name,
"pkeys": ",".join(self.pkeys), # quoting?
# we do this to make sure that constraints for
# tables who contain a schema will still work
"schema_table": dst.replace(".", "__"),
"part_field": self.conf.part_field,
"part_time": part_time,
"period": self.conf.period,
}
def exec_with_vals(tmpl):
if tmpl:
sql = tmpl % vals
curs.execute(sql)
return True
return False
exec_with_vals(self.conf.pre_part)
if not exec_with_vals(self.conf.part_template):
self.log.debug("part_template not provided, using part func")
# if part func exists call it with val arguments
if skytools.exists_function(curs, PART_FUNC, len(PART_FUNC_ARGS)):
self.log.debug("check_part.exec: func:%s, args: %s" % (PART_FUNC_CALL, vals))
curs.execute(PART_FUNC_CALL, vals)
else:
self.log.debug("part func %s not found, cloning table" % PART_FUNC)
struct = TableStruct(curs, self.table_name)
struct.create(curs, T_ALL, dst)
exec_with_vals(self.conf.post_part)
self.log.info("Created table: %s" % dst)
def check_part(self, dst, part_time):
"""Create part table if not exists.
It part_template present, execute it
else if part function present in db, call it
else clone master table"""
curs = self.dst_curs
if skytools.exists_table(curs, dst):
return
dst = quote_fqident(dst)
vals = {'dest': dst,
'part': dst,
'parent': self.fq_dest_table,
'pkeys': ",".join(self.pkeys), # quoting?
# we do this to make sure that constraints for
# tables who contain a schema will still work
'schema_table': dst.replace(".", "__"),
'part_field': self.conf.part_field,
'part_time': part_time,
'period': self.conf.period,
}
def exec_with_vals(tmpl):
if tmpl:
sql = tmpl % vals
curs.execute(sql)
return True
return False
exec_with_vals(self.conf.pre_part)
if not exec_with_vals(self.conf.part_template):
self.log.debug('part_template not provided, using part func')
# if part func exists call it with val arguments
pfargs = ', '.join('%%(%s)s' % arg for arg in PART_FUNC_ARGS)
pfcall = 'select %s(%s)' % (self.conf.part_func, pfargs)
if skytools.exists_function(curs, self.conf.part_func, len(PART_FUNC_ARGS)):
self.log.debug('check_part.exec: func:%s, args: %s' % (pfcall, vals))
curs.execute(pfcall, vals)
else:
self.log.debug('part func %s not found, cloning table' % self.conf.part_func)
struct = TableStruct(curs, self.dest_table)
struct.create(curs, T_ALL, dst)
exec_with_vals(self.conf.post_part)
self.log.info("Created table: %s" % dst)
def do_maintenance(self):
"""Helper function for running maintenance."""
db = self.get_database('db', autocommit=1)
cx = db.cursor()
if skytools.exists_function(cx, "pgq.maint_rotate_tables_step1", 1):
# rotate each queue in own TX
q = "select queue_name from pgq.get_queue_info()"
cx.execute(q)
for row in cx.fetchall():
cx.execute("select pgq.maint_rotate_tables_step1(%s)", [row[0]])
res = cx.fetchone()[0]
if res:
self.log.info('Rotating %s' % row[0])
else:
cx.execute("select pgq.maint_rotate_tables_step1();")
# finish rotation
cx.execute("select pgq.maint_rotate_tables_step2();")
# move retry events to main queue in small blocks
rcount = 0
while 1:
cx.execute('select pgq.maint_retry_events();')
res = cx.fetchone()[0]
rcount += res
if res == 0:
break
if rcount:
self.log.info('Got %d events for retry' % rcount)
# vacuum tables that are needed
cx.execute('set maintenance_work_mem = 32768')
cx.execute('select * from pgq.maint_tables_to_vacuum()')
for row in cx.fetchall():
cx.execute('vacuum %s;' % row[0])
def check_part(self, dst, part_time):
"""Create part table if not exists.
It part_template present, execute it
else if part function present in db, call it
else clone master table"""
curs = self.dst_curs
if skytools.exists_table(curs, dst):
return
dst = quote_fqident(dst)
vals = {
'dest': dst,
'part': dst,
'parent': self.fq_dest_table,
'pkeys': ",".join(self.pkeys), # quoting?
# we do this to make sure that constraints for
# tables who contain a schema will still work
'schema_table': dst.replace(".", "__"),
'part_field': self.conf.part_field,
'part_time': part_time,
'period': self.conf.period,
}
def exec_with_vals(tmpl):
if tmpl:
sql = tmpl % vals
curs.execute(sql)
return True
return False
exec_with_vals(self.conf.pre_part)
if not exec_with_vals(self.conf.part_template):
self.log.debug('part_template not provided, using part func')
# if part func exists call it with val arguments
pfargs = ', '.join('%%(%s)s' % arg for arg in PART_FUNC_ARGS)
# set up configured function
pfcall = 'select %s(%s)' % (self.conf.part_func, pfargs)
have_func = skytools.exists_function(curs, self.conf.part_func,
len(PART_FUNC_ARGS))
# backwards compat
if not have_func and self.conf.part_func == PART_FUNC_NEW:
pfcall = 'select %s(%s)' % (PART_FUNC_OLD, pfargs)
have_func = skytools.exists_function(curs, PART_FUNC_OLD,
len(PART_FUNC_ARGS))
if have_func:
self.log.debug('check_part.exec: func:%s, args: %s' %
(pfcall, vals))
curs.execute(pfcall, vals)
else:
#
# Otherwise crete simple clone.
#
# FixMe: differences from create_partitions():
# - check constraints
# - inheritance
#
self.log.debug('part func %s not found, cloning table' %
self.conf.part_func)
struct = TableStruct(curs, self.dest_table)
struct.create(curs, T_ALL, dst)
exec_with_vals(self.conf.post_part)
self.log.info("Created table: %s" % dst)
def check_part(self, dst, part_time):
"""Create part table if not exists.
It part_template present, execute it
else if part function present in db, call it
else clone master table"""
curs = self.dst_curs
if (self.conf.ignore_old_events and self.conf.retention_period and
self.is_obsolete_partition(dst, self.conf.retention_period, self.conf.period)):
self.ignored_tables.add(dst)
return
if skytools.exists_table(curs, dst):
return
dst = quote_fqident(dst)
vals = {'dest': dst,
'part': dst,
'parent': self.fq_dest_table,
'pkeys': ",".join(self.pkeys), # quoting?
# we do this to make sure that constraints for
# tables who contain a schema will still work
'schema_table': dst.replace(".", "__"),
'part_field': self.conf.part_field,
'part_time': part_time,
'period': self.conf.period,
}
def exec_with_vals(tmpl):
if tmpl:
sql = tmpl % vals
curs.execute(sql)
return True
return False
exec_with_vals(self.conf.pre_part)
if not exec_with_vals(self.conf.part_template):
self.log.debug('part_template not provided, using part func')
# if part func exists call it with val arguments
pfargs = ', '.join('%%(%s)s' % arg for arg in PART_FUNC_ARGS)
# set up configured function
pfcall = 'select %s(%s)' % (self.conf.part_func, pfargs)
have_func = skytools.exists_function(curs, self.conf.part_func, len(PART_FUNC_ARGS))
# backwards compat
if not have_func and self.conf.part_func == PART_FUNC_NEW:
pfcall = 'select %s(%s)' % (PART_FUNC_OLD, pfargs)
have_func = skytools.exists_function(curs, PART_FUNC_OLD, len(PART_FUNC_ARGS))
if have_func:
self.log.debug('check_part.exec: func: %s, args: %s', pfcall, vals)
curs.execute(pfcall, vals)
else:
#
# Otherwise create simple clone.
#
# FixMe: differences from create_partitions():
# - check constraints
# - inheritance
#
self.log.debug('part func %s not found, cloning table', self.conf.part_func)
struct = TableStruct(curs, self.dest_table)
struct.create(curs, T_ALL, dst)
exec_with_vals(self.conf.post_part)
self.log.info("Created table: %s", dst)
if self.conf.retention_period:
dropped = self.drop_obsolete_partitions(self.dest_table, self.conf.retention_period, self.conf.period)
if self.conf.ignore_old_events and dropped:
for tbl in dropped:
self.ignored_tables.add(tbl)
if tbl in self.row_handler.table_map:
del self.row_handler.table_map[tbl]
