def create_log_error_table(db, schema='public'):
"""
Logs the error's location, message and timestamp
when an it occurs.
Parameters
----------
db: connection obj
schema: name of the schema in Postgres
Returns
-------
-
Example
-------
create_log_error_table(pg, 'purr')
"""
table_name = "purr_error"
attrs = table_desc[table_name]["attrs"]
types = table_desc[table_name]["types"]
values = [int(time.time())]
try:
table.create(db, schema, table_name, attrs, types)
logger.info("[TRANSFER INFO] Created table %s." % (table_name))
except Exception as ex:
logger.error(
"[TRANSFER_INFO] Failed to create table %s: %s" % (table_name, ex))
def create_stat_table(db, schema='public'):
"""
Creates a table that holds the timestamp of the
latest successfully inserted item.
Parameters
----------
Returns
-------
-
Example
-------
create_stat_table(pg, 'purr')
"""
table_name = "purr_info"
attrs = table_desc[table_name]["attrs"]
types = table_desc[table_name]["types"]
values = [0, None, int(time.time())]
try:
table.create(db, schema, table_name, attrs, types)
ts = get_latest_successful_ts(db, schema)
if len(ts) == 0:
row.insert(db, schema, table_name, attrs, values)
logger.info("[TRANSFER INFO] Created table %s." % (table_name))
except Exception as ex:
logger.error(
"[TRANSFER_INFO] Failed to create table %s: %s" % (table_name, ex))
def create_transfer_stats_table(db, schema='public'):
"""
Logs the number, relation name, timestamp
for each collection transfer.
Parameters
----------
db: connection obj
schema: name of the schema in Postgres
Returns
-------
-
Example
-------
create_transfer_stats_table(pg, 'purr')
"""
table_name = "purr_transfer_stats"
attrs = table_desc[table_name]["attrs"]
types = table_desc[table_name]["types"]
values = [int(time.time())]
try:
table.create(db, schema, table_name, attrs, types)
logger.info("[TRANSFER INFO] Created table %s." % (table_name))
except Exception as ex:
logger.error(
"[TRANSFER_INFO] Failed to create table %s: %s" % (table_name, ex))
def create_oplog_table(db, schema='public'):
"""
Logs the operation, relation name, object id and
timestamp for each entry of the oplog.
Parameters
----------
db: connection obj
schema: name of the schema in Postgres
Returns
-------
-
Example
-------
create_oplog_table(pg, 'purr')
"""
table_name = "purr_oplog"
attrs = table_desc[table_name]["attrs"]
types = table_desc[table_name]["types"]
pks = table_desc[table_name]["pks"]
values = [int(time.time())]
try:
table.drop(db, schema, [table_name])
table.create(db, schema, table_name, attrs, types, pks)
logger.info("[TRANSFER INFO] Created table %s." % (table_name))
except Exception as ex:
logger.error(
"[TRANSFER_INFO] Failed to create table %s: %s" % (table_name, ex))
def test_create_attrs_and_types(self):
reset_dataset_pg()
table.create(pg, schema, rel_name, attrs, types)
cursor = pg.conn.cursor()
cursor.execute("SELECT * FROM %s.%s;" % (schema, rel_name))
cursor.fetchall()
columns = [x[0] for x in cursor.description]
cursor.execute("""
select column_name, data_type
from information_schema.columns
where
table_schema = '%s'
and table_name = '%s';
""" % (schema, rel_name))
res = cursor.fetchall()
res.sort(key=lambda tup: tup[0])
col_names = [x[0] for x in res]
col_types = [x[1].upper() for x in res]
equal_attrs = (set(col_names) - set(attrs)) == set()
equal_types = (set(col_types) - set(types)) == set()
print(equal_attrs)
print(equal_types)
cursor.close()
assert equal_attrs and equal_types
def test_create_pk(self):
reset_dataset_pg()
pk = ["id"]
table.create(pg, schema, rel_name, attrs, types)
cursor = pg.conn.cursor()
cursor.execute("""
select column_name
from information_schema.constraint_column_usage
where
table_schema = '%s'
and table_name = '%s';
""" % (schema, rel_name))
res = cursor.fetchall()
col_names = [x[0] for x in res]
cursor.close()
assert col_names == pk
def create_table(db, coll_map, schema='public'):
"""
Adds primary key to a PostgreSQL table.
Parameters
----------
Returns
-------
-
Example
-------
create_table(pg, 'purr')
"""
table_name = "purr_collection_map"
attrs = [
"id", "collection_name", "relation_name", "types", "updated_at",
"query_update"
]
types = ["integer", "text", "text", "jsonb[]", "timestamp", "text"]
try:
# TODO: make this fucntion accept string and list
table.drop(db, schema, [table_name])
table.create(db, schema, table_name, attrs, types)
logger.info("Created table %s.%s." % (schema, table_name), CURR_FILE)
except Exception as ex:
logger.error(
"""
Failed to create table %s.%s: %s
""" % (schema, table_name, ex), CURR_FILE)
populate_table(db, coll_map, table_name, attrs, schema)
procedure_name = 'notify_type'
procedure.drop_type_notification(db, procedure_name)
procedure.create_type_notification(db, procedure_name)
table.drop_trigger_type_notification(db, 'public', 'purr_collection_map',
'notify', procedure_name)
table.create_trigger_type_notification(db, 'public', 'purr_collection_map',
'notify', procedure_name)
def create(self, attrs, types):
table.create(self.db, self.schema, self.relation_name, attrs, types)