def clean_up(self):
"""Cleanup db changes after faking process."""
print_warn('----- Data cleanup process has been started -----', 1)
try:
if self.db.conf_is_correct():
self.db.clean_up()
print_warn('----- Data cleanup process finished -----', 1)
except Exception as e:
print_err(e.message)
def run(self):
"""Runing the data faking process and dependant checkings."""
print_warn('----- Data faking process has been started -----', 1)
try:
if self.db.conf_is_correct():
self.db.fake_tables_data()
print_warn('----- Data faking process finished -----', 1)
except Exception as e:
print_err(e.message)
def __get_inheritee(self, schema):
"""
Get schema name which inherits its schema definition.
Args:
schema (str): Inheritant schema name
Returns:
schema (str): Inheritee schema name
"""
try:
return self.schemas_map[schema]
except Exception:
print_err(
"There isn't defined inheritee schema for {}".format(schema))
def connect(self, db_params):
"""
Establishe db connection according given parameters.
Args:
db_params (dict): Connection parameters
Returns:
cursor (object): Connection cursor
"""
try:
with psycopg2.connect(**db_params) as conn:
return conn
except Exception as e:
print_err('Connection cannot be established by given parameters.')
raise Exception(e.message)
def __conf_table_columns(self, schema, table):
"""
Get coulmns list from config dict. flatten for given table.
Args:
schema (str): Schema name
table (str): Table name
Returns:
columns (list): Schema names
"""
try:
schema = self.__get_inheritee(schema)
return self.schemas_config['schemas'][schema]['tables'][table][
'columns'].keys()
except Exception as e:
print_err(e.message)
raise Exception('Json structure definition is invalid.[2]')
def __conf_table_keys(self, schema, table):
"""
Get keys list from config dict. flatten for given table.
!Note this is mandatory to have.
Args:
schema (str): Schema name
table (str): Table name
Returns:
keys (list): Schema keys
"""
try:
schema = self.__get_inheritee(schema)
return self.schemas_config['schemas'][schema]['tables'][table][
'keys']
except Exception as e:
print_err(e.message)
raise Exception('Json structure definition is invalid.[3]')
def __conf_schemas(self):
"""
Get schemas list from config dict. flatten.
Returns:
schemas (list): Schema names
"""
try:
inheritants = self.schemas_config['inheritants']
schemas_list = []
for schm_def, schemas in inheritants.items():
self.schemas_map.update(
{schema: schm_def
for schema in schemas})
schemas_list.extend(schemas)
return schemas_list
except Exception as e:
print_err('Json structure definition is invalid.[1]')
raise Exception(e.message)
def conf_is_correct(self):
"""
Check wheter defined config is correct.
Means the defined schemas, tables, columns exist.
Args:
config (dict): Database tree started from schemas to its tcolumns
that need to be shuffled
Returns:
Bool: True if defined structure exist in connected DB
False otherwise
"""
print_inf('Checking schemas correctness according to config.json')
try:
error = False
for schema in self.__conf_schemas():
if not self.__schema_exists(schema):
print_err("""Schema "{}" doesn't exist""".format(schema))
error = True
for table in self.__conf_schema_tables(schema):
if not self.__table_exists(schema, table):
print_err("""Table "{}" for schema "{}" doesn't
exist""".format(table, schema))
error = True
for column in self.__conf_table_columns(schema, table):
if not self.__column_exists(schema, table, column):
print_err("""Column "{}" in table "{}" for schema
"{}" doesn't exist""".format(
column, table, schema))
error = True
print_inf('{} success'.format(schema))
return False if error else True
except Exception as e:
print_err(e.message)
def __generate_update_query(self, schema, table, conf_table_keys):
"""
Generate update sql based on condition.
Returns:
{sql, values}(dict): Query, values.
"""
keys = ", ".join(conf_table_keys)
try:
select_sql = """SELECT {} FROM {}.{} WHERE {} IS FALSE LIMIT 1
FOR UPDATE""".format(keys, schema, table, self.state_column)
except Exception as e:
print_err(e.message)
raise Exception(
'Select sql generating problem. {}'.format(select_sql))
try:
fake_data = OrderedDict()
data_types = self.data_inst.get_types()
for column in self.__conf_table_columns(schema, table):
rule = self.__conf_coulmn_rules(schema, table, column)
if rule['type'] in data_types:
fake_data[column] = data_types[rule['type']]
else:
print_err("""Fake data type "{}" is not implemented
yet.""".format(rule['type']))
return False
fake_data[self.state_column] = True
set_sql = "=%s, ".join(fake_data.keys())
set_sql += "=%s"
update_sql = """UPDATE {}.{} SET {} WHERE {} IN ({}) RETURNING {}"""
update_sql = update_sql.format(
schema,
table,
set_sql,
keys,
select_sql,
keys,
)
return {
'sql': update_sql,
'values': [val for _, val in fake_data.items()]
}
except Exception as e:
print_err(e.message)
raise Exception('Update query generating problem.')
def clean_up(self):
"""Cleanup db changes after faking process."""
print_warn('----- Data cleanup process has been started -----', 1)
try:
if self.db.conf_is_correct():
self.db.clean_up()
print_warn('----- Data cleanup process finished -----', 1)
except Exception as e:
print_err(e.message)
threads = {}
if len(sys.argv) > 1:
threads_count = 1
if sys.argv[2] and int(sys.argv[2]) > 1:
threads_count = int(sys.argv[2])
if sys.argv[1] == 'run':
for i in range(threads_count):
threads[i] = Db_faker('Thread_{}'.format(i))
threads[i].start()
print_err('----- {} Threads are running -----'.format(
len(threads.keys())))
elif sys.argv[1] == 'cleanup':
thread1 = Db_faker('Thread_1')
thread1.clean_up()
else:
print_warn('usage: ./db_faker.py "run | cleanup" int(threads_count)')
else:
print_warn('usage: ./db_faker.py "run | cleanup" int(threads_count)')