def fetch_task_list(self) -> Generator[str, None, None]:
if self.task_file == "":
builder = QueryBuilder(self.client, self.querybuilder_title)
query = builder.invoke()
for page in query.invoke():
yield page
else:
try:
with open(self.task_file, "r", encoding='utf-8') as f:
for task in f.read().split('\n'):
yield task
except Exception as e:
print(self.lang.t("common.ioerror"))
def get_mysql_table_metadata(schema_name, table_name):
"""
Description: to fetch the meta data of particlar mysql table
parameters: 1) schema_name (to select the particular schema)
2) table_name (to select the particaular table under particular schema)
Return: mysql_table_columns (mysql table columns with column name and data type)
Exceptions: 1) TypeError exception in case if operation is applied to an object of an incorrect type
2) IndexError exception in case if index goes out of range
3) AttributeError exception in case if call any undefined attribute
"""
try:
mysql_table_columns = []
query_string = QueryBuilder.get_mysql_metadata_query(
schema_name, table_name)
query_result = Mysql.get_mysql_query_output(query_string)
#to fetch the columns name and data type from the query result set
for row in query_result:
metadata_dic = {}
metadata_dic["Name"] = row[0]
metadata_dic["Type"] = row[1]
mysql_table_columns.append(metadata_dic)
return mysql_table_columns
except TypeError as err:
print(err)
raise
except IndexError as err:
print(err)
raise
except AttributeError as err:
print(err)
raise
except Exception as err:
print(err)
def get_columns_mapping(table_id):
"""
Description: to fetch the columns mapping of particlar table
parameters: table_id (particular table id whose we fetch the columns mapping from columns_mapping_table)
Return: columns_mapping (particular table columns mapping)
Exceptions: 1) TypeError exception in case if operation is applied to an object of an incorrect type
2) IndexError exception in case if index goes out of range
3) AttributeError exception in case if call any undefined attribute
"""
try:
columns_mapping = {}
query_string = QueryBuilder.get_columns_mapping_query(
params["mysql_db_name"],
params["mysql_athena_columns_mapping_table"], table_id)
query_result = Mysql.get_mysql_query_output(query_string)
for row in query_result:
columns_mapping[row[1]] = row[2]
return columns_mapping
except TypeError as err:
print(err)
raise
except IndexError as err:
print(err)
raise
except AttributeError as err:
print(err)
raise
except Exception as err:
print(err)
def get_mysql_table_count(schema_name, table_name):
"""
Description: to fetch the count of particlar mysql table
parameters: 1) schema_name (to select the particular schema)
2) table_name (to select the particaular table under particular schema)
Return: mysql_table_count (mysql table count)
Exceptions:
"""
query_string = QueryBuilder.get_count_query(schema_name, table_name)
resultset = Mysql.get_mysql_query_output(query_string)
mysql_table_count = resultset[0][0]
return mysql_table_count
def get_active_tables():
"""
Description: to fetch all the active tables
Parameters: none
Return: active_tables (list of all the active tables)
Exception:
"""
if not bool(QueryResult.active_tables):
query_string = QueryBuilder.get_active_tables_query(
params["mysql_db_name"], params["mysql_tables_metadata_table"])
QueryResult.active_tables = Mysql.get_mysql_query_output(
query_string)
return QueryResult.active_tables
def get_dt_mapping():
"""
Description: to fetch the data types mapping across different data bases
Parameters: none
Return: datatypes_mapping (list of data types mapping of across different data bases)
Exception:
"""
if not bool(QueryResult.datatypes_mapping):
query_string = QueryBuilder.get_datatypes_mapping_query(
params["mysql_db_name"], params["datatypes_mapping_table"],
params["source_endpoint"], params["target_endpoint"])
query_result = Mysql.get_mysql_query_output(query_string)
for row in query_result:
QueryResult.datatypes_mapping[row[0]] = row[1]
return QueryResult.datatypes_mapping
def get_athena_table_count(schema_name, table_name):
"""
Description: to fetch the count of particlar athena table
parameters: 1) schema_name (to select the particular schema)
2) table_name (to select the particaular table under particular schema)
Return: athena_table_count (athena table count)
Exceptions:
"""
query_string = QueryBuilder.get_count_query(schema_name, table_name)
fileobj = Athena.get_query_result(
params["glue_db_name"], query_string,
params["s3_athena_output_location_path"], params["s3_bucket_name"])
file_body = pd.read_csv(fileobj['Body'])
resultset = pd.DataFrame(file_body)
athena_table_count = resultset.values[0][0]
return athena_table_count