def __init__(self, cfg_mgr):
"""Init."""
self.cfg_mgr = cfg_mgr
self.it_inventory = ITInventory(self.cfg_mgr)
self.export_it_inventory = ExportITInventory(self.cfg_mgr)
self.logger = get_logger(self.cfg_mgr)
self.log_msg = ''
def create(tables_fh, cfg_mgr, timeout):
"""Start it table generation."""
auto = ''
manual = ''
auto_fh = None
if not timeout:
timeout = 1800
requests, msg, _ = parse_file_by_sections(tables_fh, '[Request]', REQ_KEYS,
OPTIONAL_FIELDS)
if msg:
return auto_fh
for request in requests:
inv = ITInventory(cfg_mgr)
db_table_dict = inv.get_table_mapping(
request['source_database_name'].lower(),
request['source_table_name'].lower(), request['db_env'].lower())
if db_table_dict:
it_table_obj = ItTable(db_table_dict, cfg_mgr)
else:
it_table_obj = ItTable(request, cfg_mgr)
src_obj = get_src_obj(cfg_mgr, it_table_obj)
status, it_table_str = src_obj.generate_it_table(timeout)
if status == 'auto':
auto += it_table_str + '\n'
elif status == 'manual':
manual += it_table_str + '\n'
parent_dir = os.path.dirname(os.path.abspath(tables_fh.name))
auto_file_path = parent_dir + '/AUTO_it-tables.txt'
manual_file_path = parent_dir + '/MANUAL_it-tables.txt'
src_obj.write_it_table_files(auto_file_path, auto)
src_obj.write_it_table_files(manual_file_path, manual)
if auto:
auto_fh = open(auto_file_path, 'r')
return auto_fh
def setUpClass(cls, mock_connect):
"""Setup."""
cls.cfg_mgr = ConfigManager(UNIT_TEST_ENV)
cls.inventory = ITInventory(cls.cfg_mgr)
class RequestInventory(object):
"""Handles request file"""
def __init__(self, cfg_mgr):
"""Init."""
self.cfg_mgr = cfg_mgr
self.it_inventory = ITInventory(self.cfg_mgr)
self.export_it_inventory = ExportITInventory(self.cfg_mgr)
self.logger = get_logger(self.cfg_mgr)
self.log_msg = ''
def parse_file(self, request_file):
"""
Args:
request_file: instance of open()
Returns:
list(ibis.inventory.request_inventory.Request)
"""
requests, msg, log_msg = parse_file_by_sections(
request_file, '[Request]', REQUIRED_FIELDS, OPTIONAL_FIELDS)
self.logger.warning(log_msg)
self.log_msg = log_msg
temp_reqs = []
if not msg:
temp_reqs = [Request(req, self.cfg_mgr) for req in requests]
return temp_reqs, msg
def parse_file_export(self, request_file):
"""
Args:
request_file: instance of open()
Returns:
list(ibis.inventory.request_inventory.Request)
"""
requests, msg, log_msg = parse_file_by_sections(
request_file, '[Request]', REQUIRED_FIELDS_EXPORT,
OPTIONAL_FIELDS_EXPORT)
self.logger.warning(log_msg)
self.log_msg = log_msg
temp_reqs = []
if not msg:
temp_reqs = [Request(req, self.cfg_mgr) for req in requests]
return temp_reqs, msg
def get_available_requests(self, requests):
"""Given a list[{Request}] return a List[{ItTable}]
of tables available, not on hold in the it table, and a
List[{ItTable}] of tables on hold, due to a hold value
of 1 and List[{Request}]
Args:
requests: list(ibis.inventory.request_inventory.Request) objects
Returns:
list(ibis.model.table.ItTable),
list(ibis.model.table.ItTable),
list(ibis.inventory.request_inventory.Request)
"""
available_tables = []
hold_tables = []
unavailable_requests = []
for req_obj in requests:
src_db = req_obj.database
src_table = req_obj.table_name
if req_obj.db_env:
db_env = req_obj.db_env
else:
db_env = self.cfg_mgr.default_db_env.lower()
table = self.it_inventory.get_table_mapping(
src_db, src_table, db_env)
if table:
# A record of the table exists in the it table
if table['hold'] == 0:
available_tables.append(ItTable(table, self.cfg_mgr))
else:
# hold = 1 means it is disabled temporarily
hold_tables.append(ItTable(table, self.cfg_mgr))
else:
# Record doesn't exist
unavailable_requests.append(req_obj)
if hold_tables:
hold_tables_names = [table.table_name for table in hold_tables]
msg = 'Tables: {0} are on hold'
msg = msg.format(', '.join(hold_tables_names))
self.logger.warning(msg)
if unavailable_requests:
unavail_names = [req.table_name for req in unavailable_requests]
msg = 'Tables: {0} are missing in it_table'
msg = msg.format(', '.join(unavail_names))
self.logger.warning(msg)
return available_tables, hold_tables, unavailable_requests
def get_available_requests_export(self, requests):
"""Given a list[{Request}] return a List[{ItTableExport}]
of tables available
Args:
requests: list(ibis.inventory.request_inventory.Request) objects
Returns:
list(ibis.model.table.ItTableExport),
list(ibis.model.table.ItTableExport),
list(ibis.inventory.request_inventory.Request)
"""
tables = []
unavailable_requests = []
for req_obj in requests:
src_db = req_obj.database
src_table = req_obj.table_name
if req_obj.db_env:
db_env = req_obj.db_env
else:
db_env = self.cfg_mgr.default_db_env.lower()
table = self.export_it_inventory.get_table_mapping(
src_db, src_table, db_env)
if table:
# A record of the table exists in the it table
tables.append(ItTableExport(table, self.cfg_mgr))
else:
# Record doesn't exist
unavailable_requests.append(req_obj)
if unavailable_requests:
unavail_names = [req.table_name for req in unavailable_requests]
msg = 'Tables: {0} are missing in it table'
msg = msg.format(', '.join(unavail_names))
self.logger.warning(msg)
return tables, unavailable_requests
def get_split(self):
query = """select databasename source_database_name,
(tablename) source_table_name,
columnname,
case
when indextype='P' and \
indexnumber=1 then 2
when indextype='P' and \
indexnumber=2 then 4
when indextype='Q' and \
indexnumber=1 then 1
when indextype='Q' and \
indexnumber=2 then 3
when indextype='S' then 6
when indextype='K' then 5
else 7
end as Split_By_Find
from dbc.indices
where
databasename = '{db_name}'
and tablename = '{table_name}'
order by Split_By_Find"""
db = self.db
table = self.table
server_name = filter(
lambda server: server.lower() in self.jdbc_url.lower(),
self.cfg_mgr.teradata_server)
if server_name:
server_name = self.cfg_mgr.teradata_server[server_name[0]]
hdfs_query = """ select columnname from ibis.teradata_split_{0} \
where lower(databasename) = '{1}' and lower(tablename) = '{2}'"""
hdfs_query = hdfs_query.format(server_name, db.lower(),
table.lower())
run_query = ITInventory(self.cfg_mgr)
results = run_query.get_rows(hdfs_query)
if results:
return results[0][0]
else:
self.logger.warning('No split_by found:\
{0}'.format(hdfs_query))
return "no-split"
else:
query = query.format(table_name=table, db_name=db)
returncode, output, err = self.eval(query)
if returncode == 0:
_, result = self.fetch_rows_sqoop(output)
if result is None or len(result) == 0:
self.logger.warning('No split_by found: {0}'.format(query))
return "no-split"
else:
if result[0][3] == '7':
int_bigint_query = """SELECT columnname
FROM
(SELECT columnname,
rank() over (partition BY columntype
ORDER BY columnid) rnk
FROM dbc.columns
WHERE tablename = '{table_name}'
AND databasename = '{db_name}'
AND columntype IN ('I',
'I8') )a
WHERE rnk=1""".format(table_name=table, db_name=db)
returncode, output, err = self.eval(int_bigint_query)
if returncode == 0:
_, result = self.fetch_rows_sqoop(output)
if result is None:
self.logger.error("No split by assigned \
and one could not be found")
raise ValueError("No split by assigned \
and one could not be found")
if result:
result = result[0][0]
return result
else:
_msg = 'No split_by found: {0}'.format(query)
self.logger.warning(_msg)
return "no-split"
else:
self.logger.error(output)
self.logger.error(err)
else:
return result[0][2]
else:
self.logger.error(output)
self.logger.error(err)
raise ValueError("No split by assigned and one could not be found")