def test_unjoin_explicit_key_3():
table4 = (
("Tournament", "Year", "Winner", "Date of Birth"),
("Indiana Invitational", 1998, "Al Fredrickson", "21 July 1975"),
("Cleveland Open", 1999, "Bob Albertson", "28 September 1968"),
("Des Moines Masters", 1999, "Al Fredrickson", "21 July 1975"),
("Indiana Invitational", 1999, "Chip Masterson", "14 March 1977"),
)
# N.B., we do expect rows will get sorted
expect_left = (
("Tournament", "Year", "Winner"),
("Cleveland Open", 1999, "Bob Albertson"),
("Des Moines Masters", 1999, "Al Fredrickson"),
("Indiana Invitational", 1998, "Al Fredrickson"),
("Indiana Invitational", 1999, "Chip Masterson"),
)
expect_right = (
("Winner", "Date of Birth"),
("Al Fredrickson", "21 July 1975"),
("Bob Albertson", "28 September 1968"),
("Chip Masterson", "14 March 1977"),
)
left, right = unjoin(table4, "Date of Birth", key="Winner")
ieq(expect_left, left)
ieq(expect_left, left)
ieq(expect_right, right)
ieq(expect_right, right)
def test_unjoin_explicit_key_2():
table3 = (('Employee', 'Skill', 'Current Work Location'),
('Jones', 'Typing', '114 Main Street'),
('Jones', 'Shorthand', '114 Main Street'), ('Jones', 'Whittling',
'114 Main Street'),
('Bravo', 'Light Cleaning', '73 Industrial Way'),
('Ellis', 'Alchemy', '73 Industrial Way'), ('Ellis', 'Flying',
'73 Industrial Way'),
('Harrison', 'Light Cleaning', '73 Industrial Way'))
# N.B., we do expect rows will get sorted
expect_left = (('Employee', 'Current Work Location'),
('Bravo', '73 Industrial Way'), ('Ellis',
'73 Industrial Way'),
('Harrison', '73 Industrial Way'), ('Jones',
'114 Main Street'))
expect_right = (('Employee', 'Skill'), ('Bravo', 'Light Cleaning'),
('Ellis', 'Alchemy'), ('Ellis', 'Flying'),
('Harrison', 'Light Cleaning'), ('Jones', 'Shorthand'),
('Jones', 'Typing'), ('Jones', 'Whittling'))
left, right = unjoin(table3, 'Skill', key='Employee')
ieq(expect_left, left)
ieq(expect_left, left)
ieq(expect_right, right)
ieq(expect_right, right)
def test_unjoin_explicit_key_2():
table3 = (
("Employee", "Skill", "Current Work Location"),
("Jones", "Typing", "114 Main Street"),
("Jones", "Shorthand", "114 Main Street"),
("Jones", "Whittling", "114 Main Street"),
("Bravo", "Light Cleaning", "73 Industrial Way"),
("Ellis", "Alchemy", "73 Industrial Way"),
("Ellis", "Flying", "73 Industrial Way"),
("Harrison", "Light Cleaning", "73 Industrial Way"),
)
# N.B., we do expect rows will get sorted
expect_left = (
("Employee", "Current Work Location"),
("Bravo", "73 Industrial Way"),
("Ellis", "73 Industrial Way"),
("Harrison", "73 Industrial Way"),
("Jones", "114 Main Street"),
)
expect_right = (
("Employee", "Skill"),
("Bravo", "Light Cleaning"),
("Ellis", "Alchemy"),
("Ellis", "Flying"),
("Harrison", "Light Cleaning"),
("Jones", "Shorthand"),
("Jones", "Typing"),
("Jones", "Whittling"),
)
left, right = unjoin(table3, "Skill", key="Employee")
ieq(expect_left, left)
ieq(expect_left, left)
ieq(expect_right, right)
ieq(expect_right, right)
def test_unjoin_explicit_key():
# test the case where the join key is still present
table2 = (
("Customer ID", "First Name", "Surname", "Telephone Number"),
(123, "Robert", "Ingram", "555-861-2025"),
(456, "Jane", "Wright", "555-403-1659"),
(456, "Jane", "Wright", "555-776-4100"),
(789, "Maria", "Fernandez", "555-808-9633"),
)
expect_left = (
("Customer ID", "First Name", "Surname"),
(123, "Robert", "Ingram"),
(456, "Jane", "Wright"),
(789, "Maria", "Fernandez"),
)
expect_right = (
("Customer ID", "Telephone Number"),
(123, "555-861-2025"),
(456, "555-403-1659"),
(456, "555-776-4100"),
(789, "555-808-9633"),
)
left, right = unjoin(table2, "Telephone Number", key="Customer ID")
ieq(expect_left, left)
ieq(expect_left, left)
ieq(expect_right, right)
ieq(expect_right, right)
def test_unjoin_explicit_key_5():
table6 = (('ColA', 'ColB', 'ColC'),
('A', 1, 'apple'),
('B', 1, 'apple'),
('C', 2, 'orange'),
('D', 3, 'lemon'),
('E', 3, 'lemon'))
# N.B., we do expect rows will get sorted
expect_left = (('ColA', 'ColB'),
('A', 1),
('B', 1),
('C', 2),
('D', 3),
('E', 3))
expect_right = (('ColB', 'ColC'),
(1, 'apple'),
(2, 'orange'),
(3, 'lemon'))
left, right = unjoin(table6, 'ColC', key='ColB')
ieq(expect_left, left)
ieq(expect_left, left)
ieq(expect_right, right)
ieq(expect_right, right)
def test_unjoin_explicit_key_2():
table3 = (('Employee', 'Skill', 'Current Work Location'),
('Jones', 'Typing', '114 Main Street'),
('Jones', 'Shorthand', '114 Main Street'),
('Jones', 'Whittling', '114 Main Street'),
('Bravo', 'Light Cleaning', '73 Industrial Way'),
('Ellis', 'Alchemy', '73 Industrial Way'),
('Ellis', 'Flying', '73 Industrial Way'),
('Harrison', 'Light Cleaning', '73 Industrial Way'))
# N.B., we do expect rows will get sorted
expect_left = (('Employee', 'Current Work Location'),
('Bravo', '73 Industrial Way'),
('Ellis', '73 Industrial Way'),
('Harrison', '73 Industrial Way'),
('Jones', '114 Main Street'))
expect_right = (('Employee', 'Skill'),
('Bravo', 'Light Cleaning'),
('Ellis', 'Alchemy'),
('Ellis', 'Flying'),
('Harrison', 'Light Cleaning'),
('Jones', 'Shorthand'),
('Jones', 'Typing'),
('Jones', 'Whittling'))
left, right = unjoin(table3, 'Skill', key='Employee')
ieq(expect_left, left)
ieq(expect_left, left)
ieq(expect_right, right)
ieq(expect_right, right)
def test_unjoin_implicit_key():
# test the case where the join key needs to be reconstructed
table1 = (('foo', 'bar'), (1, 'apple'), (2, 'apple'), (3, 'orange'))
expect_left = (('foo', 'bar_id'), (1, 1), (2, 1), (3, 2))
expect_right = (('id', 'bar'), (1, 'apple'), (2, 'orange'))
left, right = unjoin(table1, 'bar')
ieq(expect_left, left)
ieq(expect_left, left)
ieq(expect_right, right)
ieq(expect_right, right)
def test_unjoin_implicit_key():
# test the case where the join key needs to be reconstructed
table1 = (("foo", "bar"), (1, "apple"), (2, "apple"), (3, "orange"))
expect_left = (("foo", "bar_id"), (1, 1), (2, 1), (3, 2))
expect_right = (("id", "bar"), (1, "apple"), (2, "orange"))
left, right = unjoin(table1, "bar")
ieq(expect_left, left)
ieq(expect_left, left)
ieq(expect_right, right)
ieq(expect_right, right)
def test_unjoin_explicit_key_5():
table6 = (
("ColA", "ColB", "ColC"),
("A", 1, "apple"),
("B", 1, "apple"),
("C", 2, "orange"),
("D", 3, "lemon"),
("E", 3, "lemon"),
)
# N.B., we do expect rows will get sorted
expect_left = (("ColA", "ColB"), ("A", 1), ("B", 1), ("C", 2), ("D", 3), ("E", 3))
expect_right = (("ColB", "ColC"), (1, "apple"), (2, "orange"), (3, "lemon"))
left, right = unjoin(table6, "ColC", key="ColB")
ieq(expect_left, left)
ieq(expect_left, left)
ieq(expect_right, right)
ieq(expect_right, right)
def test_unjoin_explicit_key_4():
table5 = (
("Restaurant", "Pizza Variety", "Delivery Area"),
("A1 Pizza", "Thick Crust", "Springfield"),
("A1 Pizza", "Thick Crust", "Shelbyville"),
("A1 Pizza", "Thick Crust", "Capital City"),
("A1 Pizza", "Stuffed Crust", "Springfield"),
("A1 Pizza", "Stuffed Crust", "Shelbyville"),
("A1 Pizza", "Stuffed Crust", "Capital City"),
("Elite Pizza", "Thin Crust", "Capital City"),
("Elite Pizza", "Stuffed Crust", "Capital City"),
("Vincenzo's Pizza", "Thick Crust", "Springfield"),
("Vincenzo's Pizza", "Thick Crust", "Shelbyville"),
("Vincenzo's Pizza", "Thin Crust", "Springfield"),
("Vincenzo's Pizza", "Thin Crust", "Shelbyville"),
)
# N.B., we do expect rows will get sorted
expect_left = (
("Restaurant", "Pizza Variety"),
("A1 Pizza", "Stuffed Crust"),
("A1 Pizza", "Thick Crust"),
("Elite Pizza", "Stuffed Crust"),
("Elite Pizza", "Thin Crust"),
("Vincenzo's Pizza", "Thick Crust"),
("Vincenzo's Pizza", "Thin Crust"),
)
expect_right = (
("Restaurant", "Delivery Area"),
("A1 Pizza", "Capital City"),
("A1 Pizza", "Shelbyville"),
("A1 Pizza", "Springfield"),
("Elite Pizza", "Capital City"),
("Vincenzo's Pizza", "Shelbyville"),
("Vincenzo's Pizza", "Springfield"),
)
left, right = unjoin(table5, "Delivery Area", key="Restaurant")
ieq(expect_left, left)
ieq(expect_left, left)
ieq(expect_right, right)
ieq(expect_right, right)
def test_unjoin_explicit_key():
# test the case where the join key is still present
table2 = (('Customer ID', 'First Name', 'Surname',
'Telephone Number'), (123, 'Robert', 'Ingram', '555-861-2025'),
(456, 'Jane', 'Wright', '555-403-1659'), (456, 'Jane', 'Wright',
'555-776-4100'),
(789, 'Maria', 'Fernandez', '555-808-9633'))
expect_left = (('Customer ID', 'First Name', 'Surname'), (123, 'Robert',
'Ingram'),
(456, 'Jane', 'Wright'), (789, 'Maria', 'Fernandez'))
expect_right = (('Customer ID', 'Telephone Number'), (123, '555-861-2025'),
(456, '555-403-1659'), (456, '555-776-4100'),
(789, '555-808-9633'))
left, right = unjoin(table2, 'Telephone Number', key='Customer ID')
ieq(expect_left, left)
ieq(expect_left, left)
ieq(expect_right, right)
ieq(expect_right, right)
def test_unjoin_implicit_key():
# test the case where the join key needs to be reconstructed
table1 = (('foo', 'bar'),
(1, 'apple'),
(2, 'apple'),
(3, 'orange'))
expect_left = (('foo', 'bar_id'),
(1, 1),
(2, 1),
(3, 2))
expect_right = (('id', 'bar'),
(1, 'apple'),
(2, 'orange'))
left, right = unjoin(table1, 'bar')
ieq(expect_left, left)
ieq(expect_left, left)
ieq(expect_right, right)
ieq(expect_right, right)
def test_unjoin_explicit_key_3():
table4 = (('Tournament', 'Year', 'Winner', 'Date of Birth'),
('Indiana Invitational', 1998, 'Al Fredrickson', '21 July 1975'),
('Cleveland Open', 1999, 'Bob Albertson', '28 September 1968'),
('Des Moines Masters', 1999, 'Al Fredrickson', '21 July 1975'),
('Indiana Invitational', 1999, 'Chip Masterson', '14 March 1977'))
# N.B., we do expect rows will get sorted
expect_left = (('Tournament', 'Year', 'Winner'),
('Cleveland Open', 1999, 'Bob Albertson'),
('Des Moines Masters', 1999, 'Al Fredrickson'),
('Indiana Invitational', 1998, 'Al Fredrickson'),
('Indiana Invitational', 1999, 'Chip Masterson'))
expect_right = (('Winner', 'Date of Birth'),
('Al Fredrickson', '21 July 1975'),
('Bob Albertson', '28 September 1968'),
('Chip Masterson', '14 March 1977'))
left, right = unjoin(table4, 'Date of Birth', key='Winner')
ieq(expect_left, left)
ieq(expect_left, left)
ieq(expect_right, right)
ieq(expect_right, right)
def test_unjoin_explicit_key_4():
table5 = (('Restaurant', 'Pizza Variety',
'Delivery Area'), ('A1 Pizza', 'Thick Crust', 'Springfield'),
('A1 Pizza', 'Thick Crust',
'Shelbyville'), ('A1 Pizza', 'Thick Crust', 'Capital City'),
('A1 Pizza', 'Stuffed Crust',
'Springfield'), ('A1 Pizza', 'Stuffed Crust', 'Shelbyville'),
('A1 Pizza', 'Stuffed Crust',
'Capital City'), ('Elite Pizza', 'Thin Crust', 'Capital City'),
('Elite Pizza', 'Stuffed Crust',
'Capital City'), ("Vincenzo's Pizza", "Thick Crust",
"Springfield"),
("Vincenzo's Pizza", "Thick Crust",
"Shelbyville"), ("Vincenzo's Pizza", "Thin Crust",
"Springfield"), ("Vincenzo's Pizza",
"Thin Crust", "Shelbyville"))
# N.B., we do expect rows will get sorted
expect_left = (('Restaurant', 'Pizza Variety'),
('A1 Pizza', 'Stuffed Crust'), ('A1 Pizza', 'Thick Crust'),
('Elite Pizza', 'Stuffed Crust'), ('Elite Pizza',
'Thin Crust'),
("Vincenzo's Pizza", "Thick Crust"), ("Vincenzo's Pizza",
"Thin Crust"))
expect_right = (('Restaurant', 'Delivery Area'),
('A1 Pizza', 'Capital City'), ('A1 Pizza', 'Shelbyville'),
('A1 Pizza', 'Springfield'), ('Elite Pizza',
'Capital City'),
("Vincenzo's Pizza", "Shelbyville"), ("Vincenzo's Pizza",
"Springfield"))
left, right = unjoin(table5, 'Delivery Area', key='Restaurant')
ieq(expect_left, left)
ieq(expect_left, left)
ieq(expect_right, right)
ieq(expect_right, right)
def test_unjoin_explicit_key():
# test the case where the join key is still present
table2 = (('Customer ID', 'First Name', 'Surname', 'Telephone Number'),
(123, 'Robert', 'Ingram', '555-861-2025'),
(456, 'Jane', 'Wright', '555-403-1659'),
(456, 'Jane', 'Wright', '555-776-4100'),
(789, 'Maria', 'Fernandez', '555-808-9633'))
expect_left = (('Customer ID', 'First Name', 'Surname'),
(123, 'Robert', 'Ingram'),
(456, 'Jane', 'Wright'),
(789, 'Maria', 'Fernandez'))
expect_right = (('Customer ID', 'Telephone Number'),
(123, '555-861-2025'),
(456, '555-403-1659'),
(456, '555-776-4100'),
(789, '555-808-9633'))
left, right = unjoin(table2, 'Telephone Number', key='Customer ID')
ieq(expect_left, left)
ieq(expect_left, left)
ieq(expect_right, right)
ieq(expect_right, right)
def test_unjoin_explicit_key_4():
table5 = (('Restaurant', 'Pizza Variety', 'Delivery Area'),
('A1 Pizza', 'Thick Crust', 'Springfield'),
('A1 Pizza', 'Thick Crust', 'Shelbyville'),
('A1 Pizza', 'Thick Crust', 'Capital City'),
('A1 Pizza', 'Stuffed Crust', 'Springfield'),
('A1 Pizza', 'Stuffed Crust', 'Shelbyville'),
('A1 Pizza', 'Stuffed Crust', 'Capital City'),
('Elite Pizza', 'Thin Crust', 'Capital City'),
('Elite Pizza', 'Stuffed Crust', 'Capital City'),
("Vincenzo's Pizza", "Thick Crust", "Springfield"),
("Vincenzo's Pizza", "Thick Crust", "Shelbyville"),
("Vincenzo's Pizza", "Thin Crust", "Springfield"),
("Vincenzo's Pizza", "Thin Crust", "Shelbyville"))
# N.B., we do expect rows will get sorted
expect_left = (('Restaurant', 'Pizza Variety'),
('A1 Pizza', 'Stuffed Crust'),
('A1 Pizza', 'Thick Crust'),
('Elite Pizza', 'Stuffed Crust'),
('Elite Pizza', 'Thin Crust'),
("Vincenzo's Pizza", "Thick Crust"),
("Vincenzo's Pizza", "Thin Crust"))
expect_right = (('Restaurant', 'Delivery Area'),
('A1 Pizza', 'Capital City'),
('A1 Pizza', 'Shelbyville'),
('A1 Pizza', 'Springfield'),
('Elite Pizza', 'Capital City'),
("Vincenzo's Pizza", "Shelbyville"),
("Vincenzo's Pizza", "Springfield"))
left, right = unjoin(table5, 'Delivery Area', key='Restaurant')
ieq(expect_left, left)
ieq(expect_left, left)
ieq(expect_right, right)
ieq(expect_right, right)
(4, 2, 17),
(2, 7, 3),
(1, 6, 1))
from petl import look, multirangeaggregate
look(table1)
table2 = multirangeaggregate(table1, keys=('x', 'y'), widths=(2, 2), aggregation=sum, mins=(0, 0), maxs=(4, 4), value='z')
look(table2)
# unjoin
table1 = (('foo', 'bar', 'baz'),
('A', 1, 'apple'),
('B', 1, 'apple'),
('C', 2, 'orange'))
table4 = (('foo', 'bar'),
('A', 'apple'),
('B', 'apple'),
('C', 'orange'))
from petl import look, unjoin
look(table1)
table2, table3 = unjoin(table1, 'baz', key='bar')
look(table2)
look(table3)
look(table4)
table5, table6 = unjoin(table4, 'bar')
look(table5)
look(table6)
def append_tailings_reports_to_code_required_reports(connection, commit=False):
src_table = etl.fromdb(
connection,
'SELECT exp_doc.mine_guid, exp_doc.exp_document_guid, req_doc.req_document_name, exp_doc.due_date, exp_doc.exp_document_status_code, exp_doc.received_date, exp_doc.active_ind, exp_doc_x.mine_document_guid, exp_doc.create_user, exp_doc.create_timestamp, exp_doc.update_user, exp_doc.update_timestamp from mine_expected_document exp_doc \
inner join mine_expected_document_xref exp_doc_x on exp_doc.exp_document_guid = exp_doc_x.exp_document_guid\
inner join mds_required_document req_doc on req_doc.req_document_guid = exp_doc.req_document_guid'
)
req_document_crr_defintion_map = [
['req_document_name', 'mine_report_definition_id'],
['Summary of TSF and Dam Safety Recommendations', 28],
['ITRB Activities Report', 27],
['Register of Tailings Storage Facilities and Dams', 47],
['Dam Safety Inspection (DSI) Report', 26],
['Dam Safety Review (DSR) Report', 31],
['“As-built” Reports', 32],
['Annual Reclamation', 25],
['MERP Record of Testing', 3],
#['Annual Manager\'s Report', __________________ ], no mapping or data, ignore.
['OMS Manual', 33],
['Annual reconciliation of water balance and water management plans', 44],
['TSF risk assessment', 46],
['Mine Emergency Preparedness and Response Plan (MERP)', 24],
['Performance of high risk dumps', 29]
]
table1 = etl.join(src_table, req_document_crr_defintion_map, 'req_document_name')
mine_report = etl.cutout(table1, 'req_document_name')
#to be inserted into db
mine_report = etl.addfield(mine_report, 'submission_year', 2019)
mine_report = etl.rename(mine_report, 'exp_document_status_code',
'mine_report_submission_status_code')
mine_report = etl.addfield(mine_report, 'deleted_ind', lambda x: not x.active_ind)
mine_report = etl.cutout(mine_report, 'active_ind')
#to determine what FK's will be so can insert into related tables
max_report_id = etl.fromdb(connection,
'select last_value from public.mine_report_mine_report_id_seq')[1][0]
max_report_submission_id = etl.fromdb(
connection,
'select last_value from public.mine_report_submission_mine_report_submission_id_seq')[1][0]
#if sequence hasn't been used yet, fix off by one
if max_report_id == 1:
max_report_id = 0
if max_report_submission_id == 1:
max_report_submission_id = 0
#get one-to-many
mine_report, mine_report_submission_documents = etl.unjoin(mine_report,
'mine_document_guid',
key='exp_document_guid')
#add PK's for mappings
mine_report_with_ids = etl.addrownumbers(mine_report,
start=max_report_id + 1,
step=1,
field='mine_report_id')
mine_report_with_ids = etl.addrownumbers(mine_report_with_ids,
start=max_report_submission_id + 1,
step=1,
field='mine_report_submission_id')
print(f'max_report_id= {max_report_id}, max_report_submission_id={max_report_submission_id}')
#copy out fields for submission tables
mine_report_submissions = etl.cut(mine_report_with_ids, [
'mine_report_id', 'exp_document_guid', 'mine_report_submission_status_code', 'create_user',
'create_timestamp', 'update_user', 'update_timestamp'
])
mine_report_submissions = etl.addfield(mine_report_submissions,
'submission_date', lambda x: x.create_timestamp)
#remove fields not in mine_report
mine_report = etl.cutout(mine_report, 'mine_report_submission_status_code')
#replace exp_document_guid FK with mine_report_submission FK
submission_id_lookup = etl.cut(mine_report_with_ids,
['mine_report_submission_id', 'exp_document_guid'])
mine_report_submission_documents = etl.join(submission_id_lookup,
mine_report_submission_documents,
key='exp_document_guid')
mine_report_submission_documents = etl.cutout(mine_report_submission_documents,
'exp_document_guid')
#removed original PK
mine_report = etl.cutout(mine_report, 'exp_document_guid')
mine_report_submissions = etl.cutout(mine_report_submissions, 'exp_document_guid')
print(etl.valuecounter(etl.distinct(table1, key='exp_document_guid'), 'req_document_name'))
print(etl.valuecounter(mine_report, 'mine_report_definition_id'))
print(table1)
print(mine_report)
print(mine_report_submissions)
print(mine_report_submission_documents)
etl.appenddb(mine_report, connection, 'mine_report', commit=False)
print('INSERT mine_report staged')
etl.appenddb(mine_report_submissions, connection, 'mine_report_submission', commit=False)
print('INSERT mine_report_submission staged')
etl.appenddb(mine_report_submission_documents,
connection,
'mine_report_document_xref',
commit=False)
print('INSERT mine_report_document_xref staged')
if commit:
connection.commit()
print('DATA CREATION COMPLETE')
else:
connection.rollback()
print('NO DATA CREATED: add --commit=true to insert report rows')
(4, 2, 17),
(2, 7, 3),
(1, 6, 1))
from petl import look, multirangeaggregate
look(table1)
table2 = multirangeaggregate(table1, keys=('x', 'y'), widths=(2, 2), aggregation=sum, mins=(0, 0), maxs=(4, 4), value='z')
look(table2)
# unjoin
table1 = (('foo', 'bar', 'baz'),
('A', 1, 'apple'),
('B', 1, 'apple'),
('C', 2, 'orange'))
table4 = (('foo', 'bar'),
('A', 'apple'),
('B', 'apple'),
('C', 'orange'))
from petl import look, unjoin
look(table1)
table2, table3 = unjoin(table1, 'baz', key='bar')
look(table2)
look(table3)
look(table4)
table5, table6 = unjoin(table4, 'bar')
look(table5)
look(table6)
table = etl.addfield(table, 'create_user', 'MMS_DO_IMPORT')
table = etl.addfield(table, 'update_user', 'MMS_DO_IMPORT')
#RENAME SOURCE COLUMNS WE WANT TO PRESERVE
print("RENAME insp_cd to mms_insp_cd")
table = etl.rename(table, 'insp_cd', 'mms_insp_cd')
print("RENAME min_acc_no to mine_incident_no")
table = etl.rename(table, 'min_acc_no', 'mine_incident_no')
#force id column SQL will reset the sequence
table = etl.addrownumbers(table, field='mine_incident_id')
table = etl.sort(table, 'incident_timestamp', reverse=True)
print('UNJOIN Recommendations into separate table')
table, recommendation_table = etl.unjoin(table,
'recommendation',
key='mine_incident_id')
recommendation_table = etl.select(recommendation_table, 'recommendation',
lambda x: x is not None and not x.isspace())
recommendation_table = etl.addfield(recommendation_table, 'create_user',
'MMS_DO_IMPORT')
recommendation_table = etl.addfield(recommendation_table, 'update_user',
'MMS_DO_IMPORT')
print("TRUNCATE public.mine_incident_recommendation")
connection.cursor().execute(
'TRUNCATE TABLE public.mine_incident_recommendation;')
print("TRUNCATE AND LOAD public.mine_incident")
etl.todb(table, connection, 'mine_incident')
