def test_valuecounter_shortrows():
"""Test the valuecounter function."""
table = (('foo', 'bar'), ('a', 7), ('b', ), ('b', 7))
actual = valuecounter(table, 'foo')
expect = {'b': 2, 'a': 1}
eq_(expect, actual)
actual = valuecounter(table, 'bar')
expect = {7: 2, None: 1}
eq_(expect, actual)
actual = valuecounter(table, 'foo', 'bar')
expect = {('a', 7): 1, ('b', None): 1, ('b', 7): 1}
eq_(expect, actual)
def test_valuecounter_shortrows():
"""Test the valuecounter function."""
table = (('foo', 'bar'), ('a', 7), ('b',), ('b', 7))
actual = valuecounter(table, 'foo')
expect = {'b': 2, 'a': 1}
eq_(expect, actual)
actual = valuecounter(table, 'bar')
expect = {7: 2, None: 1}
eq_(expect, actual)
actual = valuecounter(table, 'foo', 'bar')
expect = {('a', 7): 1, ('b', None): 1, ('b', 7): 1}
eq_(expect, actual)
def test_valuecounter_shortrows():
"""Test the valuecounter function."""
table = (("foo", "bar"), ("a", 7), ("b",), ("b", 7))
actual = valuecounter(table, "foo")
expect = {"b": 2, "a": 1}
eq_(expect, actual)
actual = valuecounter(table, "bar")
expect = {7: 2, None: 1}
eq_(expect, actual)
actual = valuecounter(table, "foo", "bar")
expect = {("a", 7): 1, ("b", None): 1, ("b", 7): 1}
eq_(expect, actual)
def test_valuecounter():
"""Test the valuecounter function."""
table = (('foo', 'bar'), ('a', 1), ('b', 2), ('b', 7))
actual = valuecounter(table, 'foo')
expect = {'b': 2, 'a': 1}
eq_(expect, actual)
def test_valuecounter():
"""Test the valuecounter function."""
table = (('foo', 'bar'), ('a', 1), ('b', 2), ('b', 7))
actual = valuecounter(table, 'foo')
expect = {'b': 2, 'a': 1}
eq_(expect, actual)
def test_valuecounter():
"""Test the valuecounter function."""
table = (("foo", "bar"), ("a", 1), ("b", 2), ("b", 7))
actual = valuecounter(table, "foo")
expect = {"b": 2, "a": 1}
eq_(expect, actual)
def collection_count(request, pk):
collection = Collection.objects.get(pk=pk)
table = collection.data
if request.method == "GET":
fields = []
button_states = {}
header, table = table[0], table[1:]
else:
fields = [f for f in FIELDS if f in request.POST]
if fields:
table = petl.cut(table, *fields)
header, counts = list(table[0]) + ["Count"], petl.valuecounter(
table, *fields)
table = []
for k, v in counts.items():
if isinstance(k, str):
table.append([k] + [v])
else:
table.append(list(k) + [v])
else:
header, table = table[0], table[1:]
button_states = {
f"{f}_checked": True
for f in FIELDS if f in request.POST
}
return render(
request,
"api/collection_detail.html",
{
"header": header,
"table": table,
"collection_id": collection.id,
"show_load_more": False,
**button_states,
},
)
['a', 1],
['b', 2],
['b', 7]]
etl.valuecount(table, 'foo', 'b')
# valuecounter()
################
import petl as etl
table = [['foo', 'bar'],
['a', True],
['b'],
['b', True],
['c', False]]
etl.valuecounter(table, 'foo').most_common()
# valuecounts()
###############
import petl as etl
table = [['foo', 'bar', 'baz'],
['a', True, 0.12],
['a', True, 0.17],
['b', False, 0.34],
['b', False, 0.44],
['b']]
etl.valuecounts(table, 'foo')
etl.valuecounts(table, 'foo', 'bar')
def append_tailings_reports_to_code_required_reports(connection, commit=False):
src_table = etl.fromdb(
connection,
'SELECT exp_doc.mine_guid, exp_doc.exp_document_guid, req_doc.req_document_name, exp_doc.due_date, exp_doc.exp_document_status_code, exp_doc.received_date, exp_doc.active_ind, exp_doc_x.mine_document_guid, exp_doc.create_user, exp_doc.create_timestamp, exp_doc.update_user, exp_doc.update_timestamp from mine_expected_document exp_doc \
inner join mine_expected_document_xref exp_doc_x on exp_doc.exp_document_guid = exp_doc_x.exp_document_guid\
inner join mds_required_document req_doc on req_doc.req_document_guid = exp_doc.req_document_guid'
)
req_document_crr_defintion_map = [
['req_document_name', 'mine_report_definition_id'],
['Summary of TSF and Dam Safety Recommendations', 28],
['ITRB Activities Report', 27],
['Register of Tailings Storage Facilities and Dams', 47],
['Dam Safety Inspection (DSI) Report', 26],
['Dam Safety Review (DSR) Report', 31],
['“As-built” Reports', 32],
['Annual Reclamation', 25],
['MERP Record of Testing', 3],
#['Annual Manager\'s Report', __________________ ], no mapping or data, ignore.
['OMS Manual', 33],
['Annual reconciliation of water balance and water management plans', 44],
['TSF risk assessment', 46],
['Mine Emergency Preparedness and Response Plan (MERP)', 24],
['Performance of high risk dumps', 29]
]
table1 = etl.join(src_table, req_document_crr_defintion_map, 'req_document_name')
mine_report = etl.cutout(table1, 'req_document_name')
#to be inserted into db
mine_report = etl.addfield(mine_report, 'submission_year', 2019)
mine_report = etl.rename(mine_report, 'exp_document_status_code',
'mine_report_submission_status_code')
mine_report = etl.addfield(mine_report, 'deleted_ind', lambda x: not x.active_ind)
mine_report = etl.cutout(mine_report, 'active_ind')
#to determine what FK's will be so can insert into related tables
max_report_id = etl.fromdb(connection,
'select last_value from public.mine_report_mine_report_id_seq')[1][0]
max_report_submission_id = etl.fromdb(
connection,
'select last_value from public.mine_report_submission_mine_report_submission_id_seq')[1][0]
#if sequence hasn't been used yet, fix off by one
if max_report_id == 1:
max_report_id = 0
if max_report_submission_id == 1:
max_report_submission_id = 0
#get one-to-many
mine_report, mine_report_submission_documents = etl.unjoin(mine_report,
'mine_document_guid',
key='exp_document_guid')
#add PK's for mappings
mine_report_with_ids = etl.addrownumbers(mine_report,
start=max_report_id + 1,
step=1,
field='mine_report_id')
mine_report_with_ids = etl.addrownumbers(mine_report_with_ids,
start=max_report_submission_id + 1,
step=1,
field='mine_report_submission_id')
print(f'max_report_id= {max_report_id}, max_report_submission_id={max_report_submission_id}')
#copy out fields for submission tables
mine_report_submissions = etl.cut(mine_report_with_ids, [
'mine_report_id', 'exp_document_guid', 'mine_report_submission_status_code', 'create_user',
'create_timestamp', 'update_user', 'update_timestamp'
])
mine_report_submissions = etl.addfield(mine_report_submissions,
'submission_date', lambda x: x.create_timestamp)
#remove fields not in mine_report
mine_report = etl.cutout(mine_report, 'mine_report_submission_status_code')
#replace exp_document_guid FK with mine_report_submission FK
submission_id_lookup = etl.cut(mine_report_with_ids,
['mine_report_submission_id', 'exp_document_guid'])
mine_report_submission_documents = etl.join(submission_id_lookup,
mine_report_submission_documents,
key='exp_document_guid')
mine_report_submission_documents = etl.cutout(mine_report_submission_documents,
'exp_document_guid')
#removed original PK
mine_report = etl.cutout(mine_report, 'exp_document_guid')
mine_report_submissions = etl.cutout(mine_report_submissions, 'exp_document_guid')
print(etl.valuecounter(etl.distinct(table1, key='exp_document_guid'), 'req_document_name'))
print(etl.valuecounter(mine_report, 'mine_report_definition_id'))
print(table1)
print(mine_report)
print(mine_report_submissions)
print(mine_report_submission_documents)
etl.appenddb(mine_report, connection, 'mine_report', commit=False)
print('INSERT mine_report staged')
etl.appenddb(mine_report_submissions, connection, 'mine_report_submission', commit=False)
print('INSERT mine_report_submission staged')
etl.appenddb(mine_report_submission_documents,
connection,
'mine_report_document_xref',
commit=False)
print('INSERT mine_report_document_xref staged')
if commit:
connection.commit()
print('DATA CREATION COMPLETE')
else:
connection.rollback()
print('NO DATA CREATED: add --commit=true to insert report rows')
if etl.valuecount(table, 'mine_guid', None)[0] > 0:
print('mine_guid, mine_no pair missing from mines.csv')
exit(1)
######
print('CONVERT AND RENAME descript1 to recommendation')
table = etl.addfield(table, 'recommendation', lambda x: x['descript1'])
table = clean_up(table, 'descript1')
######
print('CONVERTING sta_cd to status_code')
table = etl.addfield(table, 'status_code', lambda x: x['sta_cd'])
table = etl.convert(table, 'status_code', 'replace', 'O', 'F')
table = etl.convert(table, 'status_code', 'replace', 'F', 'FIN')
table = etl.convert(table, 'status_code', 'replace', 'P', 'PRE')
print(etl.valuecounter(table, 'sta_cd'))
print(etl.valuecounter(table, 'status_code'))
table = clean_up(table, 'sta_cd')
######
print('CONVERT AND RENAME rep_nm to reported_by_name')
table = etl.addfield(table, 'reported_by_name', lambda x: x['rep_nm'])
table = clean_up(table, 'rep_nm')
######
print('CONVERT AND RENAME descript to incident_description')
table = etl.addfield(table, 'incident_description', lambda x: x['descript'])
table = clean_up(table, 'descript')
######
print('COMBINING occ_dt and occ_tm into incident_timestamp')