def _changed_fields(self, obj_type, fields, local_obj, api_obj):
tmap = {
'cp': {
'name': "COUNTRY_PROGRAMME_NAME",
'wbs': "COUNTRY_PROGRAMME_WBS",
'from_date': 'CP_START_DATE',
'to_date': 'CP_END_DATE'
},
'outcome': {
'name': "OUTCOME_DESCRIPTION",
'wbs': "OUTCOME_WBS",
'from_date': 'OUTCOME_START_DATE',
'to_date': 'OUTCOME_END_DATE'
},
'output': {
'name': "OUTPUT_DESCRIPTION",
'wbs': "OUTPUT_WBS",
'from_date': 'OUTPUT_START_DATE',
'to_date': 'OUTPUT_END_DATE'
},
'activity': {
'name': "ACTIVITY_DESCRIPTION",
'wbs': "ACTIVITY_WBS",
'from_date': 'ACTIVITY_START_DATE',
'to_date': 'ACTIVITY_END_DATE',
'sic_code': 'SIC_CODE',
'sic_name': 'SIC_NAME',
'gic_code': 'GIC_CODE',
'gic_name': 'GIC_NAME',
'activity_focus_code': 'ACTIVITY_FOCUS_CODE',
'activity_focus_name': 'ACTIVITY_FOCUS_NAME',
}
}
mapping = tmap[obj_type]
for field in fields:
apiobj_field = api_obj[mapping[field]]
if field.endswith('date'):
if not wcf_json_date_as_datetime(api_obj[mapping[field]]):
apiobj_field = None
else:
apiobj_field = wcf_json_date_as_datetime(
api_obj[mapping[field]]).date()
if getattr(local_obj, field) != apiobj_field:
print "field changed", field, obj_type, getattr(
local_obj, field), 'Changed To', apiobj_field
return True
return False
def _changed_fields( fields, local_obj, api_obj):
for field in fields:
apiobj_field = api_obj[self.MAPPING[field]]
if field in ['fr_item_amount_usd','agreement_amount', 'commitment_amount', 'expenditure_amount']:
return not comp_decimals(getattr(local_obj, field), apiobj_field)
if field in ['start', 'end']:
if not wcf_json_date_as_datetime(api_obj[self.MAPPING[field]]):
apiobj_field = None
else:
apiobj_field = timezone.make_aware(wcf_json_date_as_datetime(api_obj[self.MAPPING[field]]),
timezone.get_default_timezone())
if field == 'fc_type':
apiobj_field = api_obj[self.MAPPING[field]] or 'No Record'
if getattr(local_obj, field) != apiobj_field:
print "field changed", field
return True
return False
def _save_records(self, records):
processed = 0
filtered_records = self._filter_records(records)
for partner in filtered_records:
try:
# Populate grants during import
donor = Donor.objects.get_or_create(name=partner["DONOR_NAME"])[0]
try:
grant = Grant.objects.get(name=partner["GRANT_REF"])
except Grant.DoesNotExist:
grant = Grant.objects.create(name=partner["GRANT_REF"], donor=donor)
else:
grant.donor = donor
grant.description = partner["GRANT_DESC"]
if partner["EXPIRY_DATE"] is not None:
grant.expiry = wcf_json_date_as_datetime(partner["EXPIRY_DATE"])
grant.save()
partner_org, created = PartnerOrganization.objects.get_or_create(
vendor_number=partner["VENDOR_CODE"]
)
partner_org.name = partner["VENDOR_NAME"]
partner_org.partner_type = type_mapping[partner["PARTNER_TYPE_DESC"]]
partner_org.cso_type = partner["CSO_TYPE_NAME"]
partner_org.rating = partner["RISK_RATING_NAME"]
partner_org.type_of_assessment = partner["TYPE_OF_ASSESSMENT"]
partner_org.last_assessment_date = wcf_json_date_as_datetime(partner["LAST_ASSESSMENT_DATE"])
partner_org.address = partner["STREET_ADDRESS"]
partner_org.phone_number = partner["PHONE_NUMBER"]
partner_org.email = partner["EMAIL"]
partner_org.core_values_assessment_date = wcf_json_date_as_datetime(partner["CORE_VALUE_ASSESSMENT_DT"])
partner_org.vision_synced = True
partner_org.save()
processed += 1
except Exception as exp:
print exp.message
continue
return processed
def _save_records(self, records):
processed = 0
filtered_records = self._filter_records(records)
for fc_line in filtered_records:
try:
grant = Grant.objects.get(
name=fc_line["GRANT_REF"],
)
except Grant.DoesNotExist:
print 'Grant: {} does not exist'.format(fc_line["GRANT_REF"])
else:
try:
funding_commitment, created = FundingCommitment.objects.get_or_create(
grant=grant,
fr_number=fc_line["FR_DOC_NUMBER"],
fc_ref=fc_line["COMMITMENT_REF"]
)
funding_commitment.start = wcf_json_date_as_datetime(fc_line["FR_START_DATE"])
funding_commitment.end = wcf_json_date_as_datetime(fc_line["FR_END_DATE"])
funding_commitment.wbs = fc_line["IR_WBS"]
funding_commitment.fc_type = fc_line["COMMITMENT_DOC_TYPE"]
funding_commitment.fr_item_amount_usd = fc_line["FR_ITEM_AMT"]
funding_commitment.agreement_amount = fc_line["AGREEMENT_AMT"]
funding_commitment.commitment_amount = fc_line["COMMITMENT_AMT"]
funding_commitment.expenditure_amount = fc_line["EXPENDITURE_AMT"]
try:
intervention = PCA.objects.get(fr_number=fc_line["FR_DOC_NUMBER"])
funding_commitment.intervention = intervention
except PCA.DoesNotExist:
pass
funding_commitment.save()
except FundingCommitment.MultipleObjectsReturned as exp:
exp.message += 'FC Ref ' + fc_line["COMMITMENT_REF"]
raise
processed += 1
return processed
def _changed_fields(fields, local_obj, api_obj):
for field in fields:
apiobj_field = api_obj[self.MAPPING[field]]
if field.endswith('date'):
if not wcf_json_date_as_datetime(
api_obj[self.MAPPING[field]]):
apiobj_field = None
else:
apiobj_field = wcf_json_date_as_datetime(
api_obj[self.MAPPING[field]]).date()
if field == 'partner_type':
apiobj_field = type_mapping[api_obj[self.MAPPING[field]]]
if field == 'deleted_flag':
apiobj_field = True if api_obj[
self.MAPPING[field]] else False
if getattr(local_obj, field) != apiobj_field:
print "field changed", field
return True
return False
def _process_po(po_api):
if po_api['VENDOR_CODE'] not in _vendors:
_pos.append(po_api)
_vendors.append(po_api['VENDOR_CODE'])
if not _donors.get(po_api["DONOR_NAME"], None):
temp_donor = Donor.objects.get_or_create(
name=po_api["DONOR_NAME"])[0]
_donors[po_api["DONOR_NAME"]] = temp_donor
donor_grant_pair = po_api["DONOR_NAME"] + po_api["GRANT_REF"]
if not _grants.get(donor_grant_pair, None):
try:
temp_grant = Grant.objects.get(name=po_api["GRANT_REF"])
except Grant.DoesNotExist:
temp_grant = Grant.objects.create(
name=po_api["GRANT_REF"],
donor=_donors[po_api["DONOR_NAME"]])
temp_grant.description = po_api["GRANT_DESC"]
if po_api["EXPIRY_DATE"] is not None:
temp_grant.expiry = wcf_json_date_as_datetime(
po_api["EXPIRY_DATE"])
temp_grant.save()
_grants[donor_grant_pair] = temp_grant
if not po_api["TOTAL_CASH_TRANSFERRED_CP"]:
po_api["TOTAL_CASH_TRANSFERRED_CP"] = 0
if not po_api["TOTAL_CASH_TRANSFERRED_CY"]:
po_api["TOTAL_CASH_TRANSFERRED_CY"] = 0
if not _totals_cp.get(po_api['VENDOR_CODE']):
_totals_cp[po_api['VENDOR_CODE']] = po_api[
"TOTAL_CASH_TRANSFERRED_CP"]
else:
_totals_cp[po_api['VENDOR_CODE']] += po_api[
"TOTAL_CASH_TRANSFERRED_CP"]
if not _totals_cy.get(po_api['VENDOR_CODE']):
_totals_cy[po_api['VENDOR_CODE']] = po_api[
"TOTAL_CASH_TRANSFERRED_CY"]
else:
_totals_cy[po_api['VENDOR_CODE']] += po_api[
"TOTAL_CASH_TRANSFERRED_CY"]
def _get_field_value(self, field_name, field_json_code, json_item, model):
result = None
if field_json_code in self.DATE_FIELDS:
# parsing field as date
return wcf_json_date_as_datetime(json_item[field_json_code])
elif field_name in self.MODEL_MAPPING.keys():
# this is related model, so we need to fetch somehow related object.
related_model = self.MODEL_MAPPING[field_name]
if isinstance(related_model, types.FunctionType):
# callable provided, object should be returned from it
result = related_model(data=json_item,
key_field=field_json_code)
else:
# model class provided, related object can be fetched with query by field
# analogue of field_json_code
reversed_dict = dict(
zip(self.MAPPING[field_name].values(),
self.MAPPING[field_name].keys()))
result = related_model.objects.get(
**{
reversed_dict[field_json_code]:
json_item.get(field_json_code, None)
})
else:
# field can be used as it is without custom mappings.
result = json_item.get(field_json_code, None)
# additional logic on field may be applied
value_handler = self.FIELD_HANDLERS.get({
y: x
for x, y in six.iteritems(self.MODEL_MAPPING)
}.get(model), {}).get(field_name, None)
if value_handler:
result = value_handler(result)
return result
def _save_records(self, records):
processed = 0
filtered_records = self._filter_records(records)
fetched_grants = {}
fcs = {}
def _changed_fields( fields, local_obj, api_obj):
for field in fields:
apiobj_field = api_obj[self.MAPPING[field]]
if field in ['fr_item_amount_usd','agreement_amount', 'commitment_amount', 'expenditure_amount']:
return not comp_decimals(getattr(local_obj, field), apiobj_field)
if field in ['start', 'end']:
if not wcf_json_date_as_datetime(api_obj[self.MAPPING[field]]):
apiobj_field = None
else:
apiobj_field = timezone.make_aware(wcf_json_date_as_datetime(api_obj[self.MAPPING[field]]),
timezone.get_default_timezone())
if field == 'fc_type':
apiobj_field = api_obj[self.MAPPING[field]] or 'No Record'
if getattr(local_obj, field) != apiobj_field:
print "field changed", field
return True
return False
for fc_line in filtered_records:
grant = None
saving = False
if fc_line['GRANT_REF'] == 'Unknown':
# This is a non-grant commitment
pass
else:
try:
grant = fetched_grants[fc_line["GRANT_REF"]]
except KeyError:
try:
grant = Grant.objects.get(
name=fc_line["GRANT_REF"],
)
except Grant.DoesNotExist:
print 'Grant: {} does not exist'.format(fc_line["GRANT_REF"])
continue
else:
fetched_grants[fc_line["GRANT_REF"]] = grant
try:
fc = fcs[fc_line["COMMITMENT_REF"]]
# if there are multiple fcs in the response it means their total needs to be aggregated
fc.expenditure_amount += fc_line.get("EXPENDITURE_AMT", 0)
except KeyError:
try:
fc, saving = FundingCommitment.objects.get_or_create(
grant=grant,
fr_number=fc_line["FR_DOC_NUMBER"],
fc_ref=fc_line["COMMITMENT_REF"]
)
except FundingCommitment.MultipleObjectsReturned as exp:
exp.message += 'FC Ref ' + fc_line["COMMITMENT_REF"]
raise
fc_fields = ['start', 'end', 'wbs', 'fc_type', 'fr_item_amount_usd',
'agreement_amount', 'commitment_amount', 'expenditure_amount']
if saving or _changed_fields(fc_fields, fc, fc_line):
fc.start = wcf_json_date_as_datetime(fc_line["FR_START_DATE"])
fc.end = wcf_json_date_as_datetime(fc_line["FR_END_DATE"])
fc.wbs = fc_line["IR_WBS"]
fc.fc_type = fc_line["COMMITMENT_DOC_TYPE"] or 'No Record'
fc.fr_item_amount_usd = fc_line["FR_ITEM_AMT"]
fc.agreement_amount = fc_line["AGREEMENT_AMT"]
fc.commitment_amount = fc_line["COMMITMENT_AMT"]
fc.expenditure_amount = fc_line["EXPENDITURE_AMT"]
fc.save()
processed += 1
return processed
def in_time_range(record):
end = wcf_json_date_as_datetime(record['OUTCOME_END_DATE'])
if end >= last_year:
return True
return False
def _save_records(self, records):
processed = 0
filtered_records = self._filter_records(records)
for result in filtered_records:
try:
result_structure, created = ResultStructure.objects.get_or_create(
name=result['COUNTRY_PROGRAMME_NAME'],
from_date=wcf_json_date_as_datetime(result['CP_START_DATE']),
to_date=wcf_json_date_as_datetime(result['CP_END_DATE']),
)
except ResultStructure.MultipleObjectsReturned as exp:
exp.message += 'Result Structure: ' + result['COUNTRY_PROGRAMME_NAME']
raise
try:
outcome, created = Result.objects.get_or_create(
result_structure=result_structure,
result_type=ResultType.objects.get_or_create(name='Outcome')[0],
wbs=result['OUTCOME_WBS'],
)
outcome.name = result['OUTCOME_DESCRIPTION']
outcome.from_date = wcf_json_date_as_datetime(result['OUTCOME_START_DATE'])
outcome.to_date = wcf_json_date_as_datetime(result['OUTCOME_END_DATE'])
outcome.save()
output, created = Result.objects.get_or_create(
result_structure=result_structure,
result_type=ResultType.objects.get_or_create(name='Output')[0],
wbs=result['OUTPUT_WBS'],
)
output.name = result['OUTPUT_DESCRIPTION']
output.from_date = wcf_json_date_as_datetime(result['OUTPUT_START_DATE'])
output.to_date = wcf_json_date_as_datetime(result['OUTPUT_END_DATE'])
output.parent = outcome
output.save()
activity, created = Result.objects.get_or_create(
result_structure=result_structure,
result_type=ResultType.objects.get_or_create(name='Activity')[0],
wbs=result['ACTIVITY_WBS'],
)
activity.name = result['ACTIVITY_DESCRIPTION']
activity.from_date = wcf_json_date_as_datetime(result['ACTIVITY_START_DATE'])
activity.to_date = wcf_json_date_as_datetime(result['ACTIVITY_END_DATE'])
activity.parent = output
activity.sic_code = result['SIC_CODE']
activity.sic_name = result['SIC_NAME']
activity.gic_code = result['GIC_CODE']
activity.gic_name = result['GIC_NAME']
activity.activity_focus_code = result['ACTIVITY_FOCUS_CODE']
activity.activity_focus_name = result['ACTIVITY_FOCUS_NAME']
activity.save()
processed += 1
except Result.MultipleObjectsReturned as exp:
exp.message += 'Outcome WBS: ' + result['OUTCOME_WBS'] \
+ ' Output WBS: ' + result['OUTPUT_WBS'] \
+ ' Activity WBS: ' + result['ACTIVITY_WBS']
raise
return processed
def test_datetime_positive_sign(self):
date = "/Date(00000001+1000)/"
result = utils.wcf_json_date_as_datetime(date)
self.assertEqual(result, datetime.datetime(1970, 1, 1, 10, 0, 0, 1000))
def test_datetime(self):
date = "/Date(1361336400000)/"
result = utils.wcf_json_date_as_datetime(date)
self.assertEqual(result, datetime.datetime(2013, 2, 20, 5, 0))
def test_datetime_negative_sign(self):
date = "/Date(00000001-1000)/"
result = utils.wcf_json_date_as_datetime(date)
self.assertEqual(result, datetime.datetime(1969, 12, 31, 14, 0, 0,
1000))
def in_time_range(record):
end = wcf_json_date_as_datetime(record['OUTCOME_END_DATE'])
if end >= last_year:
return True
return False
def _save_records(self, records):
processed = 0
filtered_records = self._filter_records(records)
cps = {}
outcomes = {}
outputs = {}
activities = {}
for result in filtered_records:
# Find the country programme:
updating_cp = False
country_programme = cps.get(result["COUNTRY_PROGRAMME_WBS"], None)
if not country_programme:
try:
country_programme = CountryProgramme.objects.get(
wbs=result["COUNTRY_PROGRAMME_WBS"], )
except CountryProgramme.DoesNotExist:
country_programme = CountryProgramme(
wbs=result["COUNTRY_PROGRAMME_WBS"])
updating_cp = True
except CountryProgramme.MultipleObjectsReturned as exp:
exp.message += 'Result Structure: ' + result[
'COUNTRY_PROGRAMME_NAME']
raise
else:
cps[result["COUNTRY_PROGRAMME_WBS"]] = country_programme
possible_changes = ['name', 'from_date', 'to_date']
if updating_cp or self._changed_fields('cp', possible_changes,
country_programme, result):
country_programme.name = result['COUNTRY_PROGRAMME_NAME']
country_programme.from_date = wcf_json_date_as_datetime(
result['CP_START_DATE'])
country_programme.to_date = wcf_json_date_as_datetime(
result['CP_END_DATE'])
print country_programme
print result
country_programme.save()
updating_outcome = False
outcome = outcomes.get(result['OUTCOME_WBS'], None)
if not outcome:
try:
outcome, updating_outcome = Result.objects.get_or_create(
country_programme=country_programme,
result_type=ResultType.objects.get_or_create(
name='Outcome')[0],
wbs=result['OUTCOME_WBS'],
)
except Result.MultipleObjectsReturned as exp:
exp.message += 'Outcome WBS: ' + result['OUTCOME_WBS'] \
+ ' Output WBS: ' + result['OUTPUT_WBS'] \
+ ' Activity WBS: ' + result['ACTIVITY_WBS']
raise
else:
outcomes[result['OUTCOME_WBS']] = outcome
possible_changes = ['name', 'from_date', 'to_date']
if updating_outcome or self._changed_fields(
'outcome', possible_changes, outcome, result):
# check if any of the information on the result is changing...
outcome.name = result['OUTCOME_DESCRIPTION']
outcome.from_date = wcf_json_date_as_datetime(
result['OUTCOME_START_DATE'])
outcome.to_date = wcf_json_date_as_datetime(
result['OUTCOME_END_DATE'])
if not outcome.valid_entry():
print 'Skipping outcome because of wbs missmatch: ', outcome
# we need to skip this record since the wbs's don;t match
# TODO in these cases... send an email with the record and make the country aware
continue
# raise Exception('Wbs of outcome does not map under country_programme')
outcome.save()
updating_output = False
output = outputs.get(result['OUTPUT_WBS'], None)
if not output:
try:
output, updating_output = Result.objects.get_or_create(
country_programme=country_programme,
result_type=ResultType.objects.get_or_create(
name='Output')[0],
wbs=result['OUTPUT_WBS'],
)
except Result.MultipleObjectsReturned as exp:
exp.message += 'Outcome WBS: ' + result['OUTCOME_WBS'] \
+ ' Output WBS: ' + result['OUTPUT_WBS'] \
+ ' Activity WBS: ' + result['ACTIVITY_WBS']
raise
else:
outputs[result['OUTPUT_WBS']] = output
possible_changes = ['name', 'from_date', 'to_date']
if updating_output or self._changed_fields(
'output', possible_changes, output, result):
output.name = result['OUTPUT_DESCRIPTION']
output.from_date = wcf_json_date_as_datetime(
result['OUTPUT_START_DATE'])
output.to_date = wcf_json_date_as_datetime(
result['OUTPUT_END_DATE'])
output.parent = outcome
if not output.valid_entry():
# we need to skip this record since the wbs's don;t match
# TODO in these cases... send an email with the record and make the country aware
continue
#raise Exception('Wbs of output does not map under country_programme')
output.save()
try:
activity, updating_activity = Result.objects.get_or_create(
country_programme=country_programme,
result_type=ResultType.objects.get_or_create(
name='Activity')[0],
wbs=result['ACTIVITY_WBS'],
)
except Result.MultipleObjectsReturned as exp:
exp.message += 'Outcome WBS: ' + result['OUTCOME_WBS'] \
+ ' Output WBS: ' + result['OUTPUT_WBS'] \
+ ' Activity WBS: ' + result['ACTIVITY_WBS']
raise
possible_changes = [
'name', 'from_date', 'to_date', 'sic_code', 'sic_name',
'gic_code', 'gic_name', 'activity_focus_code',
'activity_focus_name'
]
if updating_activity or self._changed_fields(
'activity', possible_changes, activity, result):
activity.name = result['ACTIVITY_DESCRIPTION']
activity.from_date = wcf_json_date_as_datetime(
result['ACTIVITY_START_DATE'])
activity.to_date = wcf_json_date_as_datetime(
result['ACTIVITY_END_DATE'])
activity.parent = output
activity.sic_code = result['SIC_CODE']
activity.sic_name = result['SIC_NAME']
activity.gic_code = result['GIC_CODE']
activity.gic_name = result['GIC_NAME']
activity.activity_focus_code = result['ACTIVITY_FOCUS_CODE']
activity.activity_focus_name = result['ACTIVITY_FOCUS_NAME']
if not activity.valid_entry():
activity.delete()
raise Exception(
'Wbs of activity does not map under country_programme')
activity.save()
if updating_cp:
print 'add cp', country_programme
if updating_activity:
print 'added activity', activity
if updating_outcome:
print 'add outcome', outcome
if updating_output:
print 'add output', output
processed += 1
return processed
def test_none(self):
self.assertIsNone(utils.wcf_json_date_as_datetime(None))
def _partner_save(processed, partner):
try:
new = False
saving = False
try:
partner_org = PartnerOrganization.objects.get(
vendor_number=partner["VENDOR_CODE"])
except PartnerOrganization.DoesNotExist:
partner_org = PartnerOrganization(
vendor_number=partner["VENDOR_CODE"])
new = True
try:
type_mapping[partner["PARTNER_TYPE_DESC"]]
except KeyError as exp:
print "Partner {} skipped, because PartnerType ={}".format(
partner['VENDOR_NAME'], exp)
# if partner organization exists in etools db (these are nameless)
if partner_org.id:
partner_org.name = "" # leaving the name blank on purpose (invalid record)
partner_org.deleted_flag = True if partner[
"DELETED_FLAG"] else False
partner_org.hidden = True
partner_org.save()
return processed
if new or _changed_fields([
'name', 'cso_type', 'rating', 'type_of_assessment',
'address', 'phone_number', 'email', 'deleted_flag',
'last_assessment_date', 'core_values_assessment_date'
], partner_org, partner):
partner_org.name = partner["VENDOR_NAME"]
partner_org.cso_type = partner["CSO_TYPE_NAME"]
partner_org.rating = partner["RISK_RATING_NAME"]
partner_org.type_of_assessment = partner[
"TYPE_OF_ASSESSMENT"]
partner_org.address = partner["STREET_ADDRESS"]
partner_org.phone_number = partner["PHONE_NUMBER"]
partner_org.email = partner["EMAIL"]
partner_org.core_values_assessment_date = wcf_json_date_as_datetime(
partner["CORE_VALUE_ASSESSMENT_DT"])
partner_org.last_assessment_date = wcf_json_date_as_datetime(
partner["LAST_ASSESSMENT_DATE"])
partner_org.partner_type = type_mapping[
partner["PARTNER_TYPE_DESC"]]
partner_org.deleted_flag = True if partner[
"DELETED_FLAG"] else False
if not partner_org.hidden:
partner_org.hidden = partner_org.deleted_flag
partner_org.vision_synced = True
saving = True
if partner_org.total_ct_cp == None or partner_org.total_ct_cy == None or \
not comp_decimals(partner_org.total_ct_cp, _totals_cp[partner["VENDOR_CODE"]]) or \
not comp_decimals(partner_org.total_ct_cy, _totals_cy[partner["VENDOR_CODE"]]):
partner_org.total_ct_cy = _totals_cy[
partner["VENDOR_CODE"]]
partner_org.total_ct_cp = _totals_cp[
partner["VENDOR_CODE"]]
saving = True
print "sums changed", partner_org
if saving:
print "Updating Partner", partner_org
partner_org.save()
del _totals_cy[partner["VENDOR_CODE"]]
del _totals_cp[partner["VENDOR_CODE"]]
processed += 1
except Exception as exp:
print "Exception message: {} " \
"Exception type: {} " \
"Exception args: {} ".format(
exp.message, type(exp).__name__, exp.args
)
return processed
