def set_next_execution(self):
"""
Set self.next_execution to the next time the import is going to be run. If run_immediately
is set, time is set to now()
"""
debug_enter(who_am_i(), who_is_parent(),
getframeinfo(currentframe()).lineno)
time_adds = {
self.EVERY_TWO_MINUTES: timedelta(seconds=120), # used for testing
self.HOURLY: timedelta(hours=1),
self.EVERY_SIX_HOURS: timedelta(hours=6),
self.DAILY: timedelta(days=1),
self.EVERY_THREE_DAYS: timedelta(days=3),
self.WEEKLY: timedelta(weeks=1),
self.BI_WEEKLY: timedelta(weeks=2),
self.EVERY_FOUR_WEEKS: timedelta(weeks=4),
}
if self.frequency:
if not self.next_execution:
self.next_execution = datetime.now() + time_adds[
self.frequency]
else:
self.next_execution += time_adds[self.frequency]
else:
self.next_execution = None
self.save()
debug_exit(who_am_i(), getframeinfo(currentframe()).lineno)
def set_next_execution(self):
"""
Set self.next_execution to the next time the import is going to be run. If run_immediately
is set, time is set to now()
"""
debug_enter(who_am_i(), who_is_parent(), getframeinfo(currentframe()).lineno)
time_adds = {
self.EVERY_TWO_MINUTES: timedelta(seconds=120), # used for testing
self.HOURLY: timedelta(hours=1),
self.EVERY_SIX_HOURS: timedelta(hours=6),
self.DAILY: timedelta(days=1),
self.EVERY_THREE_DAYS: timedelta(days=3),
self.WEEKLY: timedelta(weeks=1),
self.BI_WEEKLY: timedelta(weeks=2),
self.EVERY_FOUR_WEEKS: timedelta(weeks=4),
}
if self.frequency:
if not self.next_execution:
self.next_execution = datetime.now() + time_adds[self.frequency]
else:
self.next_execution += time_adds[self.frequency]
else:
self.next_execution = None
self.save()
debug_exit(who_am_i(), getframeinfo(currentframe()).lineno)
class GatewayNumber(models.Model):
gateway = models.ForeignKey(Gateway)
number = models.CharField(u'gateway number', max_length=30)
def __unicode__(self):
return self.number
def send_sms(self, mt_number, message):
logger.debug("Entering: %s()" % who_am_i())
try:
send_fields = GW_SENDING_FIELDS_42IT #TODO: generalize to handle any defined gateway
data = GW_SENDING_DATA_42IT #TODO: generalize to handle any defined gateway
gw_number = self.number
for field_name in [
'gw_number',
'mt_number',
'message',
]:
data[send_fields[field_name]] = locals()[field_name]
url_values = urllib.urlencode(
dict([k, v.encode('utf-8')] for k, v in data.items()))
gw = self.gateway
full_url = 'http://%s%s?%s' % (gw.host_name, gw.send_path,
url_values)
if getattr(settings, 'SMS_DEBUG', False):
print "send_sms URL call: %s" % full_url
else:
data = urllib2.urlopen(full_url)
except Exception, e:
logger.exception('%s Locals:\n %s\n\n' % (
e.message,
locals(),
))
logger.debug("Exiting: %s(). Called URL: %s" % (who_am_i(), data))
def check_execution(self):
"""
Checks if we should run a job on this import. This is true if either:
1) we find a job object with pending status, in that case this is a manual run initiated
in the admin
2) if no job is present we check if it's time for a scheduled execution of the import
:param job: an IatiImportJob object
:return:
"""
debug_enter(who_am_i(), who_is_parent(),
getframeinfo(currentframe()).lineno)
from .iati_import_log import IatiImportLog
if not self.enabled:
return
job_model = self.job_model()
try:
job = job_model.objects.get(iati_import=self,
status=LOG_ENTRY_TYPE.STATUS_PENDING)
except job_model.DoesNotExist:
job = None
except job_model.MultipleObjectsReturned:
for job in job_model.objects.filter(
iati_import=self, status=LOG_ENTRY_TYPE.STATUS_PENDING):
IatiImportLog.objects.create(
iati_import_job=job,
text=
"Fatal error executing import. Multiple pending jobs found.",
created_at=datetime.now())
return
if self.run_immediately or self.it_is_time_to_execute():
self.execute_import(job)
def create_cats_and_benches(business_units):
outsys("\nRunning {}() ".format(who_am_i()))
for internal_id, data in business_units.items():
if data.get('category'):
for name in data['benchmarknames']:
outsys('.')
new_bench, created = Benchmarkname.objects.get_or_create(
name=name)
if created:
log(
u"Created bench: {pk}, {label}",
dict(label=name,
pk=new_bench.id,
event=ACTION_BENCH_CREATE))
else:
log(
u"Found existing bench: {pk}, {label}",
dict(label=name,
pk=new_bench.id,
event=ACTION_BENCH_FOUND))
try:
data['category'].benchmarknames.add(new_bench)
except:
# we shouldn't end up here since we're testing for the existance of the category above
pass
else:
outsys('*')
log(
u"No category set of business unit with internal ID {internal_id}",
dict(internal_id=internal_id, event=ERROR_CATEGORY_MISSING))
def set_publishing_status(publishing_statuses):
outsys("\nRunning {}() ".format(who_am_i()))
cordaid = Organisation.objects.get(pk=CORDAID_ORG_ID)
for internal_id, publish in publishing_statuses.items():
try:
status = PublishingStatus.objects.get(
project__partnerships__internal_id=internal_id,
project__partnerships__organisation=cordaid,
)
status.status = PublishingStatus.STATUS_PUBLISHED if publish else PublishingStatus.STATUS_UNPUBLISHED
status.save()
log(
u"Set publishing status for project ID: {pk}: {extra}",
dict(internal_id=internal_id,
pk=status.project.pk,
event=ACTION_PUBLISHING_SET,
extra=status.status))
outsys(".")
except Exception, e:
log(
u"Error setting publishing status for project {internal_id}\nException class: {extra}",
dict(internal_id=internal_id,
event=e.__class__,
extra=e.message),
)
outsys("*")
def check_execution(self):
"""
Checks if we should run a job on this import. This is true if either:
1) we find a job object with pending status, in that case this is a manual run initiated
in the admin
2) if no job is present we check if it's time for a scheduled execution of the import
:param job: an IatiImportJob object
:return:
"""
debug_enter(who_am_i(), who_is_parent(), getframeinfo(currentframe()).lineno)
from .iati_import_log import IatiImportLog
if not self.enabled:
return
job_model = self.job_model()
try:
job = job_model.objects.get(iati_import=self, status=LOG_ENTRY_TYPE.STATUS_PENDING)
except job_model.DoesNotExist:
job = None
except job_model.MultipleObjectsReturned:
for job in job_model.objects.filter(
iati_import=self, status=LOG_ENTRY_TYPE.STATUS_PENDING):
IatiImportLog.objects.create(
iati_import_job=job,
text="Fatal error executing import. Multiple pending jobs found.",
created_at=datetime.now()
)
return
if self.run_immediately or self.it_is_time_to_execute():
self.execute_import(job)
def new_sms(cls, request, gateway):
"""
Handle callback from gateway, creating a generic object representing
an incoming SMS
"""
logger.debug("Entering: %s()" % who_am_i())
request.encoding = 'iso-8859-1' #TODO: some GWs allow this to be set I think
try:
# if we find an mms already, do nuthin...
sms, created = cls.objects.get(
msg_id=request.GET.get(gateway.msg_id)), False
logger.debug("SMS with id %s already exists!" % msg_id)
except:
try:
raw = {}
# loop over all field names and do lookup of the callback api name of
# the field to get the incoming value
for f in cls._meta.fields:
value = request.GET.get(getattr(gateway, f.name, ''),
False)
if value:
raw[f.name] = value
raw['saved_at'] = datetime.now()
sms, created = cls.objects.create(**raw), True
except Exception, e:
logger.exception(
"Exception trying to create a MoSms instance. Error: %s Locals:\n %s\n\n"
% (e.message, locals()))
return None, False
def send_sms(self, mt_number, message):
logger.debug("Entering: %s()" % who_am_i())
try:
send_fields = GW_SENDING_FIELDS_42IT #TODO: generalize to handle any defined gateway
data = GW_SENDING_DATA_42IT #TODO: generalize to handle any defined gateway
gw_number = self.number
for field_name in [
'gw_number',
'mt_number',
'message',
]:
data[send_fields[field_name]] = locals()[field_name]
url_values = urllib.urlencode(
dict([k, v.encode('utf-8')] for k, v in data.items()))
gw = self.gateway
full_url = 'http://%s%s?%s' % (gw.host_name, gw.send_path,
url_values)
if getattr(settings, 'SMS_DEBUG', False):
print "send_sms URL call: %s" % full_url
else:
data = urllib2.urlopen(full_url)
except Exception, e:
logger.exception('%s Locals:\n %s\n\n' % (
e.message,
locals(),
))
def import_orgs(xml_file):
outsys("\nRunning {}() ".format(who_am_i()))
def text_from_xpath(tree, xpath):
""" utility to get the text of an element using xpath, stripped
returns '' unless the xpath returns exactly one element
"""
element = tree.xpath(xpath)
if len(element) != 1:
return ''
return element[0].text.strip() if element[0].text else ''
def data_from_xpaths(xpaths, etree):
""" use the xpaths dict to replace the values with the actual data in the etree
that is retrieved when using text_from_xpath() with the xpath
"""
return {key: text_from_xpath(etree, xpath) for key, xpath in xpaths.items()}
def org_data_from_xml(org_etree):
# keys are Organisation field names, values are xpath expressions for getting those values from the org_etree
xpaths = dict(
name='name',
description='description',
url='url',
new_organisation_type='iati_organisation_type',
)
# get the raw data from the org_etree
org_data = data_from_xpaths(xpaths, org_etree)
# transform data
org_data['long_name'] = org_data['name']
org_data['name'] = org_data['name'][:25]
org_data['organisation_type'] = Organisation.org_type_from_iati_type(int(org_data['new_organisation_type']))
return org_data
def create_new_organisation(org_etree, internal_id):
try:
org_dict = org_data_from_xml(org_etree)
referenced_org = Organisation.objects.create(**org_dict)
log(
u"Created new organisation: {label}, Akvo ID: {pk}",
dict(
log_type=LOG_ORGANISATIONS,
internal_id=internal_id,
label=referenced_org.name,
pk=referenced_org.pk,
event=ACTION_CREATE_ORG
)
)
return referenced_org
except Exception, e:
log(
u"Error trying to create organisation with Cordaid ID {internal_id} ",
dict(
log_type=LOG_ORGANISATIONS,
internal_id=internal_id,
event=ERROR_EXCEPTION,
extra=e.message
)
)
def fix_funding(budgets):
"""
Add Cordaid as a funding partner to all its projects and "fill the project up"
"""
outsys("\nRunning {}() ".format(who_am_i()))
def assign_funding_partner(project, organisation, amount):
funding_partnership, created = Partnership.objects.get_or_create(
organisation=organisation,
project=project,
partner_type=Partnership.FUNDING_PARTNER,
defaults={'funding_amount': amount}
)
if created:
log(
u"Added {org_name} as funding partner to project {{pk}}, funding amount: {{extra}}".format(org_name=organisation.name),
dict(internal_id=internal_id, pk=project.pk, event=ACTION_FUNDING_SET, extra=amount)
)
else:
funding_partnership.funding_amount = amount
funding_partnership.save()
log(
u"Found {org_name} as funding partner to project {{pk}}, setting funding amount: {{extra}}".format(org_name=organisation.name),
dict(internal_id=internal_id, pk=project.pk, event=ACTION_FUNDING_FOUND, extra=amount)
)
cordaid = Organisation.objects.get(pk=CORDAID_ORG_ID)
others = Organisation.objects.get(pk=OTHERS_ORG_ID)
for budget in budgets:
internal_id = budget['internal_project_id']
try:
project = None
project = Project.objects.get(
partnerships__internal_id=internal_id, partnerships__organisation=cordaid
)
cordaid_funding = budget.get('cordaid_funding', 0)
others_funding = budget.get('others_funding', 0)
if cordaid_funding:
assign_funding_partner(project, cordaid, cordaid_funding)
if others_funding:
assign_funding_partner(project, others, others_funding)
total_budget = cordaid_funding + others_funding
old_budgets = BudgetItem.objects.filter(project=project)
old_budgets.delete()
BudgetItem.objects.create(
project=project,
label = BudgetItemLabel.objects.get(pk=13), #total budget label
amount = total_budget
)
log(
u"Total budget for project {pk}: {extra}",
dict(internal_id=internal_id, pk=project.pk, event=ACTION_BUDGET_SET, extra=total_budget)
)
outsys(".")
except Exception, e:
log(u"Error setting up funding partners for project {pk}\nException class: {extra}",
dict(internal_id=internal_id, pk=getattr(project, 'pk', None), event=e.__class__, extra=e.message),
)
outsys("*")
def test_inspection_definitions(self):
"""
Tests for inspecting definitions.
"""
this_definition = who_am_i()
parent_definition = who_is_parent()
self.assertEqual(this_definition, "test_inspection_definitions")
self.assertEqual(parent_definition, "_callTestMethod")
def test_inspection_definitions(self):
"""
Tests for inspecting definitions.
"""
this_definition = who_am_i()
parent_definition = who_is_parent()
self.assertEqual(this_definition, "test_inspection_definitions")
self.assertEqual(parent_definition, "run")
def find_cordaid_business_units(business_units):
outsys("\nRunning {}() ".format(who_am_i()))
known_business_units = {
"27239": dict(pk=CORDAID_ORG_ID),
"K6020": dict(pk=959, cat_name="Children & Education", fa="Education"),
"K6090": dict(pk=962, cat_name="Domestic", fa="Economic development"),
"K6030": dict(pk=961, cat_name="Disaster Recovery", fa="Economic development"),
"K6070": dict(pk=950, cat_name="Entrepreneurship", fa="Economic development"),
"K6110": dict(pk=1099, cat_name="Food Security", fa="Healthcare"),
"K6100": dict(pk=953, cat_name="Investments", fa="Economic development"),
"K6010": dict(pk=949, cat_name="Healthcare", fa="Healthcare"),
"K6060": dict(pk=1241, cat_name="Security & Justice", fa="Economic development"),
"K6080": dict(pk=946, cat_name="Urban Matters", fa="Economic development"),
"K6040": dict(pk=955, cat_name="Women's leadership", fa="Economic development"),
"K6050": dict(pk=960, cat_name="Extractives", fa="Economic development"),
}
cordaid = Organisation.objects.get(pk=CORDAID_ORG_ID)
for internal_id in business_units.keys():
cbu = known_business_units.get(internal_id, {'pk': -1})
pk, cat_name, fa_name = cbu['pk'], cbu.get('cat_name'), cbu.get('fa'),
try:
organisation = Organisation.objects.get(pk=pk)
outsys('.')
except:
outsys('*')
log(
u"No business unit with internal ID {internal_id}",
dict(
internal_id=internal_id,
event=ERROR_BUSINESS_UNIT_MISSING
)
)
continue
internal_org, created = InternalOrganisationID.objects.get_or_create(
recording_org=cordaid,
referenced_org=organisation,
identifier= internal_id
)
if cat_name:
new_cat, created = Category.objects.get_or_create(name=cat_name)
if created:
log(u"Created cat: {id}, {cat_name}",dict(id=new_cat.id, cat_name=cat_name))
new_cat.focus_area.add(FocusArea.objects.get(name=fa_name))
else:
log(u"Found existing cat: {id}, {cat_name}", dict(id=new_cat.id, cat_name=cat_name))
business_units.setdefault(internal_id, {'category': None, 'benchmarknames': []})['category'] = new_cat
cordaid.iati_org_id = CORDAID_IATI_ID
cordaid.save()
try:
dgis = Organisation.objects.get(pk=DGIS_ORG_ID)
dgis.iati_org_id = DGIS_IATI_ID
dgis.save()
except:
log(u"Can't find DGIS using ID {dgis_id}", dict(dgis_id=DGIS_ORG_ID),)
return business_units
def find_cordaid_business_units(business_units):
outsys("\nRunning {}() ".format(who_am_i()))
known_business_units = {
"27239": dict(pk=CORDAID_ORG_ID),
"K6020": dict(pk=959, cat_name="Children & Education", fa="Education"),
"K6090": dict(pk=962, cat_name="Domestic", fa="Economic development"),
"K6030": dict(pk=961, cat_name="Disaster Recovery", fa="Economic development"),
"K6070": dict(pk=950, cat_name="Entrepreneurship", fa="Economic development"),
"K6110": dict(pk=1099, cat_name="Food Security", fa="Healthcare"),
"K6100": dict(pk=953, cat_name="Investments", fa="Economic development"),
"K6010": dict(pk=949, cat_name="Healthcare", fa="Healthcare"),
"K6060": dict(pk=1241, cat_name="Security & Justice", fa="Economic development"),
"K6080": dict(pk=946, cat_name="Urban Matters", fa="Economic development"),
"K6040": dict(pk=955, cat_name="Women's leadership", fa="Economic development"),
"K6050": dict(pk=960, cat_name="Extractives", fa="Economic development"),
}
cordaid = Organisation.objects.get(pk=CORDAID_ORG_ID)
for internal_id in business_units.keys():
cbu = known_business_units.get(internal_id, {'pk': -1})
pk, cat_name, fa_name = cbu['pk'], cbu.get('cat_name'), cbu.get('fa'),
try:
organisation = Organisation.objects.get(pk=pk)
outsys('.')
except:
outsys('*')
log(
u"No business unit with internal ID {internal_id}",
dict(
internal_id=internal_id,
event=ERROR_BUSINESS_UNIT_MISSING
)
)
continue
internal_org, created = InternalOrganisationID.objects.get_or_create(
recording_org=cordaid,
referenced_org=organisation,
identifier=internal_id
)
if cat_name:
new_cat, created = Category.objects.get_or_create(name=cat_name)
if created:
log(u"Created cat: {id}, {cat_name}", dict(id=new_cat.id, cat_name=cat_name))
new_cat.focus_area.add(FocusArea.objects.get(name=fa_name))
else:
log(u"Found existing cat: {id}, {cat_name}", dict(id=new_cat.id, cat_name=cat_name))
business_units.setdefault(internal_id, {'category': None, 'benchmarknames': []})['category'] = new_cat
cordaid.iati_org_id = CORDAID_IATI_ID
cordaid.save()
try:
dgis = Organisation.objects.get(pk=DGIS_ORG_ID)
dgis.iati_org_id = DGIS_IATI_ID
dgis.save()
except:
log(u"Can't find DGIS using ID {dgis_id}", dict(dgis_id=DGIS_ORG_ID),)
return business_units
def create_cats_and_benches(business_units):
outsys("\nRunning {}() ".format(who_am_i()))
for internal_id, data in business_units.items():
for name in data['benchmarknames']:
outsys('.')
new_bench, created = Benchmarkname.objects.get_or_create(name=name)
if created:
log(u"Created bench: {id}, {name}", dict(id=new_bench.id, name=name))
else:
log(u"Found existing bench: {id}, {name}", dict(id=new_bench.id, name=name))
data['category'].benchmarknames.add(new_bench)
def setup(self):
""" Create a list where each list item is one Activity object
"""
self.activities = []
# rain_orgs = CordaidOrgs()
outsys("\nRunning {}() ".format(who_am_i()))
with open(RAIN_IATI_ACTIVITIES_XML, 'r') as f:
iati_activities = etree.fromstring(f.read())
iati_activity_list = iati_activities.xpath('//iati-activity')
for iati_activity in iati_activity_list:
outsys(".")
self.activities.append(RainActivity(iati_activity, RAIN_ACTIVITY_NS, AKVO_NS))
def handle_incoming_sms(sender, **kwargs):
"""
called through post_save.connect(handle_incoming_sms, sender=MoSms)
"""
logger.debug("Entering: %s()" % who_am_i())
# kwargs['raw'] is True when we're running manage.py loaddata
if kwargs.get('created', False) and not kwargs.get('raw', False):
new_sms = kwargs['instance']
try:
profile = get_model('rsr', 'UserProfile').objects.process_sms(new_sms)
except Exception, e:
logger.exception('%s Locals:\n %s\n\n' % (e.message, locals(), ))
def create_cordaid_business_units(business_units):
outsys("\nRunning {}() ".format(who_am_i()))
business_units_info = [
dict(pk=CORDAID_ORG_ID, internal_id="27239"),
dict(pk=959, internal_id="K6020", cat_name="Children & Education", fa="Education"),
dict(pk=962, internal_id="K6090", cat_name="Domestic", fa="Economic development"),
dict(pk=961, internal_id="K6030", cat_name="Disaster Recovery", fa="Economic development"),
dict(pk=950, internal_id="K6070", cat_name="Entrepreneurship", fa="Economic development"),
dict(pk=1099, internal_id="K6110", cat_name="Food Security", fa="Healthcare"),
dict(pk=953, internal_id="K6100", cat_name="Investments", fa="Economic development"),
dict(pk=949, internal_id="K6010", cat_name="Healthcare", fa="Healthcare"),
dict(pk=1241, internal_id="K6060", cat_name="Security & Justice", fa="Economic development"),
dict(pk=946, internal_id="K6080", cat_name="Urban Matters", fa="Economic development"),
dict(pk=955, internal_id="K6040", cat_name="Women's leadership", fa="Economic development"),
dict(pk=960, internal_id="K6050", cat_name="Extractives", fa="Economic development"),
]
cordaid = Organisation.objects.get(pk=CORDAID_ORG_ID)
for data in business_units_info:
outsys('.')
pk, identifier = data['pk'], data['internal_id']
cat_name, fa_name = data.get('cat_name'), data.get('fa')
try:
organisation = Organisation.objects.get(pk=pk)
except:
log(
u"No business unit with id {pk}, internal ID {identifier}",
dict(pk=pk, identifier=identifier),
)
continue
internal_org, created = InternalOrganisationID.objects.get_or_create(
recording_org=cordaid,
referenced_org=organisation,
identifier= identifier
)
if cat_name:
new_cat, created = Category.objects.get_or_create(name=cat_name)
if created:
log(u"Created cat: {id}, {cat_name}",dict(id=new_cat.id, cat_name=cat_name))
new_cat.focus_area.add(FocusArea.objects.get(name=fa_name))
else:
log(u"Found existing cat: {id}, {cat_name}", dict(id=new_cat.id, cat_name=cat_name))
business_units.setdefault(identifier, {'category': None, 'benchmarknames': []})['category'] = new_cat
cordaid.iati_org_id = CORDAID_IATI_ID
cordaid.save()
try:
dgis = Organisation.objects.get(pk=DGIS_ORG_ID)
dgis.iati_org_id = DGIS_IATI_ID
dgis.save()
except:
log(u"Can't find DGIS using ID {dgis_id}", dict(dgis_id=DGIS_ORG_ID),)
return business_units
def setup(self):
""" Create a list where each list item is one Activity object
"""
self.activities = []
outsys("\nRunning {}() ".format(who_am_i()))
xml = load_xml(RVO_IATI_ACTIVITES_URL)
if xml:
parser = etree.XMLParser(ns_clean=True, recover=True, encoding='utf-8')
iati_activities = etree.fromstring(xml, parser=parser)
iati_activity_list = iati_activities.xpath('//iati-activity')
for iati_activity in iati_activity_list:
outsys(".")
self.activities.append(RvoActivity(iati_activity))
def import_cordaid_benchmarks(csv_file):
outsys("\nRunning {}() ".format(who_am_i()))
# the columns to use in the CSV
COL_BENCHMARKNAME, COL_BUSINESS_UNID_ID = 1, 2
with open(csv_file, 'r') as f:
outsys('.')
indicators_reader = csv.reader(f, delimiter=',', quotechar='"')
business_units = {}
for line in indicators_reader:
business_units.setdefault(
line[COL_BUSINESS_UNID_ID], dict(benchmarknames=[]))['benchmarknames'].append(line[COL_BENCHMARKNAME]
)
return business_units
def receive_sms(request, gw_name):
'''
Handle a callback from a mobile message gateway
'''
# see if message already has been received for some reason, if so ignore
logger.debug("Entering: %s()" % who_am_i())
try:
gateway = Gateway.objects.get(name__iexact=gw_name)
logger.debug("Found a gateway: %s" % gw_name)
except Exception, e:
# general bork...bail out
logger.exception("Exception trying to create a gateway instance. Error: %s Locals:\n %s\n\n" % (e.message, locals()))
return HttpResponse("OK") #return OK under all conditions
def setup(self):
""" Create a list where each list item is one Activity object
"""
self.activities = []
outsys("\nRunning {}() ".format(who_am_i()))
xml = load_xml(RAIN_IATI_ACTIVITES_URL)
if xml:
parser = etree.XMLParser(ns_clean=True, recover=True, encoding='utf-8')
iati_activities = etree.fromstring(xml, parser=parser)
iati_activity_list = iati_activities.xpath('//iati-activity')
for iati_activity in iati_activity_list:
outsys(".")
self.activities.append(RainActivity(iati_activity, RAIN_ACTIVITY_NS, AKVO_NS))
def import_cordaid_benchmarks(csv_file):
outsys("\nRunning {}() ".format(who_am_i()))
# the columns to use in the CSV
COL_BENCHMARKNAME, COL_BUSINESS_UNID_ID = 1, 2
with open(csv_file, 'r') as f:
outsys('.')
indicators_reader = csv.reader(f, delimiter=',', quotechar='"')
business_units = {}
for line in indicators_reader:
business_units.setdefault(
line[COL_BUSINESS_UNID_ID], dict(benchmarknames=[]))['benchmarknames'].append(line[COL_BENCHMARKNAME]
)
return business_units
def receive_sms(request, gw_name):
'''
Handle a callback from a mobile message gateway
'''
# see if message already has been received for some reason, if so ignore
logger.debug("Entering: %s()" % who_am_i())
try:
gateway = Gateway.objects.get(name__iexact=gw_name)
logger.debug("Found a gateway: %s" % gw_name)
except Exception, e:
# general bork...bail out
logger.exception(
"Exception trying to create a gateway instance. Error: %s Locals:\n %s\n\n"
% (e.message, locals()))
return HttpResponse("OK") #return OK under all conditions
def handle_incoming_sms(sender, **kwargs):
"""
called through post_save.connect(handle_incoming_sms, sender=MoSms)
"""
logger.debug("Entering: %s()" % who_am_i())
# kwargs['raw'] is True when we're running manage.py loaddata
if kwargs.get('created', False) and not kwargs.get('raw', False):
new_sms = kwargs['instance']
try:
profile = get_model('rsr',
'UserProfile').objects.process_sms(new_sms)
except Exception, e:
logger.exception('%s Locals:\n %s\n\n' % (
e.message,
locals(),
))
def get_post_process_data():
""" Create a dictionary with photo IDs as keys:
{
<photo-id>: {
'internal_project_id': <internal_project_id>,
'image_caption': <image-caption>,
'image_credit': <image-credit>,
'cordaid_funding': <cordaid-funding>,
'others_funding': <others-funding>,
}
},
"""
outsys("\nRunning {}() ".format(who_am_i()))
with open(CORDAID_IATI_ACTIVITIES_XML, 'r') as f:
root = etree.fromstring(f.read())
AKVO_NS = '{{{akvo_ns}}}'.format(akvo_ns=root.nsmap['akvo'])
photos = {}
budgets = []
publishing_statuses = {}
for activity in root:
outsys(".")
photos[activity.get(AKVO_NS + 'photo-id')] = dict(
internal_project_id=activity.get(AKVO_NS +
'internal-project-id'),
image_caption=activity.get(AKVO_NS + 'image-caption',
'').strip(),
image_credit=activity.get(AKVO_NS + 'photo-credit',
'').strip(),
)
cordaid_budget = activity.findall('budget[@' + AKVO_NS +
'budget-from="Cordaid"]')
others_budget = activity.findall('budget[@' + AKVO_NS +
'budget-from="Others"]')
budgets.append(
dict(
internal_project_id=activity.get(AKVO_NS +
'internal-project-id'),
cordaid_funding=Decimal(cordaid_budget[0].find('value').
text if cordaid_budget else 0),
others_funding=Decimal(others_budget[0].find('value').
text if others_budget else 0),
))
publishing_statuses[activity.get(
AKVO_NS +
'internal-project-id')] = activity.get(AKVO_NS +
'publish') == 'true'
return photos, budgets, publishing_statuses
def import_images(image_dir, photos):
outsys("\nRunning {}() ".format(who_am_i()))
for image_name in os.listdir(image_dir):
photo_id, ext = splitext(image_name)
if ext.lower() in ['.png', '.jpg', '.jpeg', '.gif']:
try:
internal_id = photos.get(
photo_id,
{'internal_project_id': None})['internal_project_id']
project = Project.objects.get(
partnerships__internal_id=internal_id)
filename = model_and_instance_based_filename(
'Project', project.pk, 'current_image', image_name)
with open(os.path.join(image_dir, image_name), 'rb') as f:
image_data = f.read()
image_temp = NamedTemporaryFile(delete=True)
image_temp.write(image_data)
image_temp.flush()
project.current_image.save(filename,
File(image_temp),
save=True)
f.close()
project.current_image_caption = photos.get(
photo_id, {'image_caption': ''})['image_caption']
project.current_image_credit = photos.get(
photo_id, {'image_credit': ''})['image_credit']
project.save()
log(
u"Uploaded image to project {pk}",
dict(internal_id=internal_id,
pk=project.pk,
event=ACTION_SET_IMAGE))
outsys(".")
except Exception, e:
log(
u"Upload failed. internal_id: {internal_id} Exception class: {extra}",
dict(internal_id=internal_id,
event=ERROR_IMAGE_UPLOAD,
extra=e.__class__),
)
outsys("*")
def get_post_process_data():
""" Create a dictionary with photo IDs as keys:
{
<photo-id>: {
'internal_project_id': <internal_project_id>,
'image_caption': <image-caption>,
'image_credit': <image-credit>,
'cordaid_funding': <cordaid-funding>,
'others_funding': <others-funding>,
}
},
"""
outsys("\nRunning {}() ".format(who_am_i()))
with open(CORDAID_IATI_ACTIVITIES_XML, 'r') as f:
root = etree.fromstring(f.read())
AKVO_NS = '{{{akvo_ns}}}'.format(akvo_ns=root.nsmap['akvo'])
photos = {}
budgets = []
publishing_statuses = {}
for activity in root:
outsys(".")
photos[
activity.get(AKVO_NS + 'photo-id')
] = dict(
internal_project_id=activity.get(AKVO_NS + 'internal-project-id'),
image_caption=activity.get(AKVO_NS + 'image-caption', '').strip(),
image_credit=activity.get(AKVO_NS + 'photo-credit', '').strip(),
)
cordaid_budget = activity.findall('budget[@' + AKVO_NS + 'budget-from="Cordaid"]')
others_budget = activity.findall('budget[@' + AKVO_NS + 'budget-from="Others"]')
budgets.append(
dict(
internal_project_id=activity.get(AKVO_NS + 'internal-project-id'),
cordaid_funding=Decimal(cordaid_budget[0].find('value').text if cordaid_budget else 0),
others_funding=Decimal(others_budget[0].find('value').text if others_budget else 0),
)
)
publishing_statuses[
activity.get(AKVO_NS + 'internal-project-id')
] = activity.get(AKVO_NS + 'publish') == 'true'
return photos, budgets, publishing_statuses
def set_publishing_status(publishing_statuses):
outsys("\nRunning {}() ".format(who_am_i()))
cordaid = Organisation.objects.get(pk=CORDAID_ORG_ID)
for internal_id, publish in publishing_statuses.items():
try:
status = PublishingStatus.objects.get(
project__partnerships__internal_id=internal_id,
project__partnerships__organisation=cordaid,
)
status.status = PublishingStatus.STATUS_PUBLISHED if publish else PublishingStatus.STATUS_UNPUBLISHED
status.save()
log(
u"Set publishing status for project ID: {pk}: {extra}",
dict(internal_id=internal_id, pk=status.project.pk, event=ACTION_PUBLISHING_SET, extra=status.status)
)
outsys(".")
except Exception, e:
log(u"Error setting publishing status for project {internal_id}\nException class: {extra}",
dict(internal_id=internal_id, event=e.__class__, extra=e.message),
)
outsys("*")
def import_images(image_dir, photos):
outsys("\nRunning {}() ".format(who_am_i()))
for image_name in os.listdir(image_dir):
photo_id, ext = splitext(image_name)
if ext.lower() in ['.png', '.jpg', '.jpeg', '.gif']:
try:
internal_id = photos.get(
photo_id, {'internal_project_id': None}
)['internal_project_id']
project = Project.objects.get(
partnerships__internal_id=internal_id
)
filename = model_and_instance_based_filename(
'Project', project.pk, 'current_image', image_name
)
with open(os.path.join(image_dir, image_name), 'rb') as f:
image_data = f.read()
image_temp = NamedTemporaryFile(delete=True)
image_temp.write(image_data)
image_temp.flush()
project.current_image.save(filename, File(image_temp), save=True)
f.close()
project.current_image_caption = photos.get(
photo_id, {'image_caption': ''}
)['image_caption']
project.current_image_credit = photos.get(
photo_id, {'image_credit': ''}
)['image_credit']
project.save()
log(
u"Uploaded image to project {pk}",
dict(internal_id=internal_id, pk=project.pk, event=ACTION_SET_IMAGE))
outsys(".")
except Exception, e:
log(
u"Upload failed. internal_id: {internal_id} Exception class: {extra}",
dict(internal_id=internal_id, event=ERROR_IMAGE_UPLOAD, extra=e.__class__),
)
outsys("*")
def create_cats_and_benches(business_units):
outsys("\nRunning {}() ".format(who_am_i()))
for internal_id, data in business_units.items():
if data.get('category'):
for name in data['benchmarknames']:
outsys('.')
new_bench, created = Benchmarkname.objects.get_or_create(name=name)
if created:
log(u"Created bench: {pk}, {label}",
dict(
label=name,
pk=new_bench.id,
event=ACTION_BENCH_CREATE
)
)
else:
log(u"Found existing bench: {pk}, {label}",
dict(
label=name,
pk=new_bench.id,
event=ACTION_BENCH_FOUND
)
)
try:
data['category'].benchmarknames.add(new_bench)
except:
# we shouldn't end up here since we're testing for the existance of the category above
pass
else:
outsys('*')
log(
u"No category set of business unit with internal ID {internal_id}",
dict(
internal_id=internal_id,
event=ERROR_CATEGORY_MISSING
)
)
def fix_funding(budgets):
"""
Add Cordaid as a funding partner to all its projects and "fill the project up"
"""
outsys("\nRunning {}() ".format(who_am_i()))
def assign_funding_partner(project, organisation, amount):
funding_partnership, created = Partnership.objects.get_or_create(
organisation=organisation,
project=project,
iati_organisation_role=Partnership.IATI_FUNDING_PARTNER,
defaults={'funding_amount': amount}
)
if created:
log(
"Added {org_name} as funding partner to project {{pk}}, funding amount: {{extra}}".format(org_name=organisation.name),
dict(internal_id=internal_id, pk=project.pk, event=ACTION_FUNDING_SET, extra=amount)
)
else:
funding_partnership.funding_amount = amount
funding_partnership.save()
log(
"Found {org_name} as funding partner to project {{pk}}, setting funding amount: {{extra}}".format(org_name=organisation.name),
dict(internal_id=internal_id, pk=project.pk, event=ACTION_FUNDING_FOUND, extra=amount)
)
cordaid = Organisation.objects.get(pk=CORDAID_ORG_ID)
others = Organisation.objects.get(pk=OTHERS_ORG_ID)
for budget in budgets:
internal_id = budget['internal_project_id']
try:
project = None
project = Project.objects.get(
partnerships__internal_id=internal_id, partnerships__organisation=cordaid
)
project.set_reporting_org(cordaid)
project.save()
cordaid_funding = budget.get('cordaid_funding', 0)
others_funding = budget.get('others_funding', 0)
if cordaid_funding:
assign_funding_partner(project, cordaid, cordaid_funding)
if others_funding:
assign_funding_partner(project, others, others_funding)
total_budget = cordaid_funding + others_funding
old_budgets = BudgetItem.objects.filter(project=project)
old_budgets.delete()
BudgetItem.objects.create(
project=project,
label=BudgetItemLabel.objects.get(pk=BudgetItemLabel.TOTAL_BUDGET_LABEL_ID),
amount=total_budget
)
log(
"Total budget for project {pk}: {extra}",
dict(internal_id=internal_id, pk=project.pk, event=ACTION_BUDGET_SET, extra=total_budget)
)
outsys(".")
except Exception as e:
log("Error setting up funding partners for project {pk}\nException class: {extra}",
dict(internal_id=internal_id, pk=getattr(project, 'pk', None), event=e.__class__, extra=str(e)),
)
outsys("*")
outsys('\n')
from django.http import HttpResponse #, HttpResponseRedirect, HttpResponsePermanentRedirect
from models import Gateway, MoSms
from akvo.utils import who_am_i
import logging
logger = logging.getLogger('django.request')
def receive_sms(request, gw_name):
'''
Handle a callback from a mobile message gateway
'''
# see if message already has been received for some reason, if so ignore
logger.debug("Entering: %s()" % who_am_i())
try:
gateway = Gateway.objects.get(name__iexact=gw_name)
logger.debug("Found a gateway: %s" % gw_name)
except Exception, e:
# general bork...bail out
logger.exception(
"Exception trying to create a gateway instance. Error: %s Locals:\n %s\n\n"
% (e.message, locals()))
return HttpResponse("OK") #return OK under all conditions
sms, created = MoSms.new_sms(request, gateway)
logger.debug("Exiting: %s()" % who_am_i())
return HttpResponse(
"OK"
) #return OK under all conditions TODO: this should probably be configurable on the gateway
def it_is_time_to_execute(self):
debug_enter(who_am_i(), who_is_parent(), getframeinfo(currentframe()).lineno)
return self.enabled and self.next_execution and self.next_execution < datetime.now()
# logger.exception('%s Locals:\n %s\n\n' % (e.message, locals(), ))
# logger.debug("Exiting: %s()" % who_am_i())
def handle_incoming_sms(sender, **kwargs):
"""
called through post_save.connect(handle_incoming_sms, sender=MoSms)
"""
logger.debug("Entering: %s()" % who_am_i())
# kwargs['raw'] is True when we're running manage.py loaddata
if kwargs.get('created', False) and not kwargs.get('raw', False):
new_sms = kwargs['instance']
try:
profile = get_model('rsr', 'UserProfile').objects.process_sms(new_sms)
except Exception, e:
logger.exception('%s Locals:\n %s\n\n' % (e.message, locals(), ))
logger.debug("Exiting: %s()" % who_am_i())
def cleanup_reporters(profile, user):
if not profile.validation == profile.VALIDATED:
get_model('rsr', 'SmsReporter').objects.filter(userprofile=profile).delete()
def update_project_budget(sender, **kwargs):
"""
called when BudgetItem objects are added/changed/deleted
"""
# kwargs['raw'] is True when we're running manage.py loaddata
if not kwargs.get('raw', False):
try:
kwargs['instance'].project.update_budget()
kwargs['instance'].project.update_funds()
def it_is_time_to_execute(self):
debug_enter(who_am_i(), who_is_parent(),
getframeinfo(currentframe()).lineno)
return self.enabled and self.next_execution and self.next_execution < datetime.now(
)
class MoSms(models.Model):
"""
Generic storage of an incoming sms
Attributes:
sender: sender's telephone number
receiver: phone number the message was received at
message: the actual sms text
timestamp: many gateways include a time when the message arrived at the gateway
msg_id: many gateways include an id for each message
saved_at: time when the message is saved
"""
sender = models.CharField(
max_length=30,
verbose_name=u'sender',
)
receiver = models.CharField(
max_length=30,
verbose_name=u'receiver',
) #TODO: change to FK to GatewayNumber?
message = models.TextField(
blank=True,
verbose_name=u'message',
) #degenerate, but possible...
timestamp = models.CharField(
max_length=50,
verbose_name=u'timestamp',
blank=True,
)
msg_id = models.CharField(
max_length=100,
verbose_name=u'message ID',
blank=True,
)
saved_at = models.DateTimeField(verbose_name=u'time saved at', )
@classmethod
def new_sms(cls, request, gateway):
"""
Handle callback from gateway, creating a generic object representing
an incoming SMS
"""
logger.debug("Entering: %s()" % who_am_i())
request.encoding = 'iso-8859-1' #TODO: some GWs allow this to be set I think
try:
# if we find an mms already, do nuthin...
sms, created = cls.objects.get(
msg_id=request.GET.get(gateway.msg_id)), False
logger.debug("SMS with id %s already exists!" % msg_id)
except:
try:
raw = {}
# loop over all field names and do lookup of the callback api name of
# the field to get the incoming value
for f in cls._meta.fields:
value = request.GET.get(getattr(gateway, f.name, ''),
False)
if value:
raw[f.name] = value
raw['saved_at'] = datetime.now()
sms, created = cls.objects.create(**raw), True
except Exception, e:
logger.exception(
"Exception trying to create a MoSms instance. Error: %s Locals:\n %s\n\n"
% (e.message, locals()))
return None, False
logger.debug("Exiting: %s()" % who_am_i())
return sms, created