def handle(self, *args, **options):
logger.info('Starting appeal document ingest')
# get latest
url = 'https://proxy.hxlstandard.org/data.json?url=https%3A%2F%2Fdocs.google.com%2Fspreadsheets%2Fd%2F1gJ4N_PYBqtwVuJ10d8zXWxQle_i84vDx5dHNBomYWdU%2Fedit%3Fusp%3Dsharing'
response = requests.get(url)
if response.status_code != 200:
logger.error('Error querying Appeal Document HXL API')
raise Exception('Error querying Appeal Document HXL API')
records = response.json()
# some logging variables
not_found = []
existing = []
created = []
# group records by appeal code
acodes = list(set([a[2] for a in records[2:]]))
for code in acodes:
try:
appeal = Appeal.objects.get(code=code)
except ObjectDoesNotExist:
not_found.append(code)
continue
existing_docs = list(appeal.appealdocument_set.all())
docs = [a for a in records if a[2] == code]
for doc in docs:
exists = len(
[a for a in existing_docs if a.document_url == doc[0]]) > 0
if exists:
existing.append(doc[0])
else:
try:
created_at = self.parse_date(doc[5])
except:
created_at = None
AppealDocument.objects.create(
document_url=doc[0],
name=doc[4],
created_at=created_at,
appeal=appeal,
)
created.append(doc[0])
logger.info('%s appeal documents created' % len(created))
logger.info('%s existing appeal documents' % len(existing))
logger.warn('%s documents without appeals in system' % len(not_found))
def get_dbfile():
ftphost = os.environ.get('GO_FTPHOST', None)
ftpuser = os.environ.get('GO_FTPUSER', None)
ftppass = os.environ.get('GO_FTPPASS', None)
dbpass = os.environ.get('GO_DBPASS', None)
if ftphost is None or ftpuser is None or ftppass is None:
if os.path.exists('URLs.mdb'):
logger.info('No credentials in env, using local MDB database file')
logger.warn(
'If this occurs outside development, contact an administrator')
return 'URLs.mdb'
else:
raise Exception(
'FTP credentials not provided (GO_FTPHOST, GO_FTPUSER, GO_FTPPASS)'
)
if dbpass is None:
raise Exception(
'Database encryption password not provided (GO_DBPASS)')
logger.info('Attempting connection to FTP')
ftp = FTP(ftphost)
ftp.login(user=ftpuser, passwd=ftppass)
ftp.cwd('/dmis/')
data = []
ftp.dir('-t', data.append)
filename = data[-1].split()[3]
# check if we already have this file
files = glob('URLs*zip')
if filename in files and os.path.exists('URLs.mdb'):
ftp.quit()
return 'URLs.mdb'
# clean up old files
for f in files:
os.remove(f)
logger.info('Fetching %s' % filename)
with open(filename, 'wb') as f:
ftp.retrbinary('RETR ' + filename, f.write, 2014)
ftp.quit()
logger.info('Unzipping database file')
zp = ZipFile(filename)
zp.extractall('./', pwd=dbpass.encode('cp850', 'replace'))
return 'URLs.mdb'
def send_notification(subject, recipients, html):
if not username or not password:
logger.warn(
'No EMAIL_USER and/or EMAIL_PASS set as environment variables')
logger.warn('Cannot send notification')
return
msg = MIMEMultipart('alternative')
msg['Subject'] = '[IFRCGO] %s' % subject
msg['From'] = username.upper()
msg['To'] = ', '.join(recipients)
text_body = MIMEText(strip_tags(html), 'plain')
html_body = MIMEText(html, 'html')
msg.attach(text_body)
msg.attach(html_body)
SendMail(recipients, msg).start()
def handle(self, *args, **options):
logger.info('Starting Deployment ingest')
# url = 'https://proxy.hxlstandard.org/data.json?url=https%3A%2F%2Fdocs.google.com%2Fspreadsheets%2Fd%2F1CBvledFYc_uwlvHTvJE0SYS7_mPGU2L-zhrqbB4KNIA%2Fedit%23gid%3D0&header-row=1' # not enough.
url = 'https://proxy.hxlstandard.org/data.json?tagger-match-all=on&' \
+ 'tagger-01-header=year&' \
+ 'tagger-01-tag=%23a1&' \
+ 'tagger-02-header=%2Aappeal+code&' \
+ 'tagger-02-tag=%23a2&' \
+ 'tagger-03-header=region&' \
+ 'tagger-03-tag=%23a3&' \
+ 'tagger-04-header=country&' \
+ 'tagger-04-tag=%23a4&' \
+ 'tagger-05-header=location&' \
+ 'tagger-05-tag=%23a5&' \
+ 'tagger-06-header=disaster+type&' \
+ 'tagger-06-tag=%23a6&' \
+ 'tagger-07-header=%2Adisaster+name&' \
+ 'tagger-07-tag=%23a7&' \
+ 'tagger-08-header=%2Aname&' \
+ 'tagger-08-tag=%23a8&' \
+ 'tagger-09-header=%2Adeploying+ns+%2F+ifrc+office&' \
+ 'tagger-09-tag=%23a9&' \
+ 'tagger-10-header=%2Agender&' \
+ 'tagger-10-tag=%23b1&' \
+ 'tagger-11-header=language&' \
+ 'tagger-11-tag=%23b2&' \
+ 'tagger-12-header=%2Aposition&' \
+ 'tagger-12-tag=%23b3&' \
+ 'tagger-13-header=%2Atype&' \
+ 'tagger-13-tag=%23b4&' \
+ 'tagger-14-header=supported+by+ns&' \
+ 'tagger-14-tag=%23b5&' \
+ 'tagger-15-header=availability&' \
+ 'tagger-15-tag=%23b6&' \
+ 'tagger-16-header=%2Aexp+start+date&' \
+ 'tagger-16-tag=%23b7&' \
+ 'tagger-17-header=%2Aexp+duration&' \
+ 'tagger-17-tag=%23b8&' \
+ 'tagger-18-header=%2Aalert&' \
+ 'tagger-18-tag=%23b9&' \
+ 'tagger-19-header=deployment+message&' \
+ 'tagger-19-tag=%23c1&' \
+ 'tagger-20-header=%2Astart+of+mission&' \
+ 'tagger-20-tag=%23c2&' \
+ 'tagger-21-header=%2Aend+of+mission&' \
+ 'tagger-21-tag=%23c3&' \
+ 'tagger-22-header=deployment+duration&' \
+ 'tagger-22-tag=%23c4&' \
+ 'tagger-23-header=deployed&' \
+ 'tagger-23-tag=%23c5&' \
+ 'tagger-24-header=rotation&' \
+ 'tagger-24-tag=%23c6&' \
+ 'tagger-25-header=comments&' \
+ 'tagger-25-tag=%23c7&' \
+ 'url=https%3A%2F%2Fdocs.google.com%2Fspreadsheets%2Fd%2F1CBvledFYc_uwlvHTvJE0SYS7_mPGU2L-zhrqbB4KNIA%2Fedit%23gid%3D0&' \
+ 'header-row=1'
response = requests.get(url)
if response.status_code != 200:
logger.error('Error querying Deployment HXL API')
raise Exception('Error querying Deployment HXL API')
records = response.json()
# some logging variables
not_found = []
existing = []
created = []
columns = [a.replace('*', '').replace(' ', '') for a in records[0]]
# ['Year', 'AppealCode', 'Region', 'Country', 'Location', 'Disastertype', 'Disastername', 'Name', 'DeployingNS/IFRCOffice', 'Gender', 'Language', 'Position', 'Type', 'SupportedbyNS', 'Availability', 'Expstartdate', 'expduration', 'Alert', 'Deploymentmessage', 'Startofmission', 'Endofmission', 'DeploymentDuration', 'Deployed', 'Rotation', 'Comments']
# 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24
# if empty name -> Alert, otherwise -> Deployment
# OBSOLETE:
# # group records by appeal code
# acodes = list(set([a[2] for a in records[2:]]))
# for code in acodes:
# try:
# appeal = Appeal.objects.get(code=code)
# except ObjectDoesNotExist:
# not_found.append(code)
# continue
#
# existing_docs = list(appeal.appealdocument_set.all())
# docs = [a for a in records if a[2] == code]
# for doc in docs:
# exists = len([a for a in existing_docs if a.document_url == doc[0]]) > 0
# if exists:
# existing.append(doc[0])
# else:
# try:
# created_at = self.parse_date(doc[5])
# except:
# created_at = None
#
# AppealDocument.objects.create(
# document_url=doc[0],
# name=doc[4],
# created_at=created_at,
# appeal=appeal,
# )
# created.append(doc[0])
logger.info('%s Deployments created' % len(created))
logger.info('%s existing Deployments' % len(existing))
logger.warn('%s documents without appeals in system' % len(not_found))
def handle(self, *args, **options):
# get latest
filename = get_dbfile()
# numeric details records
details_rc = extract_table(filename, 'EW_Report_NumericDetails')
# check for 1 record for each field report
fids = [r['ReportID'] for r in details_rc]
if len(set(fids)) != len(fids):
raise Exception(
'More than one NumericDetails record for a field report')
# numeric details records
details_gov = extract_table(filename, 'EW_Report_NumericDetails_GOV')
# check for 1 record for each field report
fids = [r['ReportID'] for r in details_gov]
if len(set(fids)) != len(fids):
raise Exception(
'More than one NumericDetails record for a field report')
# information
info_table = extract_table(filename, 'EW_Report_InformationManagement')
fids = [r['ReportID'] for r in info_table]
if len(set(fids)) != len(fids):
raise Exception(
'More than one InformationManagement record for a field report'
)
### many-to-many
# actions taken
actions_national = extract_table(filename,
'EW_Report_ActionTakenByRedCross')
actions_foreign = extract_table(filename,
'EW_Report_ActionTakenByPnsRC')
actions_federation = extract_table(
filename, 'EW_Report_ActionTakenByFederationRC')
# source types
source_types = extract_table(filename, 'EW_lofSources')
for s in source_types:
SourceType.objects.get_or_create(
pk=s['SourceID'], defaults={'name': s['SourceName']})
source_table = extract_table(filename, 'EW_Reports_Sources')
# disaster response
dr_table = extract_table(filename, 'EW_DisasterResponseTools')
# check for 1 record for each field report
fids = [r['ReportID'] for r in dr_table]
if len(set(fids)) != len(fids):
raise Exception(
'More than one DisasterResponseTools record for a field report'
)
# contacts
contacts = extract_table(filename, 'EW_Report_Contacts')
# field report
reports = extract_table(filename, 'EW_Reports')
rids = [r.rid for r in FieldReport.objects.all()]
num_reports_created = 0
logger.info('%s reports in database' % len(reports))
for i, report in enumerate(reports):
# Skip reports that we've already ingested.
# We don't have to update them because field reports can't be updated in DMIS.
rid = report['ReportID']
if rid in rids:
continue
report_name = report['Summary']
report_description = report['BriefSummary']
report_dtype = DisasterType.objects.get(
pk=PK_MAP[report['DisasterTypeID']])
record = {
'rid':
rid,
'summary':
report_name,
'description':
report_description,
'dtype':
report_dtype,
'status':
report['StatusID'],
'request_assistance':
report['GovRequestsInternAssistance'],
'actions_others':
report['ActionTakenByOthers'],
'report_date':
datetime.strptime(report['Inserted'],
REPORT_DATE_FORMAT).replace(tzinfo=pytz.utc),
}
details = fetch_relation(details_rc, report['ReportID'])
assert (len(details) <= 1)
if len(details) > 0:
details = details[0]
record.update({
'num_injured':
details['NumberOfInjured'],
'num_dead':
details['NumberOfCasualties'],
'num_missing':
details['NumberOfMissing'],
'num_affected':
details['NumberOfAffected'],
'num_displaced':
details['NumberOfDisplaced'],
'num_assisted':
details['NumberOfAssistedByRC'],
'num_localstaff':
details['NumberOfLocalStaffInvolved'],
'num_volunteers':
details['NumberOfVolunteersInvolved'],
'num_expats_delegates':
details['NumberOfExpatsDelegates']
})
details = fetch_relation(details_gov, report['ReportID'])
assert (len(details) <= 1)
if len(details) > 0:
details = details[0]
record.update({
'gov_num_injured':
details['NumberOfInjured_GOV'],
'gov_num_dead':
details['NumberOfDead_GOV'],
'gov_num_missing':
details['NumberOfMissing_GOV'],
'gov_num_affected':
details['NumberOfAffected_GOV'],
'gov_num_displaced':
details['NumberOfDisplaced_GOV'],
'gov_num_assisted':
details['NumberOfAssistedByGov_GOV']
})
info = fetch_relation(info_table, report['ReportID'])
if len(info) > 0:
info = {k: '' if v is None else v for k, v in info[0].items()}
record.update({
'bulletin': {
'': 0,
'None': 0,
'Planned': 2,
'Published': 3
}[info['InformationBulletin']],
'dref': {
'': 0,
'No': 0,
'Planned': 2,
'Yes': 3
}[info['DREFRequested']],
'dref_amount':
0 if info['DREFRequestedAmount'] == '' else float(
info['DREFRequestedAmount']),
'appeal': {
'': 0,
'Planned': 2,
'Yes': 3,
'NB': 0,
'No': 0,
'YES': 3
}[info['EmergencyAppeal']],
'appeal_amount':
0 if info['EmergencyAppealAmount'] == '' else float(
info['EmergencyAppealAmount']),
})
# disaster response
response = fetch_relation(dr_table, report['ReportID'])
if len(response) > 0:
response = {
k: '' if v is None else v
for k, v in response[0].items()
}
record.update({
'rdrt': {
'': 0,
'No': 0,
'Yes': 3,
'Planned/Requested': 2
}[response['RDRT']],
'fact': {
'': 0,
'No': 0,
'Yes': 3,
'Planned/Requested': 2
}[response['FACT']],
'eru_relief': {
'': 0,
'Yes': 3,
'Planned/Requested': 2,
'No': 0
}[response['ERU']]
})
field_report = FieldReport(**record)
# Create an associated event object
event_record = {
'name':
report_name if len(report_name) else report_dtype.name,
'summary':
report_description,
'dtype':
report_dtype,
'disaster_start_date':
datetime.utcnow().replace(tzinfo=timezone.utc),
'auto_generated':
True,
'auto_generated_source':
SOURCES['report_ingest'],
}
event = Event(**event_record)
event.save()
logger.info('Adding %s' % report_name)
field_report.event = event
field_report.save()
num_reports_created = num_reports_created + 1
try:
country = Country.objects.select_related().get(
pk=report['CountryID'])
except ObjectDoesNotExist:
logger.warn('Could not find a matching country for %s' %
report['CountryID'])
country = None
if country is not None:
field_report.countries.add(country)
event.countries.add(country)
if country.region is not None:
# No need to add a field report region, as that happens through a trigger.
field_report.regions.add(country.region)
event.regions.add(country.region)
### add items with foreignkeys to report
# national red cross actions
actions = fetch_relation(actions_national, report['ReportID'])
if len(actions) > 0:
txt = ' '.join(
[a['Value'] for a in actions if a['Value'] is not None])
act = ActionsTaken(organization='NTLS',
summary=txt,
field_report=field_report)
act.save()
for pk in [a['ActionTakenByRedCrossID'] for a in actions]:
act.actions.add(*Action.objects.filter(pk=pk))
# foreign red cross actions
actions = fetch_relation(actions_foreign, report['ReportID'])
if len(actions) > 0:
txt = ' '.join(
[a['Value'] for a in actions if a['Value'] is not None])
act = ActionsTaken(organization='PNS',
summary=txt,
field_report=field_report)
act.save()
for pk in [a['ActionTakenByRedCrossID'] for a in actions]:
act.actions.add(*Action.objects.filter(pk=pk))
# federation red cross actions
actions = fetch_relation(actions_federation, report['ReportID'])
if len(actions) > 0:
txt = ' '.join(
[a['Value'] for a in actions if a['Value'] is not None])
act = ActionsTaken(organization='FDRN',
summary=txt,
field_report=field_report)
act.save()
for pk in [a['ActionTakenByRedCrossID'] for a in actions]:
act.actions.add(*Action.objects.filter(pk=pk))
# sources
sources = fetch_relation(source_table, report['ReportID'])
for s in sources:
spec = '' if s['Specification'] is None else s['Specification']
src = Source.objects.create(
stype=SourceType.objects.get(pk=s['SourceID']),
spec=spec,
field_report=field_report)
# disaster response
response = fetch_relation(dr_table, report['ReportID'])
# contacts
contact = fetch_relation(contacts, report['ReportID'])
if len(contact) > 0:
# make sure just one contacts record
assert (len(contact) == 1)
contact = contact[0]
fields = [
'Originator', 'Primary', 'Federation', 'NationalSociety',
'MediaNationalSociety', 'Media'
]
for f in fields:
if contact_is_valid(contact, f):
ct = FieldReportContact.objects.create(
ctype=f,
name=contact['%sName' % f],
title=contact['%sFunction' % f],
email=contact['%sContact' % f],
field_report=field_report,
)
total_reports = FieldReport.objects.all()
logger.info('%s reports created' % num_reports_created)
logger.info('%s reports in database' % total_reports.count())
# org type mapping
org_types = {
'1': 'NTLS',
'2': 'DLGN',
'3': 'SCRT',
'4': 'ICRC',
}
last_login_threshold = timezone.now() - timedelta(days=365)
# add users
user_records = extract_table(filename, 'DMISUsers')
processed_users = 0
for i, user_data in enumerate(user_records):
if user_data['LoginLastSuccess'] == '':
continue
last_login = datetime.strptime(
user_data['LoginLastSuccess'],
REPORT_DATE_FORMAT,
)
last_login = pytz.UTC.localize(last_login)
# skip users who haven't logged in for a year
if last_login < last_login_threshold:
continue
try:
user = User.objects.get(username=user_data['UserName'])
except ObjectDoesNotExist:
user = None
if user is None:
name = user_data['RealName'].split()
first_name = name[0]
last_name = ' '.join(name[1:]) if len(name) > 1 else ''
user = User.objects.create(
username=user_data['UserName'],
first_name=first_name if len(first_name) <= 30 else '',
last_name=last_name if len(last_name) <= 30 else '',
email=user_data['EmailAddress'],
last_login=last_login,
)
user.set_password(user_data['Password'])
user.is_staff = True if user_data[
'UserIsSysAdm'] == '1' else False
# set user profile info
user.profile.org = user_data['OrgTypeSpec'] if len(
user_data['OrgTypeSpec']) <= 100 else ''
user.profile.org_type = org_types.get(user_data['OrgTypeID'])
user.profile.country = Country.objects.get(
pk=user_data['CountryID'])
user.profile.city = user_data['City'] if len(
user_data['City']) <= 100 else ''
user.profile.department = user_data['Department'] if len(
user_data['Department']) <= 100 else ''
user.profile.position = user_data['Position'] if len(
user_data['Position']) <= 100 else ''
user.profile.phone_number = user_data['PhoneNumberProf'] if len(
user_data['PhoneNumberProf']) <= 100 else ''
user.save()
processed_users = processed_users + 1
logger.info('%s updated active user records' % len(processed_users))
def handle(self, *args, **options):
logger.info('Starting appeal document ingest')
if options['fullscan']:
# If the `--fullscan` option is passed, check ALL appeals
print('Doing a full scan of all Appeals')
qset = Appeal.objects.all()
else:
# By default, only check appeals for the past 3 months where Appeal Documents is 0
now = datetime.now()
three_months_ago = now - relativedelta(months=3)
qset = Appeal.objects.filter(appealdocument__isnull=True).filter(
end_date__gt=three_months_ago)
# First get all Appeal Codes
appeal_codes = [a.code for a in qset]
# Modify code taken from https://pastebin.com/ieMe9yPc to scrape `publications-and-reports` and find
# Documents for each appeal code
output = []
page_not_found = []
for code in appeal_codes:
code = code.replace(' ', '')
docs_url = 'http://www.ifrc.org/en/publications-and-reports/appeals/?ac=' + code + '&at=0&c=&co=&dt=1&f=&re=&t=&ti=&zo='
try:
response = urlopen(docs_url)
except: # if we get an error fetching page for an appeal, we ignore it
page_not_found.append(code)
continue
soup = BeautifulSoup(response.read(), "lxml")
div = soup.find('div', id='cw_content')
for t in div.findAll('tbody'):
output = output + self.makelist(t)
# Once we have all Documents in output, we add all missing Documents to the associated Appeal
not_found = []
existing = []
created = []
acodes = list(set([a[2] for a in output]))
for code in acodes:
try:
appeal = Appeal.objects.get(code=code)
except ObjectDoesNotExist:
not_found.append(code)
continue
existing_docs = list(appeal.appealdocument_set.all())
docs = [a for a in output if a[2] == code]
for doc in docs:
exists = len(
[a for a in existing_docs if a.document_url == doc[0]]) > 0
if exists:
existing.append(doc[0])
else:
try:
created_at = self.parse_date(doc[5])
except:
created_at = None
AppealDocument.objects.create(
document_url=doc[0],
name=doc[4],
created_at=created_at,
appeal=appeal,
)
created.append(doc[0])
logger.info('%s appeal documents created' % len(created))
logger.info('%s existing appeal documents' % len(existing))
logger.info('%s pages not found for appeal' % len(page_not_found))
logger.warn('%s documents without appeals in system' % len(not_found))
def handle(self, *args, **options):
logger.info('Starting appeal document ingest')
# v smoke test
baseurl = 'https://www.ifrc.org/en/publications-and-reports/appeals/'
smoke_response = urlopen(baseurl)
joy_to_the_world = False
if smoke_response.code == 200:
joy_to_the_world = True # We log the success later, when we know the numeric results.
else:
body = {
"name":
"ingest_appeal_docs",
"message":
'Error ingesting appeals_docs on url ' + baseurl +
', error_code: ' + smoke_response.code,
"status":
CronJobStatus.ERRONEOUS
}
CronJob.sync_cron(body)
# ^ smoke test
if options['fullscan']:
# If the `--fullscan` option is passed, check ALL appeals
print('Doing a full scan of all Appeals')
qset = Appeal.objects.all()
else:
# By default, only check appeals for the past 3 months where Appeal Documents is 0
now = datetime.now()
three_months_ago = now - relativedelta(months=3)
# This was the original qset, but it wouldn't get newer docs for the same Appeals
#qset = Appeal.objects.filter(appealdocument__isnull=True).filter(end_date__gt=three_months_ago)
qset = Appeal.objects.filter(end_date__gt=three_months_ago)
# First get all Appeal Codes
appeal_codes = [a.code for a in qset]
# Modify code taken from https://pastebin.com/ieMe9yPc to scrape `publications-and-reports` and find
# Documents for each appeal code
output = []
page_not_found = []
for code in appeal_codes:
code = code.replace(' ', '')
docs_url = baseurl + '?ac=' + code + '&at=0&c=&co=&dt=1&f=&re=&t=&ti=&zo='
try:
response = urlopen(docs_url)
except: # if we get an error fetching page for an appeal, we ignore it
page_not_found.append(code)
continue
soup = BeautifulSoup(response.read(), "lxml")
div = soup.find('div', id='cw_content')
for t in div.findAll('tbody'):
output = output + self.makelist(t)
# Once we have all Documents in output, we add all missing Documents to the associated Appeal
not_found = []
existing = []
created = []
acodes = list(set([a[2] for a in output]))
for code in acodes:
try:
appeal = Appeal.objects.get(code=code)
except ObjectDoesNotExist:
not_found.append(code)
continue
existing_docs = list(appeal.appealdocument_set.all())
docs = [a for a in output if a[2] == code]
for doc in docs:
doc[0] = 'https://www.ifrc.org' + doc[0] if doc[0].startswith(
'/docs'
) else doc[
0] # href only contains relative path to the document if it's available at the ifrc.org site
exists = len(
[a for a in existing_docs if a.document_url == doc[0]]) > 0
if exists:
existing.append(doc[0])
else:
try:
created_at = self.parse_date(doc[5])
except:
created_at = None
AppealDocument.objects.create(
document_url=doc[0],
name=doc[4],
created_at=created_at,
appeal=appeal,
)
created.append(doc[0])
text_to_log = []
text_to_log.append('%s appeal documents created' % len(created))
text_to_log.append('%s existing appeal documents' % len(existing))
text_to_log.append('%s pages not found for appeal' %
len(page_not_found))
for t in text_to_log:
logger.info(t)
# body = { "name": "ingest_appeal_docs", "message": t, "status": CronJobStatus.SUCCESSFUL }
# CronJob.sync_cron(body)
if len(not_found):
t = '%s documents without appeals in system' % len(not_found)
logger.warn(t)
body = {
"name": "ingest_appeal_docs",
"message": t,
"num_result": len(not_found),
"status": CronJobStatus.WARNED
}
CronJob.sync_cron(body)
if (joy_to_the_world):
body = {
"name":
"ingest_appeal_docs",
"message":
'Done ingesting appeals_docs on url ' + baseurl +
', %s appeal document(s) were created, %s already exist, %s not found'
% (len(created), len(existing), len(page_not_found)),
"num_result":
len(created),
"status":
CronJobStatus.SUCCESSFUL
}
CronJob.sync_cron(body)
def send_notification(subject, recipients, html, is_followed_event=False):
if not EMAIL_USER or not EMAIL_API_ENDPOINT:
logger.warn('Cannot send notifications.')
logger.warn(
'No username and/or API endpoint set as environment variables.')
return
# If it's not PROD only able to use test e-mail addresses which are set in the env var
to_addresses = recipients
if int(IS_PROD) != 1:
logger.info('Using test email addresses...')
to_addresses = []
for eml in test_emails:
if eml and (eml in recipients):
to_addresses.append(eml)
recipients_as_string = ','.join(to_addresses)
# Encode with base64 into bytes, then converting it back to strings for the JSON
payload = {
"FromAsBase64":
str(base64.b64encode(EMAIL_USER.encode('utf-8')), 'utf-8'),
"ToAsBase64":
str(base64.b64encode('*****@*****.**'.encode('utf-8')), 'utf-8'),
"CcAsBase64":
"",
"BccAsBase64":
str(base64.b64encode(recipients_as_string.encode('utf-8')), 'utf-8'),
"SubjectAsBase64":
str(base64.b64encode(subject.encode('utf-8')), 'utf-8'),
"BodyAsBase64":
str(base64.b64encode(html.encode('utf-8')), 'utf-8'),
"IsBodyHtml":
True,
"TemplateName":
"",
"TemplateLanguage":
""
}
# The response contains the GUID (res.text) which could be used for future requests if something is wrong
# TODO: It would be best to store the GUIDs and relevant mail parameters in some way, somewhere
res = requests.post(EMAIL_API_ENDPOINT, json=payload)
if res.status_code == 200:
logger.info('E-mails were sent successfully.')
elif res.status_code == 401 or res.status_code == 403:
logger.info(
'Authorization/authentication failed ({}) failed to the e-mail sender API.'
.format(res.status_code))
elif res.status_code == 500:
logger.error('Could not reach the e-mail sender API.')
return res.text
# # Old methods for sending email notifications (Leaving this here in case we ever need it for reference)
# class SendMail(threading.Thread):
# def __init__(self, recipients, msg, **kwargs):
# if int(IS_PROD) == 1:
# self.recipients = recipients
# else:
# logger.info('Using test email addresses...')
# self.recipients = []
# for eml in TEST_EMAILS:
# if eml and (eml in recipients):
# self.recipients.append(eml)
# self.msg = msg
# super(SendMail, self).__init__(**kwargs)
# def run(self):
# try:
# server = smtplib.SMTP(EMAIL_HOST, EMAIL_PORT)
# server.ehlo()
# server.starttls()
# server.ehlo()
# succ = server.login(EMAIL_USER, EMAIL_PASS)
# if 'successful' not in str(succ[1]):
# body = { "name": "notification", "message": 'Error contacting ' + EMAIL_HOST + ' smtp server for notifications', "status": CronJobStatus.ERRONEOUS }
# CronJob.sync_cron(body)
# # ^ Mainly for index_and_notify.py cronjob log
# if len(self.recipients) > 0:
# server.sendmail(EMAIL_USER, self.recipients, self.msg.as_string())
# server.quit()
# logger.info('Notifications sent!')
# except smtplib.SMTPAuthenticationError:
# logger.error('SMTPAuthenticationError')
# logger.error('Cannot send notification')
# logger.error(str(smtplib.SMTPAuthenticationError)[:100])
# def send_notification(subject, recipients, html):
# if not username or not password:
# logger.warn('No username and/or password set as environment variables')
# logger.warn('Cannot send notification')
# return
# msg = MIMEMultipart('alternative')
# msg['Subject'] = '[IFRCGO] %s' % subject
# msg['From'] = username.upper()
# msg['To'] = '*****@*****.**'
# text_body = MIMEText(strip_tags(html), 'plain')
# html_body = MIMEText(html, 'html')
# msg.attach(text_body)
# msg.attach(html_body)
# SendMail(['*****@*****.**'] + recipients, msg).start()
