def handle(self, *args, **options): logger.info('Starting appeal document ingest') # get latest url = 'https://proxy.hxlstandard.org/data.json?url=https%3A%2F%2Fdocs.google.com%2Fspreadsheets%2Fd%2F1gJ4N_PYBqtwVuJ10d8zXWxQle_i84vDx5dHNBomYWdU%2Fedit%3Fusp%3Dsharing' response = requests.get(url) if response.status_code != 200: logger.error('Error querying Appeal Document HXL API') raise Exception('Error querying Appeal Document HXL API') records = response.json() # some logging variables not_found = [] existing = [] created = [] # group records by appeal code acodes = list(set([a[2] for a in records[2:]])) for code in acodes: try: appeal = Appeal.objects.get(code=code) except ObjectDoesNotExist: not_found.append(code) continue existing_docs = list(appeal.appealdocument_set.all()) docs = [a for a in records if a[2] == code] for doc in docs: exists = len( [a for a in existing_docs if a.document_url == doc[0]]) > 0 if exists: existing.append(doc[0]) else: try: created_at = self.parse_date(doc[5]) except: created_at = None AppealDocument.objects.create( document_url=doc[0], name=doc[4], created_at=created_at, appeal=appeal, ) created.append(doc[0]) logger.info('%s appeal documents created' % len(created)) logger.info('%s existing appeal documents' % len(existing)) logger.warn('%s documents without appeals in system' % len(not_found))
def get_dbfile(): ftphost = os.environ.get('GO_FTPHOST', None) ftpuser = os.environ.get('GO_FTPUSER', None) ftppass = os.environ.get('GO_FTPPASS', None) dbpass = os.environ.get('GO_DBPASS', None) if ftphost is None or ftpuser is None or ftppass is None: if os.path.exists('URLs.mdb'): logger.info('No credentials in env, using local MDB database file') logger.warn( 'If this occurs outside development, contact an administrator') return 'URLs.mdb' else: raise Exception( 'FTP credentials not provided (GO_FTPHOST, GO_FTPUSER, GO_FTPPASS)' ) if dbpass is None: raise Exception( 'Database encryption password not provided (GO_DBPASS)') logger.info('Attempting connection to FTP') ftp = FTP(ftphost) ftp.login(user=ftpuser, passwd=ftppass) ftp.cwd('/dmis/') data = [] ftp.dir('-t', data.append) filename = data[-1].split()[3] # check if we already have this file files = glob('URLs*zip') if filename in files and os.path.exists('URLs.mdb'): ftp.quit() return 'URLs.mdb' # clean up old files for f in files: os.remove(f) logger.info('Fetching %s' % filename) with open(filename, 'wb') as f: ftp.retrbinary('RETR ' + filename, f.write, 2014) ftp.quit() logger.info('Unzipping database file') zp = ZipFile(filename) zp.extractall('./', pwd=dbpass.encode('cp850', 'replace')) return 'URLs.mdb'
def send_notification(subject, recipients, html): if not username or not password: logger.warn( 'No EMAIL_USER and/or EMAIL_PASS set as environment variables') logger.warn('Cannot send notification') return msg = MIMEMultipart('alternative') msg['Subject'] = '[IFRCGO] %s' % subject msg['From'] = username.upper() msg['To'] = ', '.join(recipients) text_body = MIMEText(strip_tags(html), 'plain') html_body = MIMEText(html, 'html') msg.attach(text_body) msg.attach(html_body) SendMail(recipients, msg).start()
def handle(self, *args, **options): logger.info('Starting Deployment ingest') # url = 'https://proxy.hxlstandard.org/data.json?url=https%3A%2F%2Fdocs.google.com%2Fspreadsheets%2Fd%2F1CBvledFYc_uwlvHTvJE0SYS7_mPGU2L-zhrqbB4KNIA%2Fedit%23gid%3D0&header-row=1' # not enough. url = 'https://proxy.hxlstandard.org/data.json?tagger-match-all=on&' \ + 'tagger-01-header=year&' \ + 'tagger-01-tag=%23a1&' \ + 'tagger-02-header=%2Aappeal+code&' \ + 'tagger-02-tag=%23a2&' \ + 'tagger-03-header=region&' \ + 'tagger-03-tag=%23a3&' \ + 'tagger-04-header=country&' \ + 'tagger-04-tag=%23a4&' \ + 'tagger-05-header=location&' \ + 'tagger-05-tag=%23a5&' \ + 'tagger-06-header=disaster+type&' \ + 'tagger-06-tag=%23a6&' \ + 'tagger-07-header=%2Adisaster+name&' \ + 'tagger-07-tag=%23a7&' \ + 'tagger-08-header=%2Aname&' \ + 'tagger-08-tag=%23a8&' \ + 'tagger-09-header=%2Adeploying+ns+%2F+ifrc+office&' \ + 'tagger-09-tag=%23a9&' \ + 'tagger-10-header=%2Agender&' \ + 'tagger-10-tag=%23b1&' \ + 'tagger-11-header=language&' \ + 'tagger-11-tag=%23b2&' \ + 'tagger-12-header=%2Aposition&' \ + 'tagger-12-tag=%23b3&' \ + 'tagger-13-header=%2Atype&' \ + 'tagger-13-tag=%23b4&' \ + 'tagger-14-header=supported+by+ns&' \ + 'tagger-14-tag=%23b5&' \ + 'tagger-15-header=availability&' \ + 'tagger-15-tag=%23b6&' \ + 'tagger-16-header=%2Aexp+start+date&' \ + 'tagger-16-tag=%23b7&' \ + 'tagger-17-header=%2Aexp+duration&' \ + 'tagger-17-tag=%23b8&' \ + 'tagger-18-header=%2Aalert&' \ + 'tagger-18-tag=%23b9&' \ + 'tagger-19-header=deployment+message&' \ + 'tagger-19-tag=%23c1&' \ + 'tagger-20-header=%2Astart+of+mission&' \ + 'tagger-20-tag=%23c2&' \ + 'tagger-21-header=%2Aend+of+mission&' \ + 'tagger-21-tag=%23c3&' \ + 'tagger-22-header=deployment+duration&' \ + 'tagger-22-tag=%23c4&' \ + 'tagger-23-header=deployed&' \ + 'tagger-23-tag=%23c5&' \ + 'tagger-24-header=rotation&' \ + 'tagger-24-tag=%23c6&' \ + 'tagger-25-header=comments&' \ + 'tagger-25-tag=%23c7&' \ + 'url=https%3A%2F%2Fdocs.google.com%2Fspreadsheets%2Fd%2F1CBvledFYc_uwlvHTvJE0SYS7_mPGU2L-zhrqbB4KNIA%2Fedit%23gid%3D0&' \ + 'header-row=1' response = requests.get(url) if response.status_code != 200: logger.error('Error querying Deployment HXL API') raise Exception('Error querying Deployment HXL API') records = response.json() # some logging variables not_found = [] existing = [] created = [] columns = [a.replace('*', '').replace(' ', '') for a in records[0]] # ['Year', 'AppealCode', 'Region', 'Country', 'Location', 'Disastertype', 'Disastername', 'Name', 'DeployingNS/IFRCOffice', 'Gender', 'Language', 'Position', 'Type', 'SupportedbyNS', 'Availability', 'Expstartdate', 'expduration', 'Alert', 'Deploymentmessage', 'Startofmission', 'Endofmission', 'DeploymentDuration', 'Deployed', 'Rotation', 'Comments'] # 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 # if empty name -> Alert, otherwise -> Deployment # OBSOLETE: # # group records by appeal code # acodes = list(set([a[2] for a in records[2:]])) # for code in acodes: # try: # appeal = Appeal.objects.get(code=code) # except ObjectDoesNotExist: # not_found.append(code) # continue # # existing_docs = list(appeal.appealdocument_set.all()) # docs = [a for a in records if a[2] == code] # for doc in docs: # exists = len([a for a in existing_docs if a.document_url == doc[0]]) > 0 # if exists: # existing.append(doc[0]) # else: # try: # created_at = self.parse_date(doc[5]) # except: # created_at = None # # AppealDocument.objects.create( # document_url=doc[0], # name=doc[4], # created_at=created_at, # appeal=appeal, # ) # created.append(doc[0]) logger.info('%s Deployments created' % len(created)) logger.info('%s existing Deployments' % len(existing)) logger.warn('%s documents without appeals in system' % len(not_found))
def handle(self, *args, **options): # get latest filename = get_dbfile() # numeric details records details_rc = extract_table(filename, 'EW_Report_NumericDetails') # check for 1 record for each field report fids = [r['ReportID'] for r in details_rc] if len(set(fids)) != len(fids): raise Exception( 'More than one NumericDetails record for a field report') # numeric details records details_gov = extract_table(filename, 'EW_Report_NumericDetails_GOV') # check for 1 record for each field report fids = [r['ReportID'] for r in details_gov] if len(set(fids)) != len(fids): raise Exception( 'More than one NumericDetails record for a field report') # information info_table = extract_table(filename, 'EW_Report_InformationManagement') fids = [r['ReportID'] for r in info_table] if len(set(fids)) != len(fids): raise Exception( 'More than one InformationManagement record for a field report' ) ### many-to-many # actions taken actions_national = extract_table(filename, 'EW_Report_ActionTakenByRedCross') actions_foreign = extract_table(filename, 'EW_Report_ActionTakenByPnsRC') actions_federation = extract_table( filename, 'EW_Report_ActionTakenByFederationRC') # source types source_types = extract_table(filename, 'EW_lofSources') for s in source_types: SourceType.objects.get_or_create( pk=s['SourceID'], defaults={'name': s['SourceName']}) source_table = extract_table(filename, 'EW_Reports_Sources') # disaster response dr_table = extract_table(filename, 'EW_DisasterResponseTools') # check for 1 record for each field report fids = [r['ReportID'] for r in dr_table] if len(set(fids)) != len(fids): raise Exception( 'More than one DisasterResponseTools record for a field report' ) # contacts contacts = extract_table(filename, 'EW_Report_Contacts') # field report reports = extract_table(filename, 'EW_Reports') rids = [r.rid for r in FieldReport.objects.all()] num_reports_created = 0 logger.info('%s reports in database' % len(reports)) for i, report in enumerate(reports): # Skip reports that we've already ingested. # We don't have to update them because field reports can't be updated in DMIS. rid = report['ReportID'] if rid in rids: continue report_name = report['Summary'] report_description = report['BriefSummary'] report_dtype = DisasterType.objects.get( pk=PK_MAP[report['DisasterTypeID']]) record = { 'rid': rid, 'summary': report_name, 'description': report_description, 'dtype': report_dtype, 'status': report['StatusID'], 'request_assistance': report['GovRequestsInternAssistance'], 'actions_others': report['ActionTakenByOthers'], 'report_date': datetime.strptime(report['Inserted'], REPORT_DATE_FORMAT).replace(tzinfo=pytz.utc), } details = fetch_relation(details_rc, report['ReportID']) assert (len(details) <= 1) if len(details) > 0: details = details[0] record.update({ 'num_injured': details['NumberOfInjured'], 'num_dead': details['NumberOfCasualties'], 'num_missing': details['NumberOfMissing'], 'num_affected': details['NumberOfAffected'], 'num_displaced': details['NumberOfDisplaced'], 'num_assisted': details['NumberOfAssistedByRC'], 'num_localstaff': details['NumberOfLocalStaffInvolved'], 'num_volunteers': details['NumberOfVolunteersInvolved'], 'num_expats_delegates': details['NumberOfExpatsDelegates'] }) details = fetch_relation(details_gov, report['ReportID']) assert (len(details) <= 1) if len(details) > 0: details = details[0] record.update({ 'gov_num_injured': details['NumberOfInjured_GOV'], 'gov_num_dead': details['NumberOfDead_GOV'], 'gov_num_missing': details['NumberOfMissing_GOV'], 'gov_num_affected': details['NumberOfAffected_GOV'], 'gov_num_displaced': details['NumberOfDisplaced_GOV'], 'gov_num_assisted': details['NumberOfAssistedByGov_GOV'] }) info = fetch_relation(info_table, report['ReportID']) if len(info) > 0: info = {k: '' if v is None else v for k, v in info[0].items()} record.update({ 'bulletin': { '': 0, 'None': 0, 'Planned': 2, 'Published': 3 }[info['InformationBulletin']], 'dref': { '': 0, 'No': 0, 'Planned': 2, 'Yes': 3 }[info['DREFRequested']], 'dref_amount': 0 if info['DREFRequestedAmount'] == '' else float( info['DREFRequestedAmount']), 'appeal': { '': 0, 'Planned': 2, 'Yes': 3, 'NB': 0, 'No': 0, 'YES': 3 }[info['EmergencyAppeal']], 'appeal_amount': 0 if info['EmergencyAppealAmount'] == '' else float( info['EmergencyAppealAmount']), }) # disaster response response = fetch_relation(dr_table, report['ReportID']) if len(response) > 0: response = { k: '' if v is None else v for k, v in response[0].items() } record.update({ 'rdrt': { '': 0, 'No': 0, 'Yes': 3, 'Planned/Requested': 2 }[response['RDRT']], 'fact': { '': 0, 'No': 0, 'Yes': 3, 'Planned/Requested': 2 }[response['FACT']], 'eru_relief': { '': 0, 'Yes': 3, 'Planned/Requested': 2, 'No': 0 }[response['ERU']] }) field_report = FieldReport(**record) # Create an associated event object event_record = { 'name': report_name if len(report_name) else report_dtype.name, 'summary': report_description, 'dtype': report_dtype, 'disaster_start_date': datetime.utcnow().replace(tzinfo=timezone.utc), 'auto_generated': True, 'auto_generated_source': SOURCES['report_ingest'], } event = Event(**event_record) event.save() logger.info('Adding %s' % report_name) field_report.event = event field_report.save() num_reports_created = num_reports_created + 1 try: country = Country.objects.select_related().get( pk=report['CountryID']) except ObjectDoesNotExist: logger.warn('Could not find a matching country for %s' % report['CountryID']) country = None if country is not None: field_report.countries.add(country) event.countries.add(country) if country.region is not None: # No need to add a field report region, as that happens through a trigger. field_report.regions.add(country.region) event.regions.add(country.region) ### add items with foreignkeys to report # national red cross actions actions = fetch_relation(actions_national, report['ReportID']) if len(actions) > 0: txt = ' '.join( [a['Value'] for a in actions if a['Value'] is not None]) act = ActionsTaken(organization='NTLS', summary=txt, field_report=field_report) act.save() for pk in [a['ActionTakenByRedCrossID'] for a in actions]: act.actions.add(*Action.objects.filter(pk=pk)) # foreign red cross actions actions = fetch_relation(actions_foreign, report['ReportID']) if len(actions) > 0: txt = ' '.join( [a['Value'] for a in actions if a['Value'] is not None]) act = ActionsTaken(organization='PNS', summary=txt, field_report=field_report) act.save() for pk in [a['ActionTakenByRedCrossID'] for a in actions]: act.actions.add(*Action.objects.filter(pk=pk)) # federation red cross actions actions = fetch_relation(actions_federation, report['ReportID']) if len(actions) > 0: txt = ' '.join( [a['Value'] for a in actions if a['Value'] is not None]) act = ActionsTaken(organization='FDRN', summary=txt, field_report=field_report) act.save() for pk in [a['ActionTakenByRedCrossID'] for a in actions]: act.actions.add(*Action.objects.filter(pk=pk)) # sources sources = fetch_relation(source_table, report['ReportID']) for s in sources: spec = '' if s['Specification'] is None else s['Specification'] src = Source.objects.create( stype=SourceType.objects.get(pk=s['SourceID']), spec=spec, field_report=field_report) # disaster response response = fetch_relation(dr_table, report['ReportID']) # contacts contact = fetch_relation(contacts, report['ReportID']) if len(contact) > 0: # make sure just one contacts record assert (len(contact) == 1) contact = contact[0] fields = [ 'Originator', 'Primary', 'Federation', 'NationalSociety', 'MediaNationalSociety', 'Media' ] for f in fields: if contact_is_valid(contact, f): ct = FieldReportContact.objects.create( ctype=f, name=contact['%sName' % f], title=contact['%sFunction' % f], email=contact['%sContact' % f], field_report=field_report, ) total_reports = FieldReport.objects.all() logger.info('%s reports created' % num_reports_created) logger.info('%s reports in database' % total_reports.count()) # org type mapping org_types = { '1': 'NTLS', '2': 'DLGN', '3': 'SCRT', '4': 'ICRC', } last_login_threshold = timezone.now() - timedelta(days=365) # add users user_records = extract_table(filename, 'DMISUsers') processed_users = 0 for i, user_data in enumerate(user_records): if user_data['LoginLastSuccess'] == '': continue last_login = datetime.strptime( user_data['LoginLastSuccess'], REPORT_DATE_FORMAT, ) last_login = pytz.UTC.localize(last_login) # skip users who haven't logged in for a year if last_login < last_login_threshold: continue try: user = User.objects.get(username=user_data['UserName']) except ObjectDoesNotExist: user = None if user is None: name = user_data['RealName'].split() first_name = name[0] last_name = ' '.join(name[1:]) if len(name) > 1 else '' user = User.objects.create( username=user_data['UserName'], first_name=first_name if len(first_name) <= 30 else '', last_name=last_name if len(last_name) <= 30 else '', email=user_data['EmailAddress'], last_login=last_login, ) user.set_password(user_data['Password']) user.is_staff = True if user_data[ 'UserIsSysAdm'] == '1' else False # set user profile info user.profile.org = user_data['OrgTypeSpec'] if len( user_data['OrgTypeSpec']) <= 100 else '' user.profile.org_type = org_types.get(user_data['OrgTypeID']) user.profile.country = Country.objects.get( pk=user_data['CountryID']) user.profile.city = user_data['City'] if len( user_data['City']) <= 100 else '' user.profile.department = user_data['Department'] if len( user_data['Department']) <= 100 else '' user.profile.position = user_data['Position'] if len( user_data['Position']) <= 100 else '' user.profile.phone_number = user_data['PhoneNumberProf'] if len( user_data['PhoneNumberProf']) <= 100 else '' user.save() processed_users = processed_users + 1 logger.info('%s updated active user records' % len(processed_users))
def handle(self, *args, **options): logger.info('Starting appeal document ingest') if options['fullscan']: # If the `--fullscan` option is passed, check ALL appeals print('Doing a full scan of all Appeals') qset = Appeal.objects.all() else: # By default, only check appeals for the past 3 months where Appeal Documents is 0 now = datetime.now() three_months_ago = now - relativedelta(months=3) qset = Appeal.objects.filter(appealdocument__isnull=True).filter( end_date__gt=three_months_ago) # First get all Appeal Codes appeal_codes = [a.code for a in qset] # Modify code taken from https://pastebin.com/ieMe9yPc to scrape `publications-and-reports` and find # Documents for each appeal code output = [] page_not_found = [] for code in appeal_codes: code = code.replace(' ', '') docs_url = 'http://www.ifrc.org/en/publications-and-reports/appeals/?ac=' + code + '&at=0&c=&co=&dt=1&f=&re=&t=&ti=&zo=' try: response = urlopen(docs_url) except: # if we get an error fetching page for an appeal, we ignore it page_not_found.append(code) continue soup = BeautifulSoup(response.read(), "lxml") div = soup.find('div', id='cw_content') for t in div.findAll('tbody'): output = output + self.makelist(t) # Once we have all Documents in output, we add all missing Documents to the associated Appeal not_found = [] existing = [] created = [] acodes = list(set([a[2] for a in output])) for code in acodes: try: appeal = Appeal.objects.get(code=code) except ObjectDoesNotExist: not_found.append(code) continue existing_docs = list(appeal.appealdocument_set.all()) docs = [a for a in output if a[2] == code] for doc in docs: exists = len( [a for a in existing_docs if a.document_url == doc[0]]) > 0 if exists: existing.append(doc[0]) else: try: created_at = self.parse_date(doc[5]) except: created_at = None AppealDocument.objects.create( document_url=doc[0], name=doc[4], created_at=created_at, appeal=appeal, ) created.append(doc[0]) logger.info('%s appeal documents created' % len(created)) logger.info('%s existing appeal documents' % len(existing)) logger.info('%s pages not found for appeal' % len(page_not_found)) logger.warn('%s documents without appeals in system' % len(not_found))
def handle(self, *args, **options): logger.info('Starting appeal document ingest') # v smoke test baseurl = 'https://www.ifrc.org/en/publications-and-reports/appeals/' smoke_response = urlopen(baseurl) joy_to_the_world = False if smoke_response.code == 200: joy_to_the_world = True # We log the success later, when we know the numeric results. else: body = { "name": "ingest_appeal_docs", "message": 'Error ingesting appeals_docs on url ' + baseurl + ', error_code: ' + smoke_response.code, "status": CronJobStatus.ERRONEOUS } CronJob.sync_cron(body) # ^ smoke test if options['fullscan']: # If the `--fullscan` option is passed, check ALL appeals print('Doing a full scan of all Appeals') qset = Appeal.objects.all() else: # By default, only check appeals for the past 3 months where Appeal Documents is 0 now = datetime.now() three_months_ago = now - relativedelta(months=3) # This was the original qset, but it wouldn't get newer docs for the same Appeals #qset = Appeal.objects.filter(appealdocument__isnull=True).filter(end_date__gt=three_months_ago) qset = Appeal.objects.filter(end_date__gt=three_months_ago) # First get all Appeal Codes appeal_codes = [a.code for a in qset] # Modify code taken from https://pastebin.com/ieMe9yPc to scrape `publications-and-reports` and find # Documents for each appeal code output = [] page_not_found = [] for code in appeal_codes: code = code.replace(' ', '') docs_url = baseurl + '?ac=' + code + '&at=0&c=&co=&dt=1&f=&re=&t=&ti=&zo=' try: response = urlopen(docs_url) except: # if we get an error fetching page for an appeal, we ignore it page_not_found.append(code) continue soup = BeautifulSoup(response.read(), "lxml") div = soup.find('div', id='cw_content') for t in div.findAll('tbody'): output = output + self.makelist(t) # Once we have all Documents in output, we add all missing Documents to the associated Appeal not_found = [] existing = [] created = [] acodes = list(set([a[2] for a in output])) for code in acodes: try: appeal = Appeal.objects.get(code=code) except ObjectDoesNotExist: not_found.append(code) continue existing_docs = list(appeal.appealdocument_set.all()) docs = [a for a in output if a[2] == code] for doc in docs: doc[0] = 'https://www.ifrc.org' + doc[0] if doc[0].startswith( '/docs' ) else doc[ 0] # href only contains relative path to the document if it's available at the ifrc.org site exists = len( [a for a in existing_docs if a.document_url == doc[0]]) > 0 if exists: existing.append(doc[0]) else: try: created_at = self.parse_date(doc[5]) except: created_at = None AppealDocument.objects.create( document_url=doc[0], name=doc[4], created_at=created_at, appeal=appeal, ) created.append(doc[0]) text_to_log = [] text_to_log.append('%s appeal documents created' % len(created)) text_to_log.append('%s existing appeal documents' % len(existing)) text_to_log.append('%s pages not found for appeal' % len(page_not_found)) for t in text_to_log: logger.info(t) # body = { "name": "ingest_appeal_docs", "message": t, "status": CronJobStatus.SUCCESSFUL } # CronJob.sync_cron(body) if len(not_found): t = '%s documents without appeals in system' % len(not_found) logger.warn(t) body = { "name": "ingest_appeal_docs", "message": t, "num_result": len(not_found), "status": CronJobStatus.WARNED } CronJob.sync_cron(body) if (joy_to_the_world): body = { "name": "ingest_appeal_docs", "message": 'Done ingesting appeals_docs on url ' + baseurl + ', %s appeal document(s) were created, %s already exist, %s not found' % (len(created), len(existing), len(page_not_found)), "num_result": len(created), "status": CronJobStatus.SUCCESSFUL } CronJob.sync_cron(body)
def send_notification(subject, recipients, html, is_followed_event=False): if not EMAIL_USER or not EMAIL_API_ENDPOINT: logger.warn('Cannot send notifications.') logger.warn( 'No username and/or API endpoint set as environment variables.') return # If it's not PROD only able to use test e-mail addresses which are set in the env var to_addresses = recipients if int(IS_PROD) != 1: logger.info('Using test email addresses...') to_addresses = [] for eml in test_emails: if eml and (eml in recipients): to_addresses.append(eml) recipients_as_string = ','.join(to_addresses) # Encode with base64 into bytes, then converting it back to strings for the JSON payload = { "FromAsBase64": str(base64.b64encode(EMAIL_USER.encode('utf-8')), 'utf-8'), "ToAsBase64": str(base64.b64encode('*****@*****.**'.encode('utf-8')), 'utf-8'), "CcAsBase64": "", "BccAsBase64": str(base64.b64encode(recipients_as_string.encode('utf-8')), 'utf-8'), "SubjectAsBase64": str(base64.b64encode(subject.encode('utf-8')), 'utf-8'), "BodyAsBase64": str(base64.b64encode(html.encode('utf-8')), 'utf-8'), "IsBodyHtml": True, "TemplateName": "", "TemplateLanguage": "" } # The response contains the GUID (res.text) which could be used for future requests if something is wrong # TODO: It would be best to store the GUIDs and relevant mail parameters in some way, somewhere res = requests.post(EMAIL_API_ENDPOINT, json=payload) if res.status_code == 200: logger.info('E-mails were sent successfully.') elif res.status_code == 401 or res.status_code == 403: logger.info( 'Authorization/authentication failed ({}) failed to the e-mail sender API.' .format(res.status_code)) elif res.status_code == 500: logger.error('Could not reach the e-mail sender API.') return res.text # # Old methods for sending email notifications (Leaving this here in case we ever need it for reference) # class SendMail(threading.Thread): # def __init__(self, recipients, msg, **kwargs): # if int(IS_PROD) == 1: # self.recipients = recipients # else: # logger.info('Using test email addresses...') # self.recipients = [] # for eml in TEST_EMAILS: # if eml and (eml in recipients): # self.recipients.append(eml) # self.msg = msg # super(SendMail, self).__init__(**kwargs) # def run(self): # try: # server = smtplib.SMTP(EMAIL_HOST, EMAIL_PORT) # server.ehlo() # server.starttls() # server.ehlo() # succ = server.login(EMAIL_USER, EMAIL_PASS) # if 'successful' not in str(succ[1]): # body = { "name": "notification", "message": 'Error contacting ' + EMAIL_HOST + ' smtp server for notifications', "status": CronJobStatus.ERRONEOUS } # CronJob.sync_cron(body) # # ^ Mainly for index_and_notify.py cronjob log # if len(self.recipients) > 0: # server.sendmail(EMAIL_USER, self.recipients, self.msg.as_string()) # server.quit() # logger.info('Notifications sent!') # except smtplib.SMTPAuthenticationError: # logger.error('SMTPAuthenticationError') # logger.error('Cannot send notification') # logger.error(str(smtplib.SMTPAuthenticationError)[:100]) # def send_notification(subject, recipients, html): # if not username or not password: # logger.warn('No username and/or password set as environment variables') # logger.warn('Cannot send notification') # return # msg = MIMEMultipart('alternative') # msg['Subject'] = '[IFRCGO] %s' % subject # msg['From'] = username.upper() # msg['To'] = '*****@*****.**' # text_body = MIMEText(strip_tags(html), 'plain') # html_body = MIMEText(html, 'html') # msg.attach(text_body) # msg.attach(html_body) # SendMail(['*****@*****.**'] + recipients, msg).start()