def _get_detail(self, detail_page_url: str) -> AnimeDetailInfo: """引擎管理器负责调用, 捕获异常""" try: return self.get_detail(detail_page_url) except Exception as e: logger.error(f"Catch exception: {e} when processing {detail_page_url}") return AnimeDetailInfo()
def _search(self, keyword: str) -> List[DanmakuMetaInfo]: """引擎管理器负责调用, 捕获异常""" try: return self.search(keyword) except Exception as e: logger.error(f"Catch exception: {e} when searching for {keyword}") return []
def handle(self, *args, **options): logger.info('Moving Read only users out of staff status...') users = self.get_readonly_users() if len(users) > 0: logger.info('Revoking staff status from %s user%s' % (len(users), 's' if len(users) > 1 else '')) num_updated = 0 for u in users: u.is_staff = False try: u.save() except Exception as e: logger.error(str(e)[:100]) logger.error('Could not update user %s' % u.email) continue num_updated += 1 logger.info(' %s user%s updated' % (num_updated, 's' if num_updated > 1 else '')) logger.info('... user%s moving completed' % ('s' if len(users) > 1 else '')) else: logger.info('... not found any users to be moved') ifrc_users = self.get_ifrc_domain_users() ifrc_grp = Group.objects.get(name='IFRC Admins') if len(ifrc_users) > 0: logger.info('Adding IFRC Admins Group membership to %s user%s' % (len(ifrc_users), 's' if len(ifrc_users) > 1 else '')) num_i_updated = 0 for u in ifrc_users: ifrc_grp.user_set.add(u) num_i_updated += 1 logger.info(' %s user%s added' % (num_i_updated, 's' if num_i_updated > 1 else '')) logger.info('... user%s adding to IFRC Admins Group completed' % ('s' if len(ifrc_users) > 1 else '')) else: logger.info('... not found any users to be put into IFRC Admins')
def _get_danmaku(self, cid: str) -> Dict: """引擎管理器负责调用, 捕获异常""" try: return self.get_danmaku(cid) except Exception as e: logger.error(f"Catch exception: {e} when parsing danmaku {cid}") return {}
def remove_child_events_from_es(sender, instance, using, **kwargs): ''' Handle Emergency Elasticsearch indexes ''' model = instance.__class__.__name__ try: if model == 'Event': curr_record = Event.objects.filter(id=instance.id).first() # If new record, do nothing, index_and_notify should handle it if curr_record is None: return if curr_record.parent_event is None and instance.parent_event: # Delete ES record if Emergency became a child delete_es_index(instance) elif curr_record.parent_event and instance.parent_event is None: # Add back ES record if Emergency became a parent (index_elasticsearch.py) create_es_index(instance) elif model == 'Country': curr_record = Country.objects.filter(id=instance.id).first() if instance.in_search: if not curr_record: create_es_index(instance) else: if not curr_record.in_search and instance.in_search: create_es_index(instance) elif curr_record.in_search and not instance.in_search: delete_es_index(instance) else: update_es_index(instance) else: delete_es_index(instance) except Exception as ex: logger.error(f'Failed to index a Country, error: {str(ex)[:512]}')
def _get_detail(self, play_page_url: str) -> DanmakuCollection: """引擎管理器负责调用, 捕获异常""" try: return self.get_detail(play_page_url) except Exception as e: logger.error(f"Catch exception: {e} when processing {play_page_url}") return DanmakuCollection()
def push_table_to_index(self, model): query = model.objects.all() data = [self.convert_for_bulk(s) for s in list(query)] created, errors = bulk(client=ES_CLIENT, actions=data) logger.info('Created %s records' % created) if len(errors): logger.error('Produced the following errors:') logger.error('[%s]' % ', '.join(map(str, errors)))
def make_response_for(self, video: Video) -> requests.Response: """获取视频对应的 handler 对象, 用于代理访问数据并返回响应给客户端""" if not video: logger.error(f"Invalid request") return requests.Response() target_handler = self._handlers.get(video.handler) if not target_handler: logger.error(f"VideoHandler not found: {video.handler}") return requests.Response() return target_handler(video).make_response()
def get_anime_detail(self, meta: AnimeMetaInfo) -> AnimeDetailInfo: """解析一部番剧的详情页,返回包含视频列表的详细信息""" if not meta: logger.error(f"Invalid request") return AnimeDetailInfo() target_engine = self._engines.get(meta.engine) if not target_engine: logger.error(f"Engine not found: {meta.engine}") return AnimeDetailInfo() return target_engine()._get_detail(meta.detail_page_url)
def get_danmaku_detail(self, meta: DanmakuMetaInfo) -> DanmakuCollection: """解析一部番剧的详情页,返回包含视频列表的详细信息""" if not meta: logger.error(f"Invalid request") return DanmakuCollection() target_engine = self._danmaku_engine.get(meta.dm_engine) if not target_engine: logger.error(f"Danmaku Engine not found: {meta.dm_engine}") return DanmakuCollection() return target_engine()._get_detail(meta.play_page_url)
def get_danmaku_data(self, dmk: Danmaku) -> List: """解析一部番剧的详情页,返回包含视频列表的详细信息""" if not dmk: logger.error(f"Invalid request") return [] target_engine = self._danmaku_engine.get(dmk.dm_engine) if not target_engine: logger.error(f"Danmaku Engine not found: {dmk.dm_engine}") return [] return target_engine()._get_danmaku(dmk.cid)
def mongo_connect(): if connect("mongodb://*****:*****@0.0.0.0:27017/") != False: config['mongo_url'] = "mongodb://*****:*****@0.0.0.0:27017/" logger.info("数据库连接成功") else: if connect("mongodb://*****:*****@34.64.197.155:27017/") != False: config['mongo_url'] = "mongodb://*****:*****@34.64.197.155:27017/" logger.info("数据库连接成功") else: logger.error("请检查数据库设置") sys.exit()
def get_video_url(self, video: Video) -> str: """解析视频真实 url""" if not video: logger.error(f"Invalid request") return "error" target_handler = self._handlers.get(video.handler) if not target_handler: logger.error(f"VideoHandler not found: {video.handler}") return "error" target_handler = target_handler(video) return target_handler._get_real_url()
def add_tags_to_obj(obj, tags): # We clear all tags first, and then re-add them tag_molnix_ids = [t['id'] for t in tags] obj.molnix_tags.clear() for molnix_id in tag_molnix_ids: try: t = MolnixTag.objects.get(molnix_id=molnix_id) except: logger.error('ERROR - tag not found: %d' % molnix_id) continue obj.molnix_tags.add(t) obj.save()
def push_table_to_index(self, model): if model.__name__ == 'Event': query = model.objects.filter(parent_event__isnull=True) elif model.__name__ == 'Country': query = model.objects.filter(in_search=True) else: query = model.objects.all() data = [construct_es_data(s, is_create=True) for s in list(query)] created, errors = bulk(client=ES_CLIENT, actions=data) logger.info('Created %s records' % created) if len(errors): logger.error('Produced the following errors:') logger.error('[%s]' % ', '.join(map(str, errors)))
def handle(self, *args, **options): contacts = [ # Area/field/extent (en, fr, es, ar), name, email [ 'Africa Region', 'Région Afrique', 'Región de África', 'منطقة إفريقيا', 'Elly NANDASABA MULAHA', '*****@*****.**' ], [ 'Americas Region', 'Région Amériques', 'Región de América', 'منطقة الأمريكتين', 'Luis FANOVICH', '*****@*****.**' ], [ 'Asia Pacific Region', 'Région Asie-Pacifique', 'Región de Asia-Pacífico', 'منطقة آسيا والمحيط الهادئ', 'Dedi JUNADI', '*****@*****.**' ], [ 'Europe Region', 'Région Europe', 'Región de Europa', 'منطقة أوروبا', 'Anssi ANONEN', '*****@*****.**' ], [ 'MENA Region', 'Région MENA', 'Región de MENA', 'منطقة الشرق الأوسط وشمال أفريقيا', 'Ahmad AL JAMAL', '*****@*****.**' ] ] error = False c_to_add = [] contacts_empty = not MainContact.objects.exists() if contacts_empty: for con in contacts: contact = MainContact(extent=con[0], extent_en=con[0], extent_fr=con[1], extent_es=con[2], extent_ar=con[3], name=con[4], email=con[5]) c_to_add.append(contact) try: MainContact.objects.bulk_create(c_to_add) except Exception as ex: logger.error(f'Could not create MainContacts. Error: {str(ex)}') error = True if not error: logger.info('Successfully added MainContacts.')
def handle(self, *args, **options): logger.info('Starting appeal document ingest') # get latest url = 'https://proxy.hxlstandard.org/data.json?url=https%3A%2F%2Fdocs.google.com%2Fspreadsheets%2Fd%2F1gJ4N_PYBqtwVuJ10d8zXWxQle_i84vDx5dHNBomYWdU%2Fedit%3Fusp%3Dsharing' response = requests.get(url) if response.status_code != 200: logger.error('Error querying Appeal Document HXL API') raise Exception('Error querying Appeal Document HXL API') records = response.json() # some logging variables not_found = [] existing = [] created = [] # group records by appeal code acodes = list(set([a[2] for a in records[2:]])) for code in acodes: try: appeal = Appeal.objects.get(code=code) except ObjectDoesNotExist: not_found.append(code) continue existing_docs = list(appeal.appealdocument_set.all()) docs = [a for a in records if a[2] == code] for doc in docs: exists = len( [a for a in existing_docs if a.document_url == doc[0]]) > 0 if exists: existing.append(doc[0]) else: try: created_at = self.parse_date(doc[5]) except: created_at = None AppealDocument.objects.create( document_url=doc[0], name=doc[4], created_at=created_at, appeal=appeal, ) created.append(doc[0]) logger.info('%s appeal documents created' % len(created)) logger.info('%s existing appeal documents' % len(existing)) logger.warn('%s documents without appeals in system' % len(not_found))
def handle(self, *args, **options): logger.info('Starting appeals ingest') new, modified, bilaterals = self.get_new_or_modified_appeals() logger.info('%s current appeals' % Appeal.objects.all().count()) logger.info('Creating %s new appeals' % len(new)) logger.info('Updating %s existing appeals that have been modified' % len(modified)) num_created = 0 for i, r in enumerate(new): fields = self.parse_appeal_record(r, is_new_appeal=True) if fields[ 'code'] in bilaterals: # correction of the appeal record due to appealbilaterals api fields['amount_funded'] += round(bilaterals[fields['code']], 1) try: Appeal.objects.create(**fields) except Exception as e: logger.error(str(e)[:100]) logger.error('Could not create appeal with code %s' % fields['code']) continue num_created = num_created + 1 num_updated = 0 for i, r in enumerate(modified): fields = self.parse_appeal_record(r, is_new_appeal=False) if fields[ 'code'] in bilaterals: # correction of the appeal record due to appealbilaterals api fields['amount_funded'] += round(bilaterals[fields['code']], 1) try: appeal, created = Appeal.objects.update_or_create( code=fields['code'], defaults=fields) except Exception as e: logger.error(str(e)[:100]) logger.error('Could not update appeal with code %s' % fields['code']) continue num_updated = num_updated + 1 CronJobSum = Appeal.objects.all().count() logger.info('%s appeals created' % num_created) logger.info('%s appeals updated' % num_updated) logger.info('%s total appeals' % CronJobSum) logger.info('Appeals ingest completed') body = { "name": "ingest_appeals", "message": 'Appeals ingest completed, %s total appeals (%s new, %s existing).' % (CronJobSum, num_created, num_updated), "num_result": CronJobSum, "status": CronJobStatus.SUCCESSFUL } CronJob.sync_cron(body)
def handle(self, *args, **options): try: # Update countries which should appear in search inc_c = Country.objects.filter( independent=True, is_deprecated=False, record_type=1).update(in_search=True) # Update countries which should NOT appear in search # independent can be null too thus why negated check exc_c = Country.objects.filter(~Q(independent=True) | Q(is_deprecated=True) | ~Q(record_type=1)).update( in_search=False) logger.info('Successfully set in_search for Countries') except Exception as ex: logger.error( f'Failed to set in_search for Countries. Error: {str(ex)}')
def bulk(self, actions): try: created, errors = bulk(client=ES_CLIENT, actions=actions) if len(errors): logger.error('Produced the following errors:') logger.error('[%s]' % ', '.join(map(str, errors))) except Exception as e: logger.error('Could not index records') logger.error('%s...' % str(e)[:512])
def extract_table(dbfile, table): """ Extract a table from the Access database """ cmd = 'mdb-export %s %s' % (dbfile, table) try: output = subprocess.check_output(cmd.split(' ')).splitlines() except Exception as e: logger.error(e) output = [o.decode('utf-8') for o in output] reader = csv.reader(output, delimiter=',', quotechar='"') records = [] for i, row in enumerate(reader): if i == 0: header = row else: d = {header[i]: l for i, l in enumerate(row)} records.append(d) return records
def delete_es_index(instance): ''' instance needs an es_id() ''' if ES_CLIENT and ES_PAGE_NAME: # To make sure it doesn't run for tests if hasattr(instance, 'es_id'): try: deleted, errors = bulk(client=ES_CLIENT, actions=[{ '_op_type': 'delete', '_index': ES_PAGE_NAME, '_type': 'page', '_id': instance.es_id() }]) logger.info(f'Deleted {deleted} records') log_errors(errors) except Exception: logger.error( 'Could not reach Elasticsearch server or index was already missing.' ) else: logger.warning('instance does not have an es_id() method')
def run(self): try: server = smtplib.SMTP(settings.EMAIL_HOST, settings.EMAIL_PORT) server.ehlo() server.starttls() server.ehlo() succ = server.login(settings.EMAIL_USER, settings.EMAIL_PASS) if 'successful' not in str(succ[1]): cron_rec = { "name": "notification", "message": 'Error contacting ' + settings.EMAIL_HOST + ' smtp server for notifications', "status": CronJobStatus.ERRONEOUS } CronJob.sync_cron(cron_rec) if len(self.recipients) > 0: server.sendmail(settings.EMAIL_USER, self.recipients, self.msg.as_string()) server.quit() logger.info('E-mails were sent successfully.') except Exception as exc: logger.error( 'Could not send emails with Python smtlib, exception: {} -- {}' .format(type(exc).__name__, exc.args)) ex = '' try: ex = str(exc.args) except Exception as exctwo: logger.error(exctwo.args) cron_rec = { "name": "notification", "message": 'Error sending out email with Python smtplib: {}'.format(ex), "status": CronJobStatus.ERRONEOUS } CronJob.sync_cron(cron_rec)
def get_new_or_modified_appeals(self): use_local_file = True if os.getenv( 'DJANGO_DB_NAME') == 'test' and os.path.exists( 'appeals.json') else False new = [] modified = [] if use_local_file: # read from static file for development logger.info('Using local appeals.json file') with open('appeals.json') as f: modified = json.loads(f.read()) else: # get latest, determine which appeals need to be ingested based on last modified time logger.info('Querying appeals API for new appeals data') url = 'http://go-api.ifrc.org/api/appeals' auth = (os.getenv('APPEALS_USER'), os.getenv('APPEALS_PASS')) response = requests.get(url, auth=auth) if response.status_code != 200: logger.error('Error querying Appeals API') raise Exception('Error querying Appeals API') records = response.json() # write the current record file to local disk with open('appeals.json', 'w') as outfile: json.dump(records, outfile) since_last_checked = datetime.utcnow().replace( tzinfo=timezone.utc) - timedelta(minutes=90) codes = [a.code for a in Appeal.objects.all()] for r in records: if not r['APP_code'] in codes: new.append(r) last_modified = self.parse_date(r['APP_modifyTime']) if last_modified > since_last_checked: modified.append(r) return new, modified
def detect_video_format(self) -> str: """判断视频真正的格式, url 可能没有视频后缀""" # 尝试从 url 提取后缀 url = self._get_real_url() try: ext = url.split("?")[0].split(".")[-1].lower() if ext in ["mp4", "flv"]: return ext if ext == "m3u8": return "hls" except (IndexError, AttributeError): pass # 视频的元数据中包含了视频的格式信息, 在视频开头寻找十六进制标识符推断视频格式 format_hex = { "mp4": ["69736F6D", "70617663", "6D703432", "4D50454734", "4C617666"], "flv": ["464C56"], "hls": ["4558544D3355"] } _, data_iter = self._get_stream_from_server(0, 512) if not data_iter: logger.warning("Could not get video stream from server") return "unknown" logger.debug("Detecting video format from binary stream") video_meta = next(data_iter).hex().upper() for format_, hex_list in format_hex.items(): for hex_sign in hex_list: if hex_sign in video_meta: logger.debug(f"Video format: {format_}") return format_ logger.error("Could not detect video format from stream") logger.debug("Video raw binary stream (512byte):") logger.debug(video_meta) return "unknown"
def run(self): try: server = smtplib.SMTP('smtp.office365.com', '587') server.ehlo() server.starttls() server.ehlo() server.login(username, password) server.sendmail(username, self.recipients, self.msg.as_string()) server.quit() logger.info('Notifications sent!') except SMTPAuthenticationError: logger.error('SMTPAuthenticationError') logger.error('Cannot send notification') logger.error(str(SMTPAuthenticationError)[:100])
def run(self): try: server = smtplib.SMTP(emailhost, emailport) server.ehlo() server.starttls() server.ehlo() server.login(username, password) if len(self.recipients) > 0: server.sendmail(username, self.recipients, self.msg.as_string()) server.quit() logger.info('Notifications sent!') except SMTPAuthenticationError: logger.error('SMTPAuthenticationError') logger.error('Cannot send notification') logger.error(str(SMTPAuthenticationError)[:100])
def handle(self, *args, **options): logger.info('Starting appeals ingest') new, modified = self.get_new_or_modified_appeals() logger.info('%s current appeals' % Appeal.objects.all().count()) logger.info('Creating %s new appeals' % len(new)) logger.info('Updating %s existing appeals that have been modified' % len(modified)) num_created = 0 for i, r in enumerate(new): fields = self.parse_appeal_record(r, is_new_appeal=True) try: Appeal.objects.create(**fields) except Exception as e: logger.error(str(e)[:100]) logger.error('Could not create appeal with code %s' % fields['code']) continue num_created = num_created + 1 num_updated = 0 for i, r in enumerate(modified): fields = self.parse_appeal_record(r, is_new_appeal=False) try: appeal, created = Appeal.objects.update_or_create( code=fields['code'], defaults=fields) except Exception as e: logger.error(str(e)[:100]) logger.error('Could not update appeal with code %s' % fields['code']) continue num_updated = num_updated + 1 logger.info('%s appeals created' % num_created) logger.info('%s appeals updated' % num_updated) logger.info('%s total appeals' % Appeal.objects.all().count()) logger.info('Appeals ingest completed')
def handle(self, *args, **options): molnix = MolnixApi(url=settings.MOLNIX_API_BASE, username=settings.MOLNIX_USERNAME, password=settings.MOLNIX_PASSWORD) try: molnix.login() except Exception as ex: msg = 'Failed to login to Molnix API: %s' % str(ex) logger.error(msg) create_cron_record(CRON_NAME, msg, CronJobStatus.ERRONEOUS) return try: countries = molnix.get_countries() deployments = molnix.get_deployments() open_positions = molnix.get_open_positions() except Exception as ex: msg = 'Failed to fetch data from Molnix API: %s' % str(ex) logger.error(msg) create_cron_record(CRON_NAME, msg, CronJobStatus.ERRONEOUS) return try: used_tags = get_unique_tags(deployments, open_positions) add_tags(used_tags) positions_messages, positions_warnings, positions_created = sync_open_positions( open_positions, molnix, countries) deployments_messages, deployments_warnings, deployments_created = sync_deployments( deployments, molnix, countries) except Exception as ex: msg = 'Unknown Error occurred: %s' % str(ex) logger.error(msg) create_cron_record(CRON_NAME, msg, CronJobStatus.ERRONEOUS) return msg = get_status_message(positions_messages, deployments_messages, positions_warnings, deployments_warnings) num_records = positions_created + deployments_created has_warnings = len(positions_warnings) > 0 or len( deployments_warnings) > 0 cron_status = CronJobStatus.WARNED if has_warnings else CronJobStatus.SUCCESSFUL create_cron_record(CRON_NAME, msg, cron_status, num_result=num_records) molnix.logout()
def get_new_or_modified_appeals(self): use_local_file = True if os.getenv( 'DJANGO_DB_NAME') == 'test' and os.path.exists( 'appeals.json') else False new = [] modified = [] if use_local_file: # read from static file for development logger.info('Using local appeals.json file') with open('appeals.json') as f: modified = json.loads(f.read()) logger.info('Using local appealbilaterals.json file') with open('appealbilaterals.json') as f: records = json.loads(f.read()) bilaterals = {} for r in records: # code duplication ¤ if r['APP_Code'] and r['AmountCHF']: if r['APP_Code'] in bilaterals.keys(): bilaterals[r['APP_Code']] += r['AmountCHF'] else: bilaterals[r['APP_Code']] = r['AmountCHF'] else: # get latest BILATERALS logger.info('Querying appeals API for new appeals data') url = 'http://go-api.ifrc.org/api/appealbilaterals' auth = (os.getenv('APPEALS_USER'), os.getenv('APPEALS_PASS')) response = requests.get(url, auth=auth) if response.status_code != 200: text_to_log = 'Error querying AppealBilaterals API at ' + url logger.error(text_to_log) logger.error(response.content) body = { "name": "ingest_appeals", "message": text_to_log, "status": CronJobStatus.ERRONEOUS } # not every case is catched here, e.g. if the base URL is wrong... CronJob.sync_cron(body) raise Exception(text_to_log) records = response.json() # write the current record file to local disk with open('appealbilaterals.json', 'w') as outfile: json.dump(records, outfile) bilaterals = {} for r in records: # code duplication ¤ if r['APP_Code'] and r['AmountCHF']: if r['APP_Code'] in bilaterals.keys(): bilaterals[r['APP_Code']] += r['AmountCHF'] else: bilaterals[r['APP_Code']] = r['AmountCHF'] # get latest APPEALS logger.info('Querying appeals API for new appeals data') url = 'http://go-api.ifrc.org/api/appeals' auth = (os.getenv('APPEALS_USER'), os.getenv('APPEALS_PASS')) response = requests.get(url, auth=auth) if response.status_code != 200: logger.error('Error querying Appeals API') raise Exception('Error querying Appeals API') records = response.json() # write the current record file to local disk with open('appeals.json', 'w') as outfile: json.dump(records, outfile) codes = [a.code for a in Appeal.objects.all()] for r in records: # Temporary filtering, the manual version should be kept: if r['APP_code'] in ['MDR65002', 'MDR00001', 'MDR00004']: continue #if r['APP_code'] != 'MDRMZ014': # Debug to test bilateral additions or other specific appeals # continue if not r['APP_code'] in codes: new.append(r) # We use all records, do NOT check if last_modified > since_last_checked modified.append(r) return new, modified, bilaterals