class YoutubeTemp(Youtube): def __init__(self, vinfo): super().__init__(None) self.vinfo = vinfo self.vid = None self.db = Database('Queues') self.logger = get_logger('YoutubeTemp') @staticmethod def get_temp_vid(vlink): reg = r"watch\?v=([A-Za-z0-9_-]{11})" idre = re.compile(reg) _id = vlink["_id"] vid = vlink["Link"] vid = re.search(idre, vid).group(1) return {'Vid': vid, 'Id': _id} def check(self): self.vinfo = self.get_temp_vid(self.vinfo) self.vid = self.vinfo['Vid'] html = get("https://www.youtube.com/watch?v=" f"{self.vid}") if r'"isLive\":true' in html: video_dict = self.getlive_title(self.vid) process_video(video_dict) self.db.delete(self.vinfo) else: self.logger.info(f'Not found Live') def run(self) -> None: self.check()
def save_to_database(self): db = Database() query = None buffer = [] empty = False while not empty: try: query = self.db_queue.get(timeout=240) except Empty: self.logger.warning('Database queue empty.') empty = True buffer.append(query) if len(buffer) >= self.config['database']['bufferLimit'] or ( empty and len(buffer) > 0): buffer.sort(key=lambda k: k['table']) for key, group in groupby(buffer, key=lambda k: k['table']): db_attempts = 3 while db_attempts: try: db.insert(key, [g['columns'] for g in group]) break except Exception as err: self.logger.error( 'Failed to save to database: {} Attempting {} more times.' .format(repr(err), db_attempts)) db_attempts -= 1 buffer.clear()
def upload_video(video_dict): share_url = bd_upload(video_dict['Title']) if share_url: if enable_db: db = Database('Video') db.insert(video_dict['Title'], share_url, video_dict['Date']) bot(f"[下载提示] {video_dict['Title']} 已上传, 请查看页面") else: raise RuntimeError(f'Upload {video_dict["Title"]} failed')
def get_views(self, videos): d = Database() with open('.COLLECT') as f: collect_id = int(f.readline().strip()) views = d.select('collect_video', *['video_id','MAX(view_count) as view_count'],where=['COLLECT_id > {} GROUP BY video_id'.format(collect_id - self.days - 10)]) for v in videos: try: v['view_count'] = next(item for item in views if item['video_id'] == v['video_id'])['view_count'] except StopIteration as e: print('Error with video {}:\n{}\n\n'.format(v, repr(e))) v['view_count'] = 0
def __init__(self, target_id): super().__init__(target_id) self.module = 'Youtube' self.api_key = api_key # 品质设置 self.database = Database('Queues') self.logger = get_logger('Youtube')
def get_ids(self): db = Database() results = {} try: channel_ids = db.select('channel', *['channel_id', 'yt_id']) self.channel_ids = [c['yt_id'] for c in channel_ids] for c in channel_ids: results[c['yt_id']] = c['channel_id'] video_ids = db.select('video', *['video_id', 'yt_id']) for v in video_ids: results[v['yt_id']] = v['video_id'] except Exception as err: self.logger.error('Failed to retrieve database IDs. {}'.format( repr(err))) return results
def start_temp_daemon(): db = Database('Queues') while True: event = [] for target_url in db.select(): p = YoutubeTemp(target_url) event.append(p) p.start() is_running = True while is_running: has_running = False for p in event: if p.is_alive(): has_running = True if not has_running: is_running = False logger = get_logger('YoutubeTemp') logger.info('A check has finished.') sleep(sec)
async def start_temp(): db = Database('Queues') logger = logging.getLogger('run.youtube.temp') while True: temp_tasks = [] for video in await db.select(): y = YoutubeTemp(video) temp_tasks.append(y.check()) await asyncio.gather(*temp_tasks) logger.info('Finished a check') await asyncio.sleep(config['sec'])
def upload_video(video_dict): upload_way_dict = {'bd': BDUpload, 's3': S3Upload} upload_way = upload_way_dict.get(upload_by) uploader = upload_way() uploader.upload_item(f"{ddir}/{video_dict['Title']}", video_dict['Title']) if upload_by == 'bd': share_url = uploader.share_item(video_dict['Title']) if enable_db: db = Database('Video') db.insert(video_dict['Title'], share_url, video_dict['Date']) elif upload_by == 's3': if enable_db: db = Database('Video') db.insert(video_dict['Title'], f"gets3/{quote(video_dict['Title'])}", video_dict['Date']) else: raise RuntimeError(f'Upload {video_dict["Title"]} failed') bot(f"[下载提示] {video_dict['Title']} 已上传, 请查看页面")
def upload_video(video_dict, user_config): upload_way_dict = {'bd': BDUpload, 's3': S3Upload} upload_way = upload_way_dict.get(config['upload_by']) uploader = upload_way() ddir = get_ddir(user_config) uploader.upload_item(f"{ddir}/{video_dict['Title']}", video_dict['Title']) if config['upload_by'] == 'bd': share_url = uploader.share_item(video_dict['Title']) if config['enable_mongodb']: db = Database(user_map(video_dict['User'])) db.insert(video_dict['Title'], share_url, video_dict['Date']) elif config['upload_by'] == 's3': if config['enable_mongodb']: db = Database(user_map(video_dict['User'])) db.insert(video_dict['Title'], f"gets3/{quote(video_dict['Title'])}", video_dict['Date']) else: raise RuntimeError(f'Upload {video_dict["Title"]} failed') bot(f"[下载提示] {video_dict['Title']} 已上传, 请查看https://matsuri.design/", user_config)
def insert_video(collection: str, data: dict): db = Database(collection) db.auto_insert(data['Title'], data)
def __init__(self, vinfo): super().__init__(None) self.vinfo = vinfo self.vid = None self.db = Database('Queues') self.logger = get_logger('YoutubeTemp')
def __init__(self, vinfo): super().__init__(config['youtube']['users'][0]) self.vinfo = vinfo self.db = Database('Queues') self.logger = logging.getLogger('run.youtube.temp')
def collect_info(self, key): api = APIRequest(api_key=key) db = Database() queue_attempts = 3 # Gets channel from queue while queue_attempts: try: self.api_semaphore.acquire(blocking=True, timeout=120) channel_id = self.api_queue.get(timeout=120) print('{} - Thread {}'.format(self.api_queue.qsize(), current_thread().name)) except Empty: queue_attempts -= 1 self.logger.warning( 'API queue empty. Attempting {} more time(s).'.format( queue_attempts)) continue finally: self.api_semaphore.release() # Checks if channel is already in database, otherwise saves it channel_dbid = self.db_ids.get(channel_id, 0) if not channel_dbid: try: db_id_query = db.select( 'channel', *['channel_id'], where=['yt_id LIKE "{}"'.format(channel_id)]) if not db_id_query: request, response = api.list( 'channels', **{ 'part': 'snippet', 'id': channel_id }) try: snippet = response['items'][0]['snippet'] channel_query = { 'yt_id': channel_id, 'title': snippet['title'], 'description': snippet['description'], 'published_at': self.parse_date(snippet['publishedAt'], return_datetime=True).strftime( '%Y-%m-%d %H:%M:%S') } channel_dbid = db.insert('channel', channel_query) self.db_ids[channel_id] = channel_dbid except KeyError as err: self.logger.error( 'KeyError while getting channel info: {}'. format(repr(err))) continue except Exception as err: self.logger.error( 'Error while getting channel info: {}'.format( repr(err))) continue else: channel_dbid = db_id_query[0]['channel_id'] self.db_ids[channel_id] = channel_dbid except Exception as err: self.logger.error(repr(err)) continue # Collects channel, attempting 3 times collect_attempts = 3 while collect_attempts: try: self.collect_channel(api, channel_id) break except Exception as err: collect_attempts -= 1 self.logger.error( 'Failed to collect channel {}: {} Attempting {} more times.' .format(channel_id, repr(err), collect_attempts)) if not collect_attempts: self.logger.info( 'Could not collect channel: {}'.format(channel_id)) continue # Retrieves recent videos playlist_id = 'UU' + channel_id[ 2:] # Playlist ID of channel c's uploads, can be derived from channel ID fetch_attempts = 3 request, response = None, None while fetch_attempts: try: request, response = api.list( 'playlistItems', **{ 'part': 'contentDetails', 'playlistId': playlist_id, 'maxResults': 50 }) break except Exception as err: fetch_attempts -= 1 self.logger.error( 'Error while getting videos for channel {}: {} Attempting {} more times.' .format(channel_id, repr(err), fetch_attempts)) if not fetch_attempts: self.logger.error( 'Failed to get videos for {}'.format(channel_id)) continue video_list = [] limit_reached = False while request and not limit_reached: for v in response['items']: video_id = v['contentDetails']['videoId'] published_at = self.parse_date( v['contentDetails']['videoPublishedAt']) if self.now - published_at > self.limit: limit_reached = True break try: db_id_query = \ db.select('video', 'video_id', where=['yt_id LIKE "{}"'.format(video_id)]) if not db_id_query: request, response = api.list( 'videos', **{ 'part': 'snippet,contentDetails', 'id': video_id }) try: snippet = response['items'][0]['snippet'] content_details = response['items'][0][ 'contentDetails'] video_query = { 'yt_id': video_id, 'title': snippet['title'], 'description': snippet['description'], 'channel_id': channel_dbid, 'length_seconds': int( parse_duration( content_details['duration']). total_seconds()), 'published_at': self.parse_date( snippet['publishedAt'], return_datetime=True).strftime( '%Y-%m-%d %H:%M:%S') } video_dbid = db.insert('video', video_query) self.db_ids[video_id] = video_dbid except KeyError as err: self.logger.error( 'KeyError while getting video info: {}'. format(repr(err))) continue else: video_dbid = db_id_query[0]['video_id'] self.db_ids[video_id] = video_dbid except Exception as err: self.logger.error( 'Error while getting video data: {}'.format( repr(err))) continue video_list.append(video_id) collect_attempts = 3 while collect_attempts: try: self.collect_videos(api, video_list) break except Exception as err: collect_attempts -= 1 self.logger.error( 'Failed to collect videos from channel {}: ' '{} Attempting {} more times.'.format( channel_id, repr(err), collect_attempts)) finally: video_list.clear() if not collect_attempts: self.logger.info( 'Could not collect videos from channel: {}'.format( channel_id)) if not limit_reached: execute_attempts = 3 while execute_attempts: try: request, response = api.list_next( 'playlistItems', request, response) break except Exception as err: execute_attempts -= 1 self.logger.error( 'Video fetch response error: {} Attempting {} more time(s)' .format(' '.join(err.args), execute_attempts)) continue if not execute_attempts: break queue_attempts = 3 self.logger.info('Finished execution for Thread {}'.format( get_ident()))