def test_init_logger(self) -> None: Log.init_logger(log_name=TEST_MODULE_NAME) mock_makedirs.assert_called_once_with('logs', exist_ok=True) (_, handler_kwargs) = mock_handler.call_args assert handler_kwargs['filename'] == LOG_FILE_PATH (_, config_kwargs) = mock_basic_config.call_args assert config_kwargs['level'] == logging.INFO
for tweet in tweets: self.show_tweet_media(tweet) def show_rt_media(self, user: TwitterUser) -> None: logger.info( f'Show RT tweet media. user={user.id}. pages={self.tweet_page}, count={self.tweet_count}, ' f'since_id={user.since_id}') for tweets in tweepy.Cursor(self.api.user_timeline, id=user.id, tweet_mode='extended', count=self.tweet_count, since_id=user.since_id).pages( self.tweet_page): if user.since_id < tweets.since_id: user.since_id = tweets.since_id for tweet in tweets: if not has_attributes(tweet, 'retweeted_status'): continue self.show_tweet_media(tweet) logger: logging.Logger = logging.getLogger(__name__) if __name__ == '__main__': Log.init_logger(log_name='twitter') logger = logging.getLogger(__name__) twitter_user = TwitterUser(id='TwitterJP') t = Twitter() Twitter.Debug(t).show_rt_media(twitter_user)
with self._get_connection() as connection: with connection.cursor() as cursor: cursor.execute(query=query, vars=(tweet_ids, )) return cursor.fetchall() def fetch_all_failed_upload_medias(self) -> List[Tuple[str, str, str]]: logger.debug( 'Fetch url and description from failed_upload_media table.') query: str = 'SELECT url, description, user_id ' \ 'FROM failed_upload_media' with self._get_connection() as connection: with connection.cursor() as cursor: cursor.execute(query=query) return cursor.fetchall() def delete_failed_upload_media(self, url: str) -> None: logger.debug(f'Delete row url={url} from failed_upload_media table.') query: str = 'DELETE FROM failed_upload_media ' \ 'WHERE url = %s' with self._get_connection() as connection: with connection.cursor() as cursor: cursor.execute(query=query, vars=(url, )) logger: logging.Logger = logging.getLogger(__name__) if __name__ == '__main__': Log.init_logger(log_name='store') logger = logging.getLogger(__name__) db = Store()
def main(self) -> None: interval_minutes: int = int( Env.get_environment('INTERVAL', default='5')) user_ids: str = Env.get_environment('TWITTER_USER_IDS', required=True) user_list: List[TwitterUser] = [ TwitterUser(id=user_id) for user_id in user_ids.split(',') ] while True: try: for user in user_list: logger.info( f'Crawling start. user = {user.id}, mode={self.twitter.mode}' ) self.crawling_tweets(user) except Exception as e: logger.exception(f'Crawling error exception={e.args}') logger.info(f'Interval. sleep {interval_minutes} minutes.') time.sleep(interval_minutes * 60) logger: logging.Logger = logging.getLogger(__name__) if __name__ == '__main__': Log.init_logger(log_name='crawler') logger = logging.getLogger(__name__) crawler = Crawler() crawler.main()
if 'is_video' in shortcode_media and shortcode_media[ 'is_video'] and 'video_url' in shortcode_media: media_list.append(shortcode_media['video_url']) continue if 'display_url' in shortcode_media: media_list.append(shortcode_media['display_url']) continue if 'edges' not in shortcode_media['edge_sidecar_to_children']: continue for media in shortcode_media['edge_sidecar_to_children']['edges']: if 'node' not in media: continue if 'is_video' in media['node'] and media['node'][ 'is_video'] and 'video_url' in media['node']: media_list.append(media['node']['video_url']) elif 'display_url' in media['node']: media_list.append(media['node']['display_url']) return media_list logger: logging.Logger = logging.getLogger(__name__) if __name__ == '__main__': Log.init_logger(log_name='instagram') logger = logging.getLogger(__name__) instagram = Instagram('https://www.instagram.com/p/B3IWnLkBD4M/') print(instagram.get_media_urls())
#!/usr/bin/python3 import logging from app import crawler from app.log import Log def main() -> None: crawler.Crawler().main() if __name__ == '__main__': Log.init_logger(log_name='run') logger: logging.Logger = logging.getLogger(__name__) main() logger = logging.getLogger(__name__)
def test_init_logger__env_level(self, logger_level: str, level: int) -> None: os.environ['LOGGING_LEVEL'] = logger_level Log.init_logger(log_name=TEST_MODULE_NAME) (_, config_kwargs) = mock_basic_config.call_args assert config_kwargs['level'] == level