def test_init_logger(self) -> None:
Log.init_logger(log_name=TEST_MODULE_NAME)
mock_makedirs.assert_called_once_with('logs', exist_ok=True)
(_, handler_kwargs) = mock_handler.call_args
assert handler_kwargs['filename'] == LOG_FILE_PATH
(_, config_kwargs) = mock_basic_config.call_args
assert config_kwargs['level'] == logging.INFO
for tweet in tweets:
self.show_tweet_media(tweet)
def show_rt_media(self, user: TwitterUser) -> None:
logger.info(
f'Show RT tweet media. user={user.id}. pages={self.tweet_page}, count={self.tweet_count}, '
f'since_id={user.since_id}')
for tweets in tweepy.Cursor(self.api.user_timeline,
id=user.id,
tweet_mode='extended',
count=self.tweet_count,
since_id=user.since_id).pages(
self.tweet_page):
if user.since_id < tweets.since_id:
user.since_id = tweets.since_id
for tweet in tweets:
if not has_attributes(tweet, 'retweeted_status'):
continue
self.show_tweet_media(tweet)
logger: logging.Logger = logging.getLogger(__name__)
if __name__ == '__main__':
Log.init_logger(log_name='twitter')
logger = logging.getLogger(__name__)
twitter_user = TwitterUser(id='TwitterJP')
t = Twitter()
Twitter.Debug(t).show_rt_media(twitter_user)
with self._get_connection() as connection:
with connection.cursor() as cursor:
cursor.execute(query=query, vars=(tweet_ids, ))
return cursor.fetchall()
def fetch_all_failed_upload_medias(self) -> List[Tuple[str, str, str]]:
logger.debug(
'Fetch url and description from failed_upload_media table.')
query: str = 'SELECT url, description, user_id ' \
'FROM failed_upload_media'
with self._get_connection() as connection:
with connection.cursor() as cursor:
cursor.execute(query=query)
return cursor.fetchall()
def delete_failed_upload_media(self, url: str) -> None:
logger.debug(f'Delete row url={url} from failed_upload_media table.')
query: str = 'DELETE FROM failed_upload_media ' \
'WHERE url = %s'
with self._get_connection() as connection:
with connection.cursor() as cursor:
cursor.execute(query=query, vars=(url, ))
logger: logging.Logger = logging.getLogger(__name__)
if __name__ == '__main__':
Log.init_logger(log_name='store')
logger = logging.getLogger(__name__)
db = Store()
def main(self) -> None:
interval_minutes: int = int(
Env.get_environment('INTERVAL', default='5'))
user_ids: str = Env.get_environment('TWITTER_USER_IDS', required=True)
user_list: List[TwitterUser] = [
TwitterUser(id=user_id) for user_id in user_ids.split(',')
]
while True:
try:
for user in user_list:
logger.info(
f'Crawling start. user = {user.id}, mode={self.twitter.mode}'
)
self.crawling_tweets(user)
except Exception as e:
logger.exception(f'Crawling error exception={e.args}')
logger.info(f'Interval. sleep {interval_minutes} minutes.')
time.sleep(interval_minutes * 60)
logger: logging.Logger = logging.getLogger(__name__)
if __name__ == '__main__':
Log.init_logger(log_name='crawler')
logger = logging.getLogger(__name__)
crawler = Crawler()
crawler.main()
if 'is_video' in shortcode_media and shortcode_media[
'is_video'] and 'video_url' in shortcode_media:
media_list.append(shortcode_media['video_url'])
continue
if 'display_url' in shortcode_media:
media_list.append(shortcode_media['display_url'])
continue
if 'edges' not in shortcode_media['edge_sidecar_to_children']:
continue
for media in shortcode_media['edge_sidecar_to_children']['edges']:
if 'node' not in media:
continue
if 'is_video' in media['node'] and media['node'][
'is_video'] and 'video_url' in media['node']:
media_list.append(media['node']['video_url'])
elif 'display_url' in media['node']:
media_list.append(media['node']['display_url'])
return media_list
logger: logging.Logger = logging.getLogger(__name__)
if __name__ == '__main__':
Log.init_logger(log_name='instagram')
logger = logging.getLogger(__name__)
instagram = Instagram('https://www.instagram.com/p/B3IWnLkBD4M/')
print(instagram.get_media_urls())
#!/usr/bin/python3
import logging
from app import crawler
from app.log import Log
def main() -> None:
crawler.Crawler().main()
if __name__ == '__main__':
Log.init_logger(log_name='run')
logger: logging.Logger = logging.getLogger(__name__)
main()
logger = logging.getLogger(__name__)
def test_init_logger__env_level(self, logger_level: str, level: int) -> None:
os.environ['LOGGING_LEVEL'] = logger_level
Log.init_logger(log_name=TEST_MODULE_NAME)
(_, config_kwargs) = mock_basic_config.call_args
assert config_kwargs['level'] == level
