def test_raises_value_error_if_config_file_arg_does_not_exist(self):
invalid_path = f'{self.test_dir}/does_not_exist.ini'
with self.assertRaises(ValueError) as e:
MediumConfigHelper._retrieve_cookies(invalid_path, 'handle')
msg = e.exception.args[0]
self.assertIn('does not exist', msg)
def test_incorrect_account_name_raises_value_error(self, mock_cfg_open):
mock_cfg_open.return_value = self.fake_config
with self.assertRaises(ValueError) as e:
MediumConfigHelper._validate_config('irrelevant', 'scooby_doo')
msg = e.exception.args[0]
self.assertIn("not found in section", msg)
def test_missing_uid_triggers_value_error(self, mock_cfg_open):
missing_sid = TextIOWrapper(BytesIO(b'[handle]\nsid = foo'))
mock_cfg_open.return_value = missing_sid
with self.assertRaises(ValueError) as e:
MediumConfigHelper._validate_config('irrelevant', 'handle')
msg = e.exception.args[0]
self.assertIn("not properly formed", msg)
def test_excess_key_values_does_not_trigger_error(self, mock_cfg_open):
data = '[handle]\nsid=foo\nuid=bar\nextra=helloworld\nextra2=qwerty'
mock_cfg_open.return_value = TextIOWrapper(BytesIO(bytes(data, 'utf-8')))
cfg = MediumConfigHelper._validate_config('irrelevant', 'handle')
self.assertIsInstance(cfg, configparser.ConfigParser)
def main():
## PARSE ARGS
parser = get_argparser()
args = parser.parse_args()
# TODO - make this a plain path; not a directory argument
command = args.command
## EXECUTE COMMANDS
if command == 'fetch_cookies':
from medium_stats.cookie_fetcher import MediumAuthorizer
email, password = unpack_email_pwd(args)
me = MediumAuthorizer(args.u, email, password)
me.sign_in()
me.save_cookies(args.creds)
print(section_break)
elif command in ['scrape_user', 'scrape_publication']:
args = parse_scraper_args(args, parser)
if args.creds:
cfg = MediumConfigHelper(args.creds, args.u)
sid, uid = cfg.sid, cfg.uid
else:
sid, uid = args.sid, args.uid
modes = list(args.mode)
get_folders = lambda x: [x[m]['folder'] for m in modes]
print('\nGetting Preliminary Data...', end='\n\n')
if command == 'scrape_user':
username = args.u
sg = StatGrabberUser(username,
sid,
uid,
args.start,
args.end,
already_utc=True)
folders = get_folders(user_mode_attrs)
sub_dir = create_directories(args.output_dir, sg.slug, folders)
# get summary stats to derive article_ids and user creation_time
data = sg.get_summary_stats()
articles = sg.get_article_ids(data)
if 'summary' in modes:
write_stats(sg, data, 'summary', sg.now, sub_dir)
else:
url = args.s
sg = StatGrabberPublication(url,
sid,
uid,
args.start,
args.end,
already_utc=True)
folders = get_folders(pub_mode_attrs)
sub_dir = create_directories(args.output_dir, sg.slug, folders)
data = sg.get_all_story_overview()
articles = sg.get_article_ids(data)
if 'story_overview' in modes:
write_stats(sg, data, 'story_overview', sg.now, sub_dir)
# go through remainder of modes
remaining = [
m for m in modes if m not in ('summary', 'story_overview')
]
for m in remaining:
if m == 'events':
data = get_stats(sg, m, sg.now)
else:
data = get_stats(sg, m, sg.now, articles)
write_stats(sg, data, m, sg.now, sub_dir)
print('All done!')
def test_returns_dict_of_keys_values_under_account_name_header(self):
actual = MediumConfigHelper._retrieve_cookies(self.creds_path, 'handle')
expected = {'sid': 'foo', 'uid': 'bar'}
self.assertEqual(actual, expected)
def test_returns_dict(self):
res = MediumConfigHelper._retrieve_cookies(self.creds_path, 'handle')
self.assertIsInstance(res, dict)
def test_returns_configparser_object(self, mock_cfg_open):
mock_cfg_open.return_value = self.fake_config
cfg = MediumConfigHelper._validate_config('irrelevant', 'handle')
self.assertIsInstance(cfg, configparser.ConfigParser)
def test_has_attr_uid(self):
expected_attr = 'bar'
cfg = MediumConfigHelper(self.creds_path, 'handle')
self.assertEqual(cfg.uid, expected_attr)
def test_has_attr_cookies(self):
expected_attr = {'sid': 'foo', 'uid': 'bar'}
cfg = MediumConfigHelper(self.creds_path, 'handle')
self.assertEqual(cfg.cookies, expected_attr)