def test_fifteen_retries(yahoo_response): r = yahoo_response('v1/groups/groupname/', ygError={}, status=500) yga = YahooGroupsAPI('groupname') with raises(yahoogroupsapi.Recoverable): yga.HackGroupInfo() assert len(r.calls) == 15
def test_one_retry(yahoo_response): result = {'ok': 'on second try'} r = yahoo_response('v1/groups/groupname/', ygError={}, status=500) r = yahoo_response('v1/groups/groupname/', result) yga = YahooGroupsAPI('groupname') json = yga.HackGroupInfo() assert len(r.calls) == 2 assert json == result
def test_unauthorized_error(yahoo_response): r = yahoo_response('v1/groups/groupname/', ygError={"hostname": "gapi17.grp.bf1.yahoo.com", "httpStatus": 401, "errorMessage": "User does not have READ permission for GROUP. Mess...", "errorCode": 1103, "sid": "SID:YHOO:groups.yahoo.com:00000000000000000000000000000000:0"}, ygPerms=YGPERMS_NONE, status=401) yga = YahooGroupsAPI('groupname') with raises(yahoogroupsapi.Unauthorized): yga.HackGroupInfo() assert len(r.calls) == 1
def test_get_json(yahoo_response, cookies): response = yahoo_response('v2/groups/groupname/files/a/2?param1=c¶m2=4', {'result': 'returned data'}) yga = YahooGroupsAPI('groupname', cookies) json = yga.get_json('files', 'a', 2, param1='c', param2=4) request = response.calls[0].request request_cookies = SimpleCookie() request_cookies.load(request.headers['Cookie']) assert dict(cookies) == {k: v.value for k, v in request_cookies.items()} assert json == {'result': 'returned data'}
def test_warc_enabled(): # Note that this does not use the responses mocking framework, as it conflicts with the warc captures. # This makes a real request to Yahoo, so might fail. url = 'https://groups.yahoo.com/api/v1/groups/test/' yga = YahooGroupsAPI('test') writer = BufferWARCWriter(gzip=False) yga.set_warc_writer(writer) yga.get_json('HackGroupInfo') expected = [(url, 'response'), (url, 'request')] actual = [(record.rec_headers['WARC-Target-URI'], record.rec_type) for record in ArchiveIterator(writer.get_stream())] assert expected == actual
def test_get_json(response, cookies): response.add( responses.GET, 'https://groups.yahoo.com/api/v2/groups/groupname/files/a/2?param1=c¶m2=4', json={'ygData': { 'result': 'returned data' }}) yga = YahooGroupsAPI('groupname', cookies) json = yga.get_json('files', 'a', 2, param1='c', param2=4) request = response.calls[0].request request_cookies = Cookie.SimpleCookie() request_cookies.load(request.headers['Cookie']) assert dict(cookies) == { k: v.value for k, v in request_cookies.iteritems() } assert json == {'result': 'returned data'}
help='Only archive database') pe = p.add_argument_group(title='Email Options') pe.add_argument('-r', '--no-reattach', action='store_true', help="Don't reattach attachment files to email") pe.add_argument('-s', '--no-save', action='store_true', help="Don't save email attachments as individual files") p.add_argument('--overwrite', action='store_true', help="Re-download and overwrite existing files and messages") p.add_argument('group', type=str) args = p.parse_args() yga = YahooGroupsAPI(args.group, args.cookie_t, args.cookie_y) if args.username: password = args.password or getpass.getpass() print "logging in..." if not yga.login(args.username, password): print "Login failed" sys.exit(1) if not (args.email or args.files or args.photos or args.database): args.email = args.files = args.photos = args.database = True skip_existing = not args.overwrite with Mkchdir(args.group): if args.email: with Mkchdir('email'): archive_email(yga,
raise e coloredlogs.install(level=log_level, **log_format) else: log_stdout_handler = logging.StreamHandler(sys.stdout) log_stdout_handler.setLevel(log_level) log_stdout_handler.setFormatter(log_formatter) root_logger.addHandler(log_stdout_handler) cookie_jar = init_cookie_jar(args.cookie_file, args.cookie_t, args.cookie_y, args.cookie_e) headers = {} if args.user_agent: headers['User-Agent'] = args.user_agent yga = YahooGroupsAPI(args.group, cookie_jar, headers, delay=args.delay) if not (args.email or args.files or args.photos or args.database or args.links or args.calendar or args.about or args.polls or args.attachments or args.members): args.email = args.files = args.photos = args.database = args.links = args.calendar = args.about = \ args.polls = args.attachments = args.members = True with Mkchdir(args.group, sanitize=False): log_file_handler = logging.FileHandler('archive.log') log_file_handler.setFormatter(log_formatter) root_logger.addHandler(log_file_handler) if args.warc: try: from warcio import WARCWriter
help= 'Output WARC file of raw network requests. [Requires warcio package installed]' ) p.add_argument('-v', '--verbose', action='store_true') p.add_argument('group', type=str) args = p.parse_args() if not args.verbose: log_stdout_handler.setLevel(logging.INFO) cookie_jar = init_cookie_jar(args.cookie_file, args.cookie_t, args.cookie_y, args.cookie_e) yga = YahooGroupsAPI(args.group, cookie_jar) if not (args.email or args.files or args.photos or args.database or args.links or args.calendar or args.about or args.polls or args.attachments or args.members): args.email = args.files = args.photos = args.database = args.links = args.calendar = args.about = \ args.polls = args.attachments = args.members = True with Mkchdir(args.group): log_file_handler = logging.FileHandler('archive.log') log_file_handler.setFormatter(log_formatter) root_logger.addHandler(log_file_handler) if args.warc: try: from warcio import WARCWriter
def test_not_authenticated_error(yahoo_response): r = yahoo_response('v1/groups/groupname/', status=307) yga = YahooGroupsAPI('groupname') with raises(yahoogroupsapi.NotAuthenticated): yga.HackGroupInfo() assert len(r.calls) == 15
def test_correct_ua(yahoo_response): r = yahoo_response('v1/groups/groupname/', {}) yga = YahooGroupsAPI('groupname', headers={'User-Agent': 'test'}) yga.HackGroupInfo() assert r.calls[0].request.headers['user-agent'] == 'test'
def test_not_authenticated_error(yahoo_response): r = yahoo_response('v1/groups/groupname/', status=307) yga = YahooGroupsAPI('groupname') with raises(yahoogroupsapi.Recoverable): # Temporary fix, replaced: yahoogroupsapi.NotAuthenticated yga.HackGroupInfo() assert len(r.calls) == 15
def main(): args = parse_arguments() # Setup logging root_logger = logging.getLogger() root_logger.setLevel(logging.DEBUG) log_format = { 'fmt': '%(asctime)s %(levelname)s %(name)s %(message)s', 'datefmt': '%Y-%m-%d %H:%M:%S.%f %Z' } log_formatter = CustomFormatter(**log_format) if args.verbose: log_level = logging.DEBUG elif args.quiet: log_level = logging.ERROR else: log_level = logging.INFO if args.colour: try: import coloredlogs except ImportError as e: sys.exit( "Error: Coloured logging output requires the 'coloredlogs' package to be installed." ) coloredlogs.install(level=log_level, **log_format) else: log_stdout_handler = logging.StreamHandler(sys.stdout) log_stdout_handler.setLevel(log_level) log_stdout_handler.setFormatter(log_formatter) root_logger.addHandler(log_stdout_handler) cookie_jar = init_cookie_jar(args.cookie_file, args.cookie_t, args.cookie_y, args.cookie_e) headers = {} if args.user_agent: headers['User-Agent'] = args.user_agent yga = YahooGroupsAPI(args.group, cookie_jar, headers, min_delay=args.delay) # Default to all unique content. This includes topics and raw email, # but not the full email download since that would duplicate html emails we get through topics. if not (args.email or args.files or args.photos or args.database or args.links or args.calendar or args.about or args.polls or args.attachments or args.members or args.topics or args.raw): args.files = args.photos = args.database = args.links = args.calendar = args.about = \ args.polls = args.attachments = args.members = args.topics = args.raw = True with Mkchdir(args.group, sanitize=False): log_file_handler = logging.FileHandler('archive.log', 'a', 'utf-8') log_file_handler.setFormatter(log_formatter) root_logger.addHandler(log_file_handler) if args.warc: try: from warcio import WARCWriter fhwarc = open('data.warc.gz', 'ab') warc_writer = WARCWriter(fhwarc) warcmeta = warc_writer.create_warcinfo_record( fhwarc.name, WARC_META_PARAMS) warc_writer.write_record(warcmeta) yga.set_warc_writer(warc_writer) except ImportError: logging.error( 'WARC output requires the warcio package to be installed.') exit(1) if args.overwrite: hacky_vars['file'] = True if args.email: with Mkchdir('email'): archive_email(yga, message_subset=args.ids, start=args.start, stop=args.stop, noAttachments=args.noattachments) if args.files: with Mkchdir('files'): archive_files(yga) if args.photos: with Mkchdir('photos'): archive_photos(yga) if args.topics: with Mkchdir('topics'): archive_topics(yga, noAttachments=args.noattachments) if args.raw: with Mkchdir('email'): archive_email(yga, message_subset=args.ids, start=args.start, stop=args.stop, skipHTML=True) if args.database: with Mkchdir('databases'): archive_db(yga) if args.links: with Mkchdir('links'): archive_links(yga) if args.about: with Mkchdir('about'): archive_about(yga) if args.polls: with Mkchdir('polls'): archive_polls(yga) if args.attachments: with Mkchdir('attachments'): archive_attachments(yga) if args.members: with Mkchdir('members'): archive_members(yga) if args.calendar: with Mkchdir('calendar'): archive_calendar(yga) if args.warc: fhwarc.close()