def search_action(cli_args): enricher = casanova.enricher(cli_args.file, cli_args.output, add=YOUTUBE_VIDEO_SNIPPET_CSV_HEADERS, keep=cli_args.select) loading_bar = LoadingBar('Searching videos', unit='video') def before_sleep_until_midnight(seconds): loading_bar.print( 'API limits reached. Will now wait until midnight Pacific time!') client = YouTubeAPIClient( cli_args.key, before_sleep_until_midnight=before_sleep_until_midnight) for row, query in enricher.cells(cli_args.column, with_rows=True): loading_bar.print('Searching for "%s"' % query) searcher = client.search(query, order=cli_args.order) if cli_args.limit: searcher = islice(searcher, cli_args.limit) for video in searcher: loading_bar.update() enricher.writerow(row, video.as_csv_row())
def twitter_scrape_action(cli_args): scraper = TwitterAPIScraper() # Stats loading_bar = LoadingBar('Collecting tweets', total=cli_args.limit, unit='tweet', stats={ 'tokens': 1, 'queries': 0 }) enricher = casanova.enricher(cli_args.file, cli_args.output, add=TWEET_FIELDS + ADDITIONAL_TWEET_FIELDS, keep=cli_args.select) def before_sleep(retry_state): exc = retry_state.outcome.exception() if isinstance(exc, TwitterPublicAPIRateLimitError): loading_bar.inc('tokens') else: loading_bar.inc('failures') loading_bar.print( 'Failed to call Twitter search. Will retry in %s' % prettyprint_seconds(retry_state.idle_for)) for row, query in enricher.cells(cli_args.query, with_rows=True): # Templating? if cli_args.query_template is not None: query = CUSTOM_FORMATTER.format(cli_args.query_template, value=query) loading_bar.print('Searching for "%s"' % query) loading_bar.inc('queries') iterator = scraper.search( query, limit=cli_args.limit, before_sleep=before_sleep, include_referenced_tweets=cli_args.include_refs, with_meta=True) try: for tweet, meta in iterator: loading_bar.update() tweet_row = format_tweet_as_csv_row(tweet) enricher.writerow(row, tweet_row + format_meta_row(meta)) except TwitterPublicAPIOverCapacityError: loading_bar.die('Got an "Over Capacity" error. Shutting down...')
def facebook_comments_action(cli_args): try: scraper = FacebookMobileScraper(cli_args.cookie, throttle=cli_args.throttle) except FacebookInvalidCookieError: if cli_args.cookie in COOKIE_BROWSERS: die([ 'Could not extract relevant cookie from "%s".' % cli_args.cookie ]) die([ 'Relevant cookie not found.', 'A Facebook authentication cookie is necessary to be able to scrape Facebook comments.', 'Use the --cookie flag to choose a browser from which to extract the cookie or give your cookie directly.' ]) # Enricher enricher = casanova.enricher(cli_args.file, cli_args.output, keep=cli_args.select, add=FACEBOOK_COMMENT_CSV_HEADERS) # Loading bar loading_bar = LoadingBar(desc='Scraping comments', unit='comment') for i, (row, url) in enumerate(enricher.cells(cli_args.column, with_rows=True), 1): try: batches = scraper.comments(url, per_call=True, detailed=True) except FacebookInvalidTargetError: loading_bar.print( 'Given url (line %i) is probably not a Facebook resource having comments: %s' % (i, url)) continue for details, batch in batches: for comment in batch: enricher.writerow(row, comment.as_csv_row()) loading_bar.update(len(batch)) loading_bar.update_stats(calls=details['calls'], replies=details['replies'], q=details['queue_size'], posts=i)
def twitter_scrape_action(cli_args): scraper = TwitterAPIScraper() # Stats loading_bar = LoadingBar( 'Collecting tweets', total=cli_args.limit, unit='tweet', stats={'tokens': 1, 'queries': 0} ) enricher = casanova.enricher( cli_args.file, cli_args.output, add=TWEET_FIELDS + ADDITIONAL_TWEET_FIELDS, keep=cli_args.select ) for row, query in enricher.cells(cli_args.query, with_rows=True): # Templating? if cli_args.query_template is not None: query = CUSTOM_FORMATTER.format( cli_args.query_template, value=query ) loading_bar.print('Searching for "%s"' % query) loading_bar.inc('queries') iterator = scraper.search( query, limit=cli_args.limit, include_referenced_tweets=cli_args.include_refs, with_meta=True ) try: for tweet, meta in iterator: loading_bar.update() tweet_row = format_tweet_as_csv_row(tweet) enricher.writerow(row, tweet_row + format_meta_row(meta)) except TwitterPublicAPIOverCapacityError: loading_bar.die('Got an "Over Capacity" error. Shutting down...')
def search_action(namespace, output_file): # Handling output single_query = namespace.file is sys.stdin and sys.stdin.isatty() if single_query: edit_namespace_with_csv_io(namespace, 'query') enricher = casanova.enricher( namespace.file, output_file, add=YOUTUBE_VIDEO_SNIPPET_CSV_HEADERS, keep=namespace.select ) loading_bar = LoadingBar( 'Searching videos', unit='video' ) def before_sleep_until_midnight(seconds): loading_bar.print('API limits reached. Will now wait until midnight Pacific time!') client = YouTubeAPIClient( namespace.key, before_sleep_until_midnight=before_sleep_until_midnight ) for row, query in enricher.cells(namespace.column, with_rows=True): loading_bar.print('Searching for "%s"' % query) searcher = client.search(query, order=namespace.order) if namespace.limit: searcher = islice(searcher, namespace.limit) for video in searcher: loading_bar.update() enricher.writerow(row, video.as_csv_row()) loading_bar.close()
def facebook_post_authors_action(cli_args): try: scraper = FacebookMobileScraper(cli_args.cookie, throttle=cli_args.throttle) except FacebookInvalidCookieError: if cli_args.cookie in COOKIE_BROWSERS: die([ 'Could not extract relevant cookie from "%s".' % cli_args.cookie ]) die([ 'Relevant cookie not found.', 'A Facebook authentication cookie is necessary to be able to scrape Facebook comments.', 'Use the --cookie flag to choose a browser from which to extract the cookie or give your cookie directly.' ]) # Enricher enricher = casanova.enricher( cli_args.file, cli_args.output, keep=cli_args.select, add=FACEBOOK_USER_CSV_HEADERS ) # Loading bar loading_bar = LoadingBar( desc='Finding authors', unit='post' ) for i, (row, post_url) in enumerate(enricher.cells(cli_args.column, with_rows=True), 1): loading_bar.update() try: author = scraper.post_author(post_url) except FacebookInvalidTargetError: loading_bar.print('Given url (line %i) is probably not a Facebook group post: %s' % (i, post_url)) continue enricher.writerow(row, author.as_csv_row() if author is not None else None)
def facebook_posts_action(cli_args): try: scraper = FacebookMobileScraper(cli_args.cookie, throttle=cli_args.throttle) except FacebookInvalidCookieError: if cli_args.cookie in COOKIE_BROWSERS: die([ 'Could not extract relevant cookie from "%s".' % cli_args.cookie ]) die([ 'Relevant cookie not found.', 'A Facebook authentication cookie is necessary to be able to scrape Facebook groups.', 'Use the --cookie flag to choose a browser from which to extract the cookie or give your cookie directly.' ]) # Enricher enricher = casanova.enricher( cli_args.file, cli_args.output, keep=cli_args.select, add=FACEBOOK_POST_CSV_HEADERS ) # Loading bar loading_bar = LoadingBar( desc='Scraping posts', unit='post' ) translated_langs = set() for i, (row, url) in enumerate(enricher.cells(cli_args.column, with_rows=True), 1): loading_bar.inc('groups') try: posts = scraper.posts(url) except FacebookInvalidTargetError: loading_bar.print('Given url (line %i) is probably not a Facebook group: %s' % (i, url)) continue for post in posts: if post.translated_text and post.translated_from not in translated_langs: translated_langs.add(post.translated_from) lines = [ 'Found text translated from %s!' % post.translated_from, 'Since it means original text may not be entirely retrieved you might want', 'to edit your Facebook language settings to add "%s" to' % post.translated_from, 'the "Languages you don\'t want to be offered translations for" list here:', 'https://www.facebook.com/settings/?tab=language' ] for line in lines: loading_bar.print(line) loading_bar.print() loading_bar.update() enricher.writerow(row, post.as_csv_row())
def fetch_action(cli_args, resolve=False, defer=None): # If we are hitting a single url we enable contents_in_report by default if not resolve and isinstance(cli_args.file, StringIO) and cli_args.contents_in_report is None: cli_args.contents_in_report = True if not resolve and cli_args.contents_in_report and cli_args.compress: raise InvalidArgumentsError('Cannot both --compress and output --contents-in-report!') # HTTP method http_method = cli_args.method # Cookie grabber get_cookie = None if cli_args.grab_cookies: get_cookie = grab_cookies(cli_args.grab_cookies) # Global headers global_headers = None if cli_args.headers: global_headers = {} for header in cli_args.headers: k, v = parse_http_header(header) global_headers[k] = v # Resume listener skipped_rows = 0 resuming_reader_loading = None if cli_args.resume and cli_args.output.can_resume(): resuming_reader_loading = LoadingBar( desc='Resuming', unit='line' ) def output_read_listener(event, row): nonlocal skipped_rows if event != 'output.row': return skipped_rows += 1 resuming_reader_loading.update() cli_args.output.listener = output_read_listener if resolve: additional_headers = RESOLVE_ADDITIONAL_HEADERS else: additional_headers = FETCH_ADDITIONAL_HEADERS if cli_args.contents_in_report: additional_headers = additional_headers + ['raw_contents'] # Enricher multiplex = None if cli_args.separator is not None: multiplex = (cli_args.column, cli_args.separator) enricher = casanova.threadsafe_enricher( cli_args.file, cli_args.output, add=additional_headers, keep=cli_args.select, total=cli_args.total, prebuffer_bytes=DEFAULT_PREBUFFER_BYTES, multiplex=multiplex ) if resuming_reader_loading is not None: resuming_reader_loading.close() if cli_args.column not in enricher.headers: raise InvalidArgumentsError('Could not find the "%s" column containing the urls in the given CSV file.' % cli_args.column) url_pos = enricher.headers[cli_args.column] filename_pos = None if not resolve and cli_args.filename is not None: if cli_args.filename not in enricher.headers: raise InvalidArgumentsError('Could not find the "%s" column containing the filenames in the given CSV file.' % cli_args.filename) filename_pos = enricher.headers[cli_args.filename] # Loading bar loading_bar = LoadingBar( desc='Fetching pages', total=enricher.total, unit='url', initial=skipped_rows ) defer(loading_bar.close) # NOTE: it could be dangerous with multithreaded execution, not to close it ourselves def update_loading_bar(result): nonlocal errors if result.error is not None: errors += 1 else: if resolve: status = result.stack[-1].status else: status = result.response.status if status >= 400: status_codes[status] += 1 stats = {'errors': errors} for code, count in status_codes.most_common(1): stats[str(code)] = count loading_bar.update_stats(**stats) loading_bar.update() only_shortened = getattr(cli_args, 'only_shortened', False) def url_key(item): url = item[1][url_pos].strip() if not url: return if only_shortened and not is_shortened_url(url): return # Url templating if cli_args.url_template: return cli_args.url_template.format(value=url) return url def request_args(domain, url, item): cookie = None # Cookie if get_cookie: cookie = get_cookie(url) # Headers headers = None if global_headers: headers = global_headers return { 'method': http_method, 'cookie': cookie, 'headers': headers } # Worker callback internals filename_builder = None files_writer = None if not resolve: try: filename_builder = FilenameBuilder( folder_strategy=cli_args.folder_strategy, template=cli_args.filename_template ) except TypeError: die([ 'Invalid "%s" --folder-strategy!' % cli_args.folder_strategy, 'Check the list at the end of the command help:', ' $ minet fetch -h' ]) files_writer = ThreadSafeFilesWriter(cli_args.output_dir) def worker_callback(result): # NOTE: at this point the callback is only fired on success row = result.item[1] response = result.response meta = result.meta if cli_args.keep_failed_contents and response.status != 200: return # First we need to build a filename filename_cell = row[filename_pos] if filename_pos else None formatter_kwargs = {} if cli_args.filename_template and 'line' in cli_args.filename_template: formatter_kwargs['line'] = enricher.wrap(row) try: filename = filename_builder( result.resolved, filename=filename_cell, ext=meta.get('ext'), formatter_kwargs=formatter_kwargs, compressed=cli_args.compress ) except FilenameFormattingError as e: result.error = e return meta['filename'] = filename # Decoding the response data? is_text = meta.get('is_text', False) original_encoding = meta.get('encoding', 'utf-8') data = response.data binary = True if is_text and (cli_args.standardize_encoding or cli_args.contents_in_report): data = data.decode(original_encoding, errors='replace') binary = False if cli_args.contents_in_report: meta['decoded_contents'] = data # Writing the file? # TODO: specify what should happen when contents are empty (e.g. POST queries) if data and not cli_args.contents_in_report: files_writer.write( filename, data, binary=binary, compress=cli_args.compress ) def write_fetch_output(index, row, resolved=None, status=None, error=None, filename=None, encoding=None, mimetype=None, data=None): addendum = [ resolved or '', status or '', error or '', filename or '', mimetype or '', encoding or '' ] if cli_args.contents_in_report: addendum.append(data or '') enricher.writerow(index, row, addendum) def write_resolve_output(index, row, resolved=None, status=None, error=None, redirects=None, chain=None): addendum = [ resolved or '', status or '', error or '', redirects or '', chain or '' ] enricher.writerow(index, row, addendum) errors = 0 status_codes = Counter() common_kwargs = { 'key': url_key, 'insecure': cli_args.insecure, 'threads': cli_args.threads, 'throttle': cli_args.throttle, 'domain_parallelism': cli_args.domain_parallelism, 'max_redirects': cli_args.max_redirects, 'wait': False, 'daemonic': True } if cli_args.timeout is not None: common_kwargs['timeout'] = cli_args.timeout # Normal fetch if not resolve: multithreaded_iterator = multithreaded_fetch( enricher, request_args=request_args, callback=worker_callback, **common_kwargs ) for result in multithreaded_iterator: index, row = result.item if not result.url: write_fetch_output( index, row ) loading_bar.update() continue # Updating stats update_loading_bar(result) # No error if result.error is None: meta = result.meta # Final url target resolved_url = result.resolved if resolved_url == result.url: resolved_url = None # Reporting in output write_fetch_output( index, row, resolved=resolved_url, status=result.response.status, filename=meta.get('filename'), encoding=meta.get('encoding'), mimetype=meta.get('mimetype'), data=meta.get('decoded_contents') ) # Handling potential errors else: error_code = report_error(result.error) resolved = None if isinstance(result.error, InvalidURLError): resolved = result.error.url if isinstance(result.error, FilenameFormattingError): loading_bar.print(report_filename_formatting_error(result.error)) write_fetch_output( index, row, error=error_code, resolved=resolved ) # Resolve else: multithreaded_iterator = multithreaded_resolve( enricher, resolve_args=request_args, follow_meta_refresh=cli_args.follow_meta_refresh, follow_js_relocation=cli_args.follow_js_relocation, infer_redirection=cli_args.infer_redirection, **common_kwargs ) for result in multithreaded_iterator: index, row = result.item if not result.url: write_resolve_output( index, row ) loading_bar.update() continue # Updating stats update_loading_bar(result) # No error if result.error is None: # Reporting in output last = result.stack[-1] write_resolve_output( index, row, resolved=last.url, status=last.status, redirects=len(result.stack) - 1, chain='|'.join(step.type for step in result.stack) ) # Handling potential errors else: error_code = report_error(result.error) write_resolve_output( index, row, error=error_code, redirects=(len(result.stack) - 1) if result.stack else None, chain='|'.join(step.type for step in result.stack) if result.stack else None )
def scrape_action(cli_args): # Parsing scraper definition try: scraper = Scraper(cli_args.scraper, strain=cli_args.strain) except DefinitionInvalidFormatError: die(['Unknown scraper format!', 'It should be a JSON or YAML file.']) except FileNotFoundError: die('Could not find scraper file!') except InvalidScraperError as error: print('Your scraper is invalid! You need to fix the following errors:', file=sys.stderr) print(file=sys.stderr) sys.stderr.write( report_scraper_validation_errors(error.validation_errors)) die() except CSSSelectorTooComplex: die([ 'Your strainer\'s CSS selector %s is too complex.' % colored(cli_args.strain, 'blue'), 'You cannot use relations to create a strainer.', 'Try to simplify the selector you passed to --strain.' ]) if cli_args.validate: print('Your scraper is valid.', file=sys.stderr) sys.exit(0) if scraper.headers is None and cli_args.format == 'csv': die([ 'Your scraper does not yield tabular data.', 'Try changing it or setting --format to "jsonl".' ]) loading_bar = LoadingBar(desc='Scraping pages', total=cli_args.total, unit='page') worker_args = (cli_args.format, cli_args.separator) def on_irrelevant_row(reason, row): loading_bar.update() if cli_args.glob is not None: files = create_glob_iterator(cli_args, worker_args) else: reader = casanova.reader(cli_args.report) try: files = create_report_iterator(cli_args, reader, worker_args=worker_args, on_irrelevant_row=on_irrelevant_row) except NotADirectoryError: loading_bar.die([ 'Could not find the "%s" directory!' % cli_args.input_dir, 'Did you forget to specify it with -i/--input-dir?' ]) if cli_args.format == 'csv': output_writer = csv.DictWriter(cli_args.output, fieldnames=scraper.headers) output_writer.writeheader() else: output_writer = ndjson.writer(cli_args.output) pool = LazyPool(cli_args.processes, initializer=init_process, initargs=(scraper.definition, cli_args.strain)) loading_bar.update_stats(p=pool.processes) with pool: for error, items in pool.imap_unordered(worker, files): loading_bar.update() if error is not None: if isinstance(error, (ScraperEvalError, ScraperEvalTypeError, ScraperEvalNoneError)): loading_bar.print(report_scraper_evaluation_error(error), end='') loading_bar.inc('errors') continue for item in items: output_writer.writerow(item)
def twitter_user_tweets_action(namespace, output_file): wrapper = TwitterWrapper(namespace.access_token, namespace.access_token_secret, namespace.api_key, namespace.api_secret_key) enricher = casanova.enricher(namespace.file, output_file, keep=namespace.select, add=TWEET_FIELDS) loading_bar = LoadingBar('Retrieving tweets', total=namespace.total, unit='tweet') for row, user in enricher.cells(namespace.column, with_rows=True): max_id = None loading_bar.update_stats(user=user) while True: if namespace.ids: kwargs = {'user_id': user} else: kwargs = {'screen_name': user} kwargs['include_rts'] = not namespace.exclude_retweets kwargs['count'] = TWITTER_API_MAX_STATUSES_COUNT kwargs['tweet_mode'] = 'extended' if max_id is not None: kwargs['max_id'] = max_id loading_bar.inc('calls') try: tweets = wrapper.call(['statuses', 'user_timeline'], **kwargs) except TwitterHTTPError as e: loading_bar.inc('errors') if e.e.code == 404: loading_bar.print('Could not find user "%s"' % user) else: loading_bar.print( 'An error happened when attempting to retrieve tweets from "%s"' % user) break if not tweets: break loading_bar.update(len(tweets)) max_id = min(int(tweet['id_str']) for tweet in tweets) - 1 for tweet in tweets: tweet = normalize_tweet(tweet, collection_source='api') addendum = format_tweet_as_csv_row(tweet) enricher.writerow(row, addendum) loading_bar.inc('done') loading_bar.close()