def extract_action(cli_args): if cli_args.glob is None and cli_args.input_dir is None: cli_args.input_dir = DEFAULT_CONTENT_FOLDER input_data = cli_args.report if cli_args.glob is not None: input_data = dummy_csv_file_from_glob(cli_args.glob, cli_args.input_dir) enricher = casanova.enricher( input_data, cli_args.output, keep=cli_args.select, add=OUTPUT_ADDITIONAL_HEADERS ) loading_bar = LoadingBar( desc='Extracting content', total=cli_args.total, unit='doc' ) def on_irrelevant_row(reason, row, i): loading_bar.update() loading_bar.print('Row n°{n} could not be processed: {reason}'.format(n=i + 1, reason=reason)) enricher.writerow(row, format_error(reason)) if ( cli_args.glob is None and 'raw_contents' not in enricher.headers and not isdir(cli_args.input_dir) ): loading_bar.die([ 'Could not find the "%s" directory!' % cli_args.input_dir, 'Did you forget to specify it with -i/--input-dir?' ]) files = create_report_iterator( cli_args, enricher, on_irrelevant_row=on_irrelevant_row ) pool = LazyPool(cli_args.processes) loading_bar.update_stats(p=pool.processes) with pool: for error, row, result in pool.imap_unordered(worker, files): loading_bar.update() if error is not None: enricher.writerow(row, format_error(report_error(error))) continue if result is None: enricher.writerow(row, format_error('no-result')) continue enricher.writerow(row, result)
def extract_action(namespace): output_file = open_output_file(namespace.output) enricher = casanova.enricher( namespace.report, output_file, keep=namespace.select, add=OUTPUT_ADDITIONAL_HEADERS ) loading_bar = LoadingBar( desc='Extracting content', total=namespace.total, unit='doc' ) def on_irrelevant_row(reason, row): loading_bar.update() enricher.writerow(row, format_error(reason)) try: files = create_report_iterator( namespace, enricher, on_irrelevant_row=on_irrelevant_row ) except NotADirectoryError: loading_bar.die([ 'Could not find the "%s" directory!' % namespace.input_dir, 'Did you forget to specify it with -i/--input-dir?' ]) pool = LazyPool(namespace.processes) loading_bar.update_stats(p=pool.processes) with pool: for error, row, result in pool.imap_unordered(worker, files): loading_bar.update() if error is not None: enricher.writerow(row, format_error(report_error(error))) continue if result is None: enricher.writerow(row, format_error('no-content')) continue enricher.writerow(row, result) loading_bar.close() output_file.close()
def facebook_comments_action(cli_args): try: scraper = FacebookMobileScraper(cli_args.cookie, throttle=cli_args.throttle) except FacebookInvalidCookieError: if cli_args.cookie in COOKIE_BROWSERS: die([ 'Could not extract relevant cookie from "%s".' % cli_args.cookie ]) die([ 'Relevant cookie not found.', 'A Facebook authentication cookie is necessary to be able to scrape Facebook comments.', 'Use the --cookie flag to choose a browser from which to extract the cookie or give your cookie directly.' ]) # Enricher enricher = casanova.enricher(cli_args.file, cli_args.output, keep=cli_args.select, add=FACEBOOK_COMMENT_CSV_HEADERS) # Loading bar loading_bar = LoadingBar(desc='Scraping comments', unit='comment') for i, (row, url) in enumerate(enricher.cells(cli_args.column, with_rows=True), 1): try: batches = scraper.comments(url, per_call=True, detailed=True) except FacebookInvalidTargetError: loading_bar.print( 'Given url (line %i) is probably not a Facebook resource having comments: %s' % (i, url)) continue for details, batch in batches: for comment in batch: enricher.writerow(row, comment.as_csv_row()) loading_bar.update(len(batch)) loading_bar.update_stats(calls=details['calls'], replies=details['replies'], q=details['queue_size'], posts=i)
def action(cli_args): enricher = casanova.batch_enricher(cli_args.file, cli_args.output, keep=cli_args.select, add=csv_headers) loading_bar = LoadingBar(desc='Retrieving ids', unit=method_name[:-1], stats={'users': 0}) # TODO: this is temp debug def listener(event, data): loading_bar.print(event) loading_bar.print(repr(data)) wrapper = TwitterWrapper(cli_args.access_token, cli_args.access_token_secret, cli_args.api_key, cli_args.api_secret_key, listener=listener) resuming_state = None if cli_args.resume: resuming_state = cli_args.output.pop_state() for row, user in enricher.cells(cli_args.column, with_rows=True): loading_bar.update_stats(user=user) all_ids = [] next_cursor = -1 result = None if resuming_state is not None and resuming_state.last_cursor: next_cursor = int(resuming_state.last_cursor) if cli_args.ids: wrapper_kwargs = {'user_id': user} else: wrapper_kwargs = {'screen_name': user} while next_cursor != 0: wrapper_kwargs['cursor'] = next_cursor skip_in_output = None if resuming_state: skip_in_output = resuming_state.values_to_skip resuming_state = None try: result = wrapper.call([method_name, 'ids'], **wrapper_kwargs) except TwitterHTTPError as e: # The user does not exist loading_bar.inc('users_not_found') break if result is not None: all_ids = result.get('ids', []) next_cursor = result.get('next_cursor', 0) loading_bar.update(len(all_ids)) batch = [] for user_id in all_ids: if skip_in_output and user_id in skip_in_output: continue batch.append([user_id]) enricher.writebatch(row, batch, next_cursor or None) else: next_cursor = 0 loading_bar.inc('users')
def scrape_action(cli_args): # Parsing scraper definition try: scraper = Scraper(cli_args.scraper, strain=cli_args.strain) except DefinitionInvalidFormatError: die(['Unknown scraper format!', 'It should be a JSON or YAML file.']) except FileNotFoundError: die('Could not find scraper file!') except InvalidScraperError as error: print('Your scraper is invalid! You need to fix the following errors:', file=sys.stderr) print(file=sys.stderr) sys.stderr.write( report_scraper_validation_errors(error.validation_errors)) die() except CSSSelectorTooComplex: die([ 'Your strainer\'s CSS selector %s is too complex.' % colored(cli_args.strain, 'blue'), 'You cannot use relations to create a strainer.', 'Try to simplify the selector you passed to --strain.' ]) if cli_args.validate: print('Your scraper is valid.', file=sys.stderr) sys.exit(0) if scraper.headers is None and cli_args.format == 'csv': die([ 'Your scraper does not yield tabular data.', 'Try changing it or setting --format to "jsonl".' ]) loading_bar = LoadingBar(desc='Scraping pages', total=cli_args.total, unit='page') worker_args = (cli_args.format, cli_args.separator) def on_irrelevant_row(reason, row): loading_bar.update() if cli_args.glob is not None: files = create_glob_iterator(cli_args, worker_args) else: reader = casanova.reader(cli_args.report) try: files = create_report_iterator(cli_args, reader, worker_args=worker_args, on_irrelevant_row=on_irrelevant_row) except NotADirectoryError: loading_bar.die([ 'Could not find the "%s" directory!' % cli_args.input_dir, 'Did you forget to specify it with -i/--input-dir?' ]) if cli_args.format == 'csv': output_writer = csv.DictWriter(cli_args.output, fieldnames=scraper.headers) output_writer.writeheader() else: output_writer = ndjson.writer(cli_args.output) pool = LazyPool(cli_args.processes, initializer=init_process, initargs=(scraper.definition, cli_args.strain)) loading_bar.update_stats(p=pool.processes) with pool: for error, items in pool.imap_unordered(worker, files): loading_bar.update() if error is not None: if isinstance(error, (ScraperEvalError, ScraperEvalTypeError, ScraperEvalNoneError)): loading_bar.print(report_scraper_evaluation_error(error), end='') loading_bar.inc('errors') continue for item in items: output_writer.writerow(item)
def twitter_user_tweets_action(namespace, output_file): wrapper = TwitterWrapper(namespace.access_token, namespace.access_token_secret, namespace.api_key, namespace.api_secret_key) enricher = casanova.enricher(namespace.file, output_file, keep=namespace.select, add=TWEET_FIELDS) loading_bar = LoadingBar('Retrieving tweets', total=namespace.total, unit='tweet') for row, user in enricher.cells(namespace.column, with_rows=True): max_id = None loading_bar.update_stats(user=user) while True: if namespace.ids: kwargs = {'user_id': user} else: kwargs = {'screen_name': user} kwargs['include_rts'] = not namespace.exclude_retweets kwargs['count'] = TWITTER_API_MAX_STATUSES_COUNT kwargs['tweet_mode'] = 'extended' if max_id is not None: kwargs['max_id'] = max_id loading_bar.inc('calls') try: tweets = wrapper.call(['statuses', 'user_timeline'], **kwargs) except TwitterHTTPError as e: loading_bar.inc('errors') if e.e.code == 404: loading_bar.print('Could not find user "%s"' % user) else: loading_bar.print( 'An error happened when attempting to retrieve tweets from "%s"' % user) break if not tweets: break loading_bar.update(len(tweets)) max_id = min(int(tweet['id_str']) for tweet in tweets) - 1 for tweet in tweets: tweet = normalize_tweet(tweet, collection_source='api') addendum = format_tweet_as_csv_row(tweet) enricher.writerow(row, addendum) loading_bar.inc('done') loading_bar.close()
def action(cli_args): resume = getattr(cli_args, 'resume', False) # Validation if resume: if cli_args.sort_by != 'date': die('Cannot --resume if --sort_by is not `date`.') if cli_args.format != 'csv': die('Cannot --resume jsonl format yet.') if cli_args.format == 'csv': fieldnames = csv_headers(cli_args) if callable( csv_headers) else csv_headers writer = casanova.writer(cli_args.output, fieldnames) else: writer = ndjson.writer(cli_args.output) # Acquiring state from resumer if getattr(cli_args, 'resume', False): last_date = cli_args.output.pop_state() if last_date is not None: cli_args.end_date = last_date.replace(' ', 'T') print_err('Resuming from: %s' % cli_args.end_date) if callable(announce): print_err(announce(cli_args)) # Loading bar loading_bar = LoadingBar(desc='Fetching %s' % item_name, unit=item_name[:-1], total=cli_args.limit) args = [] if callable(get_args): args = get_args(cli_args) client = CrowdTangleAPIClient(cli_args.token, rate_limit=cli_args.rate_limit) create_iterator = getattr(client, method_name) iterator = create_iterator(*args, limit=cli_args.limit, raw=cli_args.format != 'csv', per_call=True, detailed=True, namespace=cli_args) try: for details, items in iterator: loading_bar.update(len(items)) if details is not None: loading_bar.update_stats(**details) for item in items: if cli_args.format == 'csv': item = item.as_csv_row() writer.writerow(item) except CrowdTangleInvalidTokenError: loading_bar.die([ 'Your API token is invalid.', 'Check that you indicated a valid one using the `--token` argument.' ])
def action(cli_args): resume = getattr(cli_args, 'resume', False) # Validation if resume: if cli_args.sort_by != 'date': die('Cannot --resume if --sort_by is not `date`.') if cli_args.format != 'csv': die('Cannot --resume jsonl format yet.') if cli_args.format == 'csv': fieldnames = csv_headers(cli_args) if callable( csv_headers) else csv_headers writer = casanova.writer(cli_args.output, fieldnames) else: writer = ndjson.writer(cli_args.output) # Acquiring state from resumer if getattr(cli_args, 'resume', False): last_date = cli_args.output.pop_state() if last_date is not None: cli_args.end_date = last_date.replace(' ', 'T') print_err('Resuming from: %s' % cli_args.end_date) if callable(announce): print_err(announce(cli_args)) # Loading bar loading_bar = LoadingBar(desc='Fetching %s' % item_name, unit=item_name[:-1], total=cli_args.limit) client = CrowdTangleAPIClient(cli_args.token, rate_limit=cli_args.rate_limit) args = [] if callable(get_args): args = get_args(cli_args) def before_sleep(retry_state): exc = retry_state.outcome.exception() if isinstance(exc, CrowdTangleRateLimitExceeded): reason = 'Call failed because of rate limit!' elif isinstance(exc, CrowdTangleInvalidJSONError): reason = 'Call failed because of invalid JSON payload!' else: reason = 'Call failed because of server timeout!' loading_bar.print( '%s\nWill wait for %s before attempting again.' % (reason, prettyprint_seconds(retry_state.idle_for, granularity=2))) create_iterator = getattr(client, method_name) iterator = create_iterator(*args, limit=cli_args.limit, raw=cli_args.format != 'csv', per_call=True, detailed=True, namespace=cli_args, before_sleep=before_sleep) try: for details, items in iterator: loading_bar.update(len(items)) if details is not None: loading_bar.update_stats(**details) for item in items: if cli_args.format == 'csv': item = item.as_csv_row() writer.writerow(item) except CrowdTangleInvalidTokenError: loading_bar.die([ 'Your API token is invalid.', 'Check that you indicated a valid one using the `--token` argument.' ])