def captions_action(namespace, output_file): # Handling output single_video = namespace.file is sys.stdin and sys.stdin.isatty() if single_video: edit_namespace_with_csv_io(namespace, 'video') enricher = casanova.enricher(namespace.file, output_file, add=YOUTUBE_CAPTIONS_CSV_HEADERS, keep=namespace.select) loading_bar = LoadingBar('Retrieving captions', unit='video') for row, video in enricher.cells(namespace.column, with_rows=True): result = get_video_captions(video, langs=namespace.lang) loading_bar.update() if result is None: continue track, lines = result prefix = [track.lang, '1' if track.generated else ''] for line in lines: enricher.writerow(row, prefix + list(line)) loading_bar.close()
def comments_action(namespace, output_file): # Handling output single_video = namespace.file is sys.stdin and sys.stdin.isatty() if single_video: edit_namespace_with_csv_io(namespace, 'video') enricher = casanova.enricher(namespace.file, output_file, add=YOUTUBE_COMMENT_CSV_HEADERS, keep=namespace.select) loading_bar = LoadingBar('Retrieving comments', unit='comment', stats={'videos': 0}) def before_sleep_until_midnight(seconds): loading_bar.print( 'API limits reached. Will now wait until midnight Pacific time!') client = YouTubeAPIClient( namespace.key, before_sleep_until_midnight=before_sleep_until_midnight) for row, video in enricher.cells(namespace.column, with_rows=True): generator = client.comments(video) for comment in generator: loading_bar.update() enricher.writerow(row, comment.as_csv_row()) loading_bar.inc('videos') loading_bar.close()
def videos_action(namespace, output_file): enricher = casanova.enricher(namespace.file, output_file, add=YOUTUBE_VIDEO_CSV_HEADERS, keep=namespace.select) loading_bar = LoadingBar('Retrieving videos', unit='video', total=namespace.total) def before_sleep_until_midnight(seconds): loading_bar.print( 'API limits reached. Will now wait until midnight Pacific time!') client = YouTubeAPIClient( namespace.key, before_sleep_until_midnight=before_sleep_until_midnight) iterator = enricher.cells(namespace.column, with_rows=True) for (row, _), video in client.videos(iterator, key=itemgetter(1)): loading_bar.update() enricher.writerow(row, video.as_csv_row() if video else None) loading_bar.close()
def url_join_action(cli_args): left_reader = casanova.reader(cli_args.file1) left_headers = left_reader.fieldnames left_idx = None if cli_args.select: left_idx = left_reader.pos.collect(cli_args.select) left_headers = list(cli_args.select) # Applying column prefix now left_headers = [cli_args.match_column_prefix + h for h in left_headers] right_enricher = casanova.enricher(cli_args.file2, cli_args.output, add=left_headers) loading_bar = LoadingBar(desc='Indexing left file', unit='line') # First step is to index left file trie = NormalizedLRUTrie() for row, cell in left_reader.cells(cli_args.column1, with_rows=True): loading_bar.update() if left_idx is not None: row = [row[i] for i in left_idx] urls = [cell] if cli_args.separator is not None: urls = cell.split(cli_args.separator) for url in urls: url = url.strip() # NOTE: should we filter invalid urls here? if url: trie.set(url, row) loading_bar.close() loading_bar = LoadingBar(desc='Matching right file', unit='line') for row, url in right_enricher.cells(cli_args.column2, with_rows=True): loading_bar.update() url = url.strip() match = None # NOTE: should we filter invalid urls here? if url: match = trie.match(url) if match is None: right_enricher.writerow(row) continue right_enricher.writerow(row, match)
def extract_action(namespace): output_file = open_output_file(namespace.output) enricher = casanova.enricher( namespace.report, output_file, keep=namespace.select, add=OUTPUT_ADDITIONAL_HEADERS ) loading_bar = LoadingBar( desc='Extracting content', total=namespace.total, unit='doc' ) def on_irrelevant_row(reason, row): loading_bar.update() enricher.writerow(row, format_error(reason)) try: files = create_report_iterator( namespace, enricher, on_irrelevant_row=on_irrelevant_row ) except NotADirectoryError: loading_bar.die([ 'Could not find the "%s" directory!' % namespace.input_dir, 'Did you forget to specify it with -i/--input-dir?' ]) pool = LazyPool(namespace.processes) loading_bar.update_stats(p=pool.processes) with pool: for error, row, result in pool.imap_unordered(worker, files): loading_bar.update() if error is not None: enricher.writerow(row, format_error(report_error(error))) continue if result is None: enricher.writerow(row, format_error('no-content')) continue enricher.writerow(row, result) loading_bar.close() output_file.close()
def search_action(namespace, output_file): # Handling output single_query = namespace.file is sys.stdin and sys.stdin.isatty() if single_query: edit_namespace_with_csv_io(namespace, 'query') enricher = casanova.enricher( namespace.file, output_file, add=YOUTUBE_VIDEO_SNIPPET_CSV_HEADERS, keep=namespace.select ) loading_bar = LoadingBar( 'Searching videos', unit='video' ) def before_sleep_until_midnight(seconds): loading_bar.print('API limits reached. Will now wait until midnight Pacific time!') client = YouTubeAPIClient( namespace.key, before_sleep_until_midnight=before_sleep_until_midnight ) for row, query in enricher.cells(namespace.column, with_rows=True): loading_bar.print('Searching for "%s"' % query) searcher = client.search(query, order=namespace.order) if namespace.limit: searcher = islice(searcher, namespace.limit) for video in searcher: loading_bar.update() enricher.writerow(row, video.as_csv_row()) loading_bar.close()
def hyphe_dump_action(cli_args): # Paths output_dir = 'hyphe_corpus_%s' % cli_args.corpus if cli_args.output_dir is not None: output_dir = cli_args.output_dir os.makedirs(output_dir, exist_ok=True) webentities_output_path = join(output_dir, 'webentities.csv') pages_output_path = join(output_dir, 'pages.csv') if cli_args.body: body_output_dir = join(output_dir, 'content') os.makedirs(body_output_dir, exist_ok=True) client = HypheAPIClient(cli_args.url) corpus = client.corpus(cli_args.corpus, password=cli_args.password) try: corpus.ensure_is_started() except HypheCorpusAuthenticationError: die([ 'Wrong password for the "%s" corpus!' % cli_args.corpus, 'Don\'t forget to provide a password for this corpus using --password' ]) # Then we gather some handy statistics counts = corpus.count(statuses=cli_args.statuses) # Then we fetch webentities webentities_file = open(webentities_output_path, 'w', encoding='utf-8') webentities_writer = csv.writer(webentities_file) webentities_writer.writerow(WEBENTITY_CSV_HEADERS) loading_bar = LoadingBar(desc='Paginating web entities', unit='webentity', unit_plural='webentities', total=counts['webentities']) webentities = {} for webentity in corpus.webentities(statuses=cli_args.statuses): loading_bar.update() webentities[webentity['id']] = webentity webentities_writer.writerow(format_webentity_for_csv(webentity)) webentities_file.close() loading_bar.close() # Finally we paginate pages pages_file = open(pages_output_path, 'w', encoding='utf-8') pages_writer = csv.writer(pages_file) pages_writer.writerow(PAGE_CSV_HEADERS + (ADDITIONAL_PAGE_HEADERS if cli_args.body else [])) loading_bar = LoadingBar(desc='Fetching pages', unit='page', total=counts['pages']) for webentity in webentities.values(): for page in corpus.webentity_pages(webentity['id'], include_body=cli_args.body): loading_bar.update() filename = None if cli_args.body and 'body' in page: filename = format_page_filename(webentity, page) filepath = join(body_output_dir, filename) os.makedirs(dirname(filepath), exist_ok=True) with open(filter, 'wb') as f: binary = base64.b64decode(page['body']) binary = zlib.decompress(binary) binary = gzip.compress(binary) f.write(binary) pages_writer.writerow( format_page_for_csv(webentity, page, filename=filename))
def fetch_action(cli_args, resolve=False, defer=None): # If we are hitting a single url we enable contents_in_report by default if not resolve and isinstance(cli_args.file, StringIO) and cli_args.contents_in_report is None: cli_args.contents_in_report = True if not resolve and cli_args.contents_in_report and cli_args.compress: raise InvalidArgumentsError('Cannot both --compress and output --contents-in-report!') # HTTP method http_method = cli_args.method # Cookie grabber get_cookie = None if cli_args.grab_cookies: get_cookie = grab_cookies(cli_args.grab_cookies) # Global headers global_headers = None if cli_args.headers: global_headers = {} for header in cli_args.headers: k, v = parse_http_header(header) global_headers[k] = v # Resume listener skipped_rows = 0 resuming_reader_loading = None if cli_args.resume and cli_args.output.can_resume(): resuming_reader_loading = LoadingBar( desc='Resuming', unit='line' ) def output_read_listener(event, row): nonlocal skipped_rows if event != 'output.row': return skipped_rows += 1 resuming_reader_loading.update() cli_args.output.listener = output_read_listener if resolve: additional_headers = RESOLVE_ADDITIONAL_HEADERS else: additional_headers = FETCH_ADDITIONAL_HEADERS if cli_args.contents_in_report: additional_headers = additional_headers + ['raw_contents'] # Enricher multiplex = None if cli_args.separator is not None: multiplex = (cli_args.column, cli_args.separator) enricher = casanova.threadsafe_enricher( cli_args.file, cli_args.output, add=additional_headers, keep=cli_args.select, total=cli_args.total, prebuffer_bytes=DEFAULT_PREBUFFER_BYTES, multiplex=multiplex ) if resuming_reader_loading is not None: resuming_reader_loading.close() if cli_args.column not in enricher.headers: raise InvalidArgumentsError('Could not find the "%s" column containing the urls in the given CSV file.' % cli_args.column) url_pos = enricher.headers[cli_args.column] filename_pos = None if not resolve and cli_args.filename is not None: if cli_args.filename not in enricher.headers: raise InvalidArgumentsError('Could not find the "%s" column containing the filenames in the given CSV file.' % cli_args.filename) filename_pos = enricher.headers[cli_args.filename] # Loading bar loading_bar = LoadingBar( desc='Fetching pages', total=enricher.total, unit='url', initial=skipped_rows ) defer(loading_bar.close) # NOTE: it could be dangerous with multithreaded execution, not to close it ourselves def update_loading_bar(result): nonlocal errors if result.error is not None: errors += 1 else: if resolve: status = result.stack[-1].status else: status = result.response.status if status >= 400: status_codes[status] += 1 stats = {'errors': errors} for code, count in status_codes.most_common(1): stats[str(code)] = count loading_bar.update_stats(**stats) loading_bar.update() only_shortened = getattr(cli_args, 'only_shortened', False) def url_key(item): url = item[1][url_pos].strip() if not url: return if only_shortened and not is_shortened_url(url): return # Url templating if cli_args.url_template: return cli_args.url_template.format(value=url) return url def request_args(domain, url, item): cookie = None # Cookie if get_cookie: cookie = get_cookie(url) # Headers headers = None if global_headers: headers = global_headers return { 'method': http_method, 'cookie': cookie, 'headers': headers } # Worker callback internals filename_builder = None files_writer = None if not resolve: try: filename_builder = FilenameBuilder( folder_strategy=cli_args.folder_strategy, template=cli_args.filename_template ) except TypeError: die([ 'Invalid "%s" --folder-strategy!' % cli_args.folder_strategy, 'Check the list at the end of the command help:', ' $ minet fetch -h' ]) files_writer = ThreadSafeFilesWriter(cli_args.output_dir) def worker_callback(result): # NOTE: at this point the callback is only fired on success row = result.item[1] response = result.response meta = result.meta if cli_args.keep_failed_contents and response.status != 200: return # First we need to build a filename filename_cell = row[filename_pos] if filename_pos else None formatter_kwargs = {} if cli_args.filename_template and 'line' in cli_args.filename_template: formatter_kwargs['line'] = enricher.wrap(row) try: filename = filename_builder( result.resolved, filename=filename_cell, ext=meta.get('ext'), formatter_kwargs=formatter_kwargs, compressed=cli_args.compress ) except FilenameFormattingError as e: result.error = e return meta['filename'] = filename # Decoding the response data? is_text = meta.get('is_text', False) original_encoding = meta.get('encoding', 'utf-8') data = response.data binary = True if is_text and (cli_args.standardize_encoding or cli_args.contents_in_report): data = data.decode(original_encoding, errors='replace') binary = False if cli_args.contents_in_report: meta['decoded_contents'] = data # Writing the file? # TODO: specify what should happen when contents are empty (e.g. POST queries) if data and not cli_args.contents_in_report: files_writer.write( filename, data, binary=binary, compress=cli_args.compress ) def write_fetch_output(index, row, resolved=None, status=None, error=None, filename=None, encoding=None, mimetype=None, data=None): addendum = [ resolved or '', status or '', error or '', filename or '', mimetype or '', encoding or '' ] if cli_args.contents_in_report: addendum.append(data or '') enricher.writerow(index, row, addendum) def write_resolve_output(index, row, resolved=None, status=None, error=None, redirects=None, chain=None): addendum = [ resolved or '', status or '', error or '', redirects or '', chain or '' ] enricher.writerow(index, row, addendum) errors = 0 status_codes = Counter() common_kwargs = { 'key': url_key, 'insecure': cli_args.insecure, 'threads': cli_args.threads, 'throttle': cli_args.throttle, 'domain_parallelism': cli_args.domain_parallelism, 'max_redirects': cli_args.max_redirects, 'wait': False, 'daemonic': True } if cli_args.timeout is not None: common_kwargs['timeout'] = cli_args.timeout # Normal fetch if not resolve: multithreaded_iterator = multithreaded_fetch( enricher, request_args=request_args, callback=worker_callback, **common_kwargs ) for result in multithreaded_iterator: index, row = result.item if not result.url: write_fetch_output( index, row ) loading_bar.update() continue # Updating stats update_loading_bar(result) # No error if result.error is None: meta = result.meta # Final url target resolved_url = result.resolved if resolved_url == result.url: resolved_url = None # Reporting in output write_fetch_output( index, row, resolved=resolved_url, status=result.response.status, filename=meta.get('filename'), encoding=meta.get('encoding'), mimetype=meta.get('mimetype'), data=meta.get('decoded_contents') ) # Handling potential errors else: error_code = report_error(result.error) resolved = None if isinstance(result.error, InvalidURLError): resolved = result.error.url if isinstance(result.error, FilenameFormattingError): loading_bar.print(report_filename_formatting_error(result.error)) write_fetch_output( index, row, error=error_code, resolved=resolved ) # Resolve else: multithreaded_iterator = multithreaded_resolve( enricher, resolve_args=request_args, follow_meta_refresh=cli_args.follow_meta_refresh, follow_js_relocation=cli_args.follow_js_relocation, infer_redirection=cli_args.infer_redirection, **common_kwargs ) for result in multithreaded_iterator: index, row = result.item if not result.url: write_resolve_output( index, row ) loading_bar.update() continue # Updating stats update_loading_bar(result) # No error if result.error is None: # Reporting in output last = result.stack[-1] write_resolve_output( index, row, resolved=last.url, status=last.status, redirects=len(result.stack) - 1, chain='|'.join(step.type for step in result.stack) ) # Handling potential errors else: error_code = report_error(result.error) write_resolve_output( index, row, error=error_code, redirects=(len(result.stack) - 1) if result.stack else None, chain='|'.join(step.type for step in result.stack) if result.stack else None )
def scrape_action(namespace): output_file = open_output_file(namespace.output) # Parsing scraper definition try: scraper = Scraper(namespace.scraper, strain=namespace.strain) except DefinitionInvalidFormatError: die(['Unknown scraper format!', 'It should be a JSON or YAML file.']) except FileNotFoundError: die('Could not find scraper file!') except InvalidScraperError as error: print('Your scraper is invalid! You need to fix the following errors:', file=sys.stderr) print(file=sys.stderr) sys.stderr.write( report_scraper_validation_errors(error.validation_errors)) die() except CSSSelectorTooComplex: die([ 'Your strainer\'s CSS selector %s is too complex.' % colored(namespace.strain, 'blue'), 'You cannot use relations to create a strainer.', 'Try to simplify the selector you passed to --strain.' ]) if namespace.validate: print('Your scraper is valid.', file=sys.stderr) sys.exit(0) if scraper.headers is None and namespace.format == 'csv': die([ 'Your scraper does not yield tabular data.', 'Try changing it or setting --format to "jsonl".' ]) loading_bar = LoadingBar(desc='Scraping pages', total=namespace.total, unit='page') proc_args = (namespace.format, namespace.separator) def on_irrelevant_row(reason, row): loading_bar.update() if namespace.glob is not None: files = create_glob_iterator(namespace, proc_args) else: reader = casanova.reader(namespace.report) try: files = create_report_iterator(namespace, reader, args=proc_args, on_irrelevant_row=on_irrelevant_row) except NotADirectoryError: loading_bar.die([ 'Could not find the "%s" directory!' % namespace.input_dir, 'Did you forget to specify it with -i/--input-dir?' ]) if namespace.format == 'csv': output_writer = csv.DictWriter(output_file, fieldnames=scraper.headers) output_writer.writeheader() else: output_writer = ndjson.writer(output_file) pool = LazyPool(namespace.processes, initializer=init_process, initargs=(scraper.definition, namespace.strain)) loading_bar.update_stats(p=pool.processes) with pool: for error, items in pool.imap_unordered(worker, files): loading_bar.update() if error is not None: if isinstance(error, (ScraperEvalError, ScraperEvalTypeError, ScraperEvalNoneError)): loading_bar.print(report_scraper_evaluation_error(error), end='') loading_bar.inc('errors') continue for item in items: output_writer.writerow(item) loading_bar.close() output_file.close()
def twitter_scrape_action(namespace, output_file): single_query = namespace.file is sys.stdin and sys.stdin.isatty() if single_query: edit_namespace_with_csv_io(namespace, 'query', attr_name='query') scraper = TwitterAPIScraper() # Stats loading_bar = LoadingBar('Collecting tweets', total=namespace.limit, unit='tweet', stats={ 'tokens': 1, 'queries': 0 }) enricher = casanova.enricher(namespace.file, output_file, add=TWEET_FIELDS + ADDITIONAL_TWEET_FIELDS, keep=namespace.select) def before_sleep(retry_state): exc = retry_state.outcome.exception() if isinstance(exc, TwitterPublicAPIRateLimitError): loading_bar.inc('tokens') else: loading_bar.inc('failures') loading_bar.print( 'Failed to call Twitter search. Will retry in %s' % prettyprint_seconds(retry_state.idle_for)) for row, query in enricher.cells(namespace.query, with_rows=True): # Templating? if namespace.query_template is not None: query = CUSTOM_FORMATTER.format(namespace.query_template, value=query) loading_bar.print('Searching for "%s"' % query) loading_bar.inc('queries') iterator = scraper.search( query, limit=namespace.limit, before_sleep=before_sleep, include_referenced_tweets=namespace.include_refs, with_meta=True) try: for tweet, meta in iterator: loading_bar.update() tweet_row = format_tweet_as_csv_row(tweet) enricher.writerow(row, tweet_row + format_meta_row(meta)) except TwitterPublicAPIOverCapacityError: loading_bar.die('Got an "Over Capacity" error. Shutting down...') loading_bar.close()
def twitter_user_tweets_action(namespace, output_file): wrapper = TwitterWrapper(namespace.access_token, namespace.access_token_secret, namespace.api_key, namespace.api_secret_key) enricher = casanova.enricher(namespace.file, output_file, keep=namespace.select, add=TWEET_FIELDS) loading_bar = LoadingBar('Retrieving tweets', total=namespace.total, unit='tweet') for row, user in enricher.cells(namespace.column, with_rows=True): max_id = None loading_bar.update_stats(user=user) while True: if namespace.ids: kwargs = {'user_id': user} else: kwargs = {'screen_name': user} kwargs['include_rts'] = not namespace.exclude_retweets kwargs['count'] = TWITTER_API_MAX_STATUSES_COUNT kwargs['tweet_mode'] = 'extended' if max_id is not None: kwargs['max_id'] = max_id loading_bar.inc('calls') try: tweets = wrapper.call(['statuses', 'user_timeline'], **kwargs) except TwitterHTTPError as e: loading_bar.inc('errors') if e.e.code == 404: loading_bar.print('Could not find user "%s"' % user) else: loading_bar.print( 'An error happened when attempting to retrieve tweets from "%s"' % user) break if not tweets: break loading_bar.update(len(tweets)) max_id = min(int(tweet['id_str']) for tweet in tweets) - 1 for tweet in tweets: tweet = normalize_tweet(tweet, collection_source='api') addendum = format_tweet_as_csv_row(tweet) enricher.writerow(row, addendum) loading_bar.inc('done') loading_bar.close()