def extract_action(cli_args):
if cli_args.glob is None and cli_args.input_dir is None:
cli_args.input_dir = DEFAULT_CONTENT_FOLDER
input_data = cli_args.report
if cli_args.glob is not None:
input_data = dummy_csv_file_from_glob(cli_args.glob, cli_args.input_dir)
enricher = casanova.enricher(
input_data,
cli_args.output,
keep=cli_args.select,
add=OUTPUT_ADDITIONAL_HEADERS
)
loading_bar = LoadingBar(
desc='Extracting content',
total=cli_args.total,
unit='doc'
)
def on_irrelevant_row(reason, row, i):
loading_bar.update()
loading_bar.print('Row n°{n} could not be processed: {reason}'.format(n=i + 1, reason=reason))
enricher.writerow(row, format_error(reason))
if (
cli_args.glob is None and
'raw_contents' not in enricher.headers and
not isdir(cli_args.input_dir)
):
loading_bar.die([
'Could not find the "%s" directory!' % cli_args.input_dir,
'Did you forget to specify it with -i/--input-dir?'
])
files = create_report_iterator(
cli_args,
enricher,
on_irrelevant_row=on_irrelevant_row
)
pool = LazyPool(cli_args.processes)
loading_bar.update_stats(p=pool.processes)
with pool:
for error, row, result in pool.imap_unordered(worker, files):
loading_bar.update()
if error is not None:
enricher.writerow(row, format_error(report_error(error)))
continue
if result is None:
enricher.writerow(row, format_error('no-result'))
continue
enricher.writerow(row, result)
def extract_action(namespace):
output_file = open_output_file(namespace.output)
enricher = casanova.enricher(
namespace.report,
output_file,
keep=namespace.select,
add=OUTPUT_ADDITIONAL_HEADERS
)
loading_bar = LoadingBar(
desc='Extracting content',
total=namespace.total,
unit='doc'
)
def on_irrelevant_row(reason, row):
loading_bar.update()
enricher.writerow(row, format_error(reason))
try:
files = create_report_iterator(
namespace,
enricher,
on_irrelevant_row=on_irrelevant_row
)
except NotADirectoryError:
loading_bar.die([
'Could not find the "%s" directory!' % namespace.input_dir,
'Did you forget to specify it with -i/--input-dir?'
])
pool = LazyPool(namespace.processes)
loading_bar.update_stats(p=pool.processes)
with pool:
for error, row, result in pool.imap_unordered(worker, files):
loading_bar.update()
if error is not None:
enricher.writerow(row, format_error(report_error(error)))
continue
if result is None:
enricher.writerow(row, format_error('no-content'))
continue
enricher.writerow(row, result)
loading_bar.close()
output_file.close()
def facebook_comments_action(cli_args):
try:
scraper = FacebookMobileScraper(cli_args.cookie,
throttle=cli_args.throttle)
except FacebookInvalidCookieError:
if cli_args.cookie in COOKIE_BROWSERS:
die([
'Could not extract relevant cookie from "%s".' %
cli_args.cookie
])
die([
'Relevant cookie not found.',
'A Facebook authentication cookie is necessary to be able to scrape Facebook comments.',
'Use the --cookie flag to choose a browser from which to extract the cookie or give your cookie directly.'
])
# Enricher
enricher = casanova.enricher(cli_args.file,
cli_args.output,
keep=cli_args.select,
add=FACEBOOK_COMMENT_CSV_HEADERS)
# Loading bar
loading_bar = LoadingBar(desc='Scraping comments', unit='comment')
for i, (row,
url) in enumerate(enricher.cells(cli_args.column, with_rows=True),
1):
try:
batches = scraper.comments(url, per_call=True, detailed=True)
except FacebookInvalidTargetError:
loading_bar.print(
'Given url (line %i) is probably not a Facebook resource having comments: %s'
% (i, url))
continue
for details, batch in batches:
for comment in batch:
enricher.writerow(row, comment.as_csv_row())
loading_bar.update(len(batch))
loading_bar.update_stats(calls=details['calls'],
replies=details['replies'],
q=details['queue_size'],
posts=i)
def action(cli_args):
enricher = casanova.batch_enricher(cli_args.file,
cli_args.output,
keep=cli_args.select,
add=csv_headers)
loading_bar = LoadingBar(desc='Retrieving ids',
unit=method_name[:-1],
stats={'users': 0})
# TODO: this is temp debug
def listener(event, data):
loading_bar.print(event)
loading_bar.print(repr(data))
wrapper = TwitterWrapper(cli_args.access_token,
cli_args.access_token_secret,
cli_args.api_key,
cli_args.api_secret_key,
listener=listener)
resuming_state = None
if cli_args.resume:
resuming_state = cli_args.output.pop_state()
for row, user in enricher.cells(cli_args.column, with_rows=True):
loading_bar.update_stats(user=user)
all_ids = []
next_cursor = -1
result = None
if resuming_state is not None and resuming_state.last_cursor:
next_cursor = int(resuming_state.last_cursor)
if cli_args.ids:
wrapper_kwargs = {'user_id': user}
else:
wrapper_kwargs = {'screen_name': user}
while next_cursor != 0:
wrapper_kwargs['cursor'] = next_cursor
skip_in_output = None
if resuming_state:
skip_in_output = resuming_state.values_to_skip
resuming_state = None
try:
result = wrapper.call([method_name, 'ids'],
**wrapper_kwargs)
except TwitterHTTPError as e:
# The user does not exist
loading_bar.inc('users_not_found')
break
if result is not None:
all_ids = result.get('ids', [])
next_cursor = result.get('next_cursor', 0)
loading_bar.update(len(all_ids))
batch = []
for user_id in all_ids:
if skip_in_output and user_id in skip_in_output:
continue
batch.append([user_id])
enricher.writebatch(row, batch, next_cursor or None)
else:
next_cursor = 0
loading_bar.inc('users')
def scrape_action(cli_args):
# Parsing scraper definition
try:
scraper = Scraper(cli_args.scraper, strain=cli_args.strain)
except DefinitionInvalidFormatError:
die(['Unknown scraper format!', 'It should be a JSON or YAML file.'])
except FileNotFoundError:
die('Could not find scraper file!')
except InvalidScraperError as error:
print('Your scraper is invalid! You need to fix the following errors:',
file=sys.stderr)
print(file=sys.stderr)
sys.stderr.write(
report_scraper_validation_errors(error.validation_errors))
die()
except CSSSelectorTooComplex:
die([
'Your strainer\'s CSS selector %s is too complex.' %
colored(cli_args.strain, 'blue'),
'You cannot use relations to create a strainer.',
'Try to simplify the selector you passed to --strain.'
])
if cli_args.validate:
print('Your scraper is valid.', file=sys.stderr)
sys.exit(0)
if scraper.headers is None and cli_args.format == 'csv':
die([
'Your scraper does not yield tabular data.',
'Try changing it or setting --format to "jsonl".'
])
loading_bar = LoadingBar(desc='Scraping pages',
total=cli_args.total,
unit='page')
worker_args = (cli_args.format, cli_args.separator)
def on_irrelevant_row(reason, row):
loading_bar.update()
if cli_args.glob is not None:
files = create_glob_iterator(cli_args, worker_args)
else:
reader = casanova.reader(cli_args.report)
try:
files = create_report_iterator(cli_args,
reader,
worker_args=worker_args,
on_irrelevant_row=on_irrelevant_row)
except NotADirectoryError:
loading_bar.die([
'Could not find the "%s" directory!' % cli_args.input_dir,
'Did you forget to specify it with -i/--input-dir?'
])
if cli_args.format == 'csv':
output_writer = csv.DictWriter(cli_args.output,
fieldnames=scraper.headers)
output_writer.writeheader()
else:
output_writer = ndjson.writer(cli_args.output)
pool = LazyPool(cli_args.processes,
initializer=init_process,
initargs=(scraper.definition, cli_args.strain))
loading_bar.update_stats(p=pool.processes)
with pool:
for error, items in pool.imap_unordered(worker, files):
loading_bar.update()
if error is not None:
if isinstance(error, (ScraperEvalError, ScraperEvalTypeError,
ScraperEvalNoneError)):
loading_bar.print(report_scraper_evaluation_error(error),
end='')
loading_bar.inc('errors')
continue
for item in items:
output_writer.writerow(item)
def twitter_user_tweets_action(namespace, output_file):
wrapper = TwitterWrapper(namespace.access_token,
namespace.access_token_secret, namespace.api_key,
namespace.api_secret_key)
enricher = casanova.enricher(namespace.file,
output_file,
keep=namespace.select,
add=TWEET_FIELDS)
loading_bar = LoadingBar('Retrieving tweets',
total=namespace.total,
unit='tweet')
for row, user in enricher.cells(namespace.column, with_rows=True):
max_id = None
loading_bar.update_stats(user=user)
while True:
if namespace.ids:
kwargs = {'user_id': user}
else:
kwargs = {'screen_name': user}
kwargs['include_rts'] = not namespace.exclude_retweets
kwargs['count'] = TWITTER_API_MAX_STATUSES_COUNT
kwargs['tweet_mode'] = 'extended'
if max_id is not None:
kwargs['max_id'] = max_id
loading_bar.inc('calls')
try:
tweets = wrapper.call(['statuses', 'user_timeline'], **kwargs)
except TwitterHTTPError as e:
loading_bar.inc('errors')
if e.e.code == 404:
loading_bar.print('Could not find user "%s"' % user)
else:
loading_bar.print(
'An error happened when attempting to retrieve tweets from "%s"'
% user)
break
if not tweets:
break
loading_bar.update(len(tweets))
max_id = min(int(tweet['id_str']) for tweet in tweets) - 1
for tweet in tweets:
tweet = normalize_tweet(tweet, collection_source='api')
addendum = format_tweet_as_csv_row(tweet)
enricher.writerow(row, addendum)
loading_bar.inc('done')
loading_bar.close()
def action(cli_args):
resume = getattr(cli_args, 'resume', False)
# Validation
if resume:
if cli_args.sort_by != 'date':
die('Cannot --resume if --sort_by is not `date`.')
if cli_args.format != 'csv':
die('Cannot --resume jsonl format yet.')
if cli_args.format == 'csv':
fieldnames = csv_headers(cli_args) if callable(
csv_headers) else csv_headers
writer = casanova.writer(cli_args.output, fieldnames)
else:
writer = ndjson.writer(cli_args.output)
# Acquiring state from resumer
if getattr(cli_args, 'resume', False):
last_date = cli_args.output.pop_state()
if last_date is not None:
cli_args.end_date = last_date.replace(' ', 'T')
print_err('Resuming from: %s' % cli_args.end_date)
if callable(announce):
print_err(announce(cli_args))
# Loading bar
loading_bar = LoadingBar(desc='Fetching %s' % item_name,
unit=item_name[:-1],
total=cli_args.limit)
args = []
if callable(get_args):
args = get_args(cli_args)
client = CrowdTangleAPIClient(cli_args.token,
rate_limit=cli_args.rate_limit)
create_iterator = getattr(client, method_name)
iterator = create_iterator(*args,
limit=cli_args.limit,
raw=cli_args.format != 'csv',
per_call=True,
detailed=True,
namespace=cli_args)
try:
for details, items in iterator:
loading_bar.update(len(items))
if details is not None:
loading_bar.update_stats(**details)
for item in items:
if cli_args.format == 'csv':
item = item.as_csv_row()
writer.writerow(item)
except CrowdTangleInvalidTokenError:
loading_bar.die([
'Your API token is invalid.',
'Check that you indicated a valid one using the `--token` argument.'
])
def action(cli_args):
resume = getattr(cli_args, 'resume', False)
# Validation
if resume:
if cli_args.sort_by != 'date':
die('Cannot --resume if --sort_by is not `date`.')
if cli_args.format != 'csv':
die('Cannot --resume jsonl format yet.')
if cli_args.format == 'csv':
fieldnames = csv_headers(cli_args) if callable(
csv_headers) else csv_headers
writer = casanova.writer(cli_args.output, fieldnames)
else:
writer = ndjson.writer(cli_args.output)
# Acquiring state from resumer
if getattr(cli_args, 'resume', False):
last_date = cli_args.output.pop_state()
if last_date is not None:
cli_args.end_date = last_date.replace(' ', 'T')
print_err('Resuming from: %s' % cli_args.end_date)
if callable(announce):
print_err(announce(cli_args))
# Loading bar
loading_bar = LoadingBar(desc='Fetching %s' % item_name,
unit=item_name[:-1],
total=cli_args.limit)
client = CrowdTangleAPIClient(cli_args.token,
rate_limit=cli_args.rate_limit)
args = []
if callable(get_args):
args = get_args(cli_args)
def before_sleep(retry_state):
exc = retry_state.outcome.exception()
if isinstance(exc, CrowdTangleRateLimitExceeded):
reason = 'Call failed because of rate limit!'
elif isinstance(exc, CrowdTangleInvalidJSONError):
reason = 'Call failed because of invalid JSON payload!'
else:
reason = 'Call failed because of server timeout!'
loading_bar.print(
'%s\nWill wait for %s before attempting again.' %
(reason,
prettyprint_seconds(retry_state.idle_for, granularity=2)))
create_iterator = getattr(client, method_name)
iterator = create_iterator(*args,
limit=cli_args.limit,
raw=cli_args.format != 'csv',
per_call=True,
detailed=True,
namespace=cli_args,
before_sleep=before_sleep)
try:
for details, items in iterator:
loading_bar.update(len(items))
if details is not None:
loading_bar.update_stats(**details)
for item in items:
if cli_args.format == 'csv':
item = item.as_csv_row()
writer.writerow(item)
except CrowdTangleInvalidTokenError:
loading_bar.die([
'Your API token is invalid.',
'Check that you indicated a valid one using the `--token` argument.'
])
