Esempio n. 1
0
        format='[%(asctime)s] - %(name)s - %(levelname)s - %(message)s')

    logging.info(
        'Starting collecting Twitter data with the following parameters:\n{}'.
        format(pprint.pformat(vars(args))))

    logging.info('Reading tweet ids file name ...')
    with open(args.tweet_ids_filename, encoding='ascii') as fd:
        tweet_ids = [line.strip() for line in fd]

    logging.info('Connecting to Twitter ...')
    app_name = '<your application name>'
    consumer_key = '<your application consumer key>'
    consumer_secret = '<your application consumer secret>'
    twitter_conn = twitter.TwitterReader(app_name,
                                         consumer_key,
                                         consumer_secret,
                                         debug_connection=(args.debug == 2))
    twitter_conn.connect()

    logging.info('Retrieving tweets ...')
    retry = True
    while retry:
        try:
            tweets = twitter_conn.hydrate_tweets(tweet_ids, extended=True)
        except Exception as e:
            logging.error(
                'Error trying to retrieve tweets. Error: {}'.format(e))
            traceback.print_exc()
            if args.stop_on_error:
                logging.error('Exiting on error ...')
                twitter_conn.cleanup()
                exprs.append(expr)
                expr_ids[expr] = len(expr_ids)
    with open(os.sep.join([args.destination_dir, 'expression_ids.txt']),
              mode='xt',
              encoding='utf-8') as fd:
        for expr in sorted(expr_ids.keys()):
            fd.write('{} - \'{}\'\n'.format(expr, expr_ids[expr]))
    with open(os.sep.join([args.destination_dir, 'expression_ids.json']),
              mode='xt',
              encoding='utf-8') as fd:
        json.dump(expr_ids, fd, sort_keys=True)

    logging.info('Connecting to Twitter ...')
    twitter_conn = twitter.TwitterReader(
        credentials['app_name'],
        credentials['consumer_key'],
        credentials['consumer_secret'],
        debug_connection=(args.debug == 2),
    )
    twitter_conn.connect()

    logging.info('Retrieving tweets ...')
    for expr in exprs:
        retry = True
        while retry:
            logging.debug(''.join([
                '\tSearching tweets by expression \'', expr, '\' , id = ',
                str(expr_ids[expr]), '...'
            ]))
            filename = os.path.join(args.destination_dir,
                                    str(expr_ids[expr]) + '.json.gz')
            if os.path.exists(filename):
Esempio n. 3
0
#!/usr/bin/env python3

import sys  # if you include the directory where twitter.py is located in your path you don't need this block
sys.path.append('..')

import twitter
twitter_conn = twitter.TwitterReader(
    '<your Twitter application name>',  # App name
    '<application consumer key>',  # consumer key
    '<application secret key>'  # secret key
)
twitter_conn.connect()
users = twitter_conn.search_users('about', max_results=100)
user_id = list(users)[0]
user = twitter_conn.get_user_info(user_id)
tweets = twitter_conn.get_user_timeline(user_id)

import json
print(json.dumps(user, indent=4, sort_keys=True))
print()
print(json.dumps(tweets[0], indent=4, sort_keys=True))
Esempio n. 4
0
    app = flask.Flask(__name__)
    # flask_cors.CORS(app)

    # database_service.MongoSession().configure_instance(
    #           app, kwargs['raw_data_database_name'])
    # auth_service.AuthSession(constants['salt'],
    #        private_key, public_key)
    # get_and_save_tweets(**kwargs)

    add_routes(app)
    return app


#  logging.basicConfig(level=logging.INFO)
app = create_app()
app.logger.info("In app.py")
settings = read_settings()
# Set up communication from TwitterReader to MongoWriter
tweet_queue = queue.Queue()

mongo = mongo_lib.MongoWriter(app, tweet_queue,
                              settings['raw_data_database_name'])

twitter = twitter_lib.TwitterReader(app, tweet_queue, settings,
                                    mongo.latest_tweet)

#if __name__ == "__main__":
#    app.logger.info("Starting server")
#    app.run()
Esempio n. 5
0
            'Destination directory ', args.destination_directory,
            ' already exists. Quitting ...'
        ]))
        sys.exit(1)
    os.makedirs(args.destination_directory)

    logging.info('Reading credentials data ...')
    with open(args.credentials_filename, mode='rt', encoding='ascii') as fd:
        credentials = json.load(fd)

    logging.info('Connecting to Twitter ...')
    twitter_conn = twitter.TwitterReader(
        credentials['app_name'],
        credentials['consumer_key'],
        credentials['consumer_secret'],
        user_auth=True,
        access_token=credentials['access_token'],
        access_token_secret=credentials['access_token_secret'],
        debug_connection=(args.debug == 2),
    )
    twitter_conn.connect()

    logging.info('Reading user IDs ...')
    user_ids = []
    with open(args.user_ids_filename, mode='rt', encoding='ascii') as fd:
        for line in fd:
            user_ids.append(line.strip())

    logging.info('Retrieving Twitter data ...')
    user_location_map = {}
    step_count = 0