format='[%(asctime)s] - %(name)s - %(levelname)s - %(message)s')
logging.info(
'Starting collecting Twitter data with the following parameters:\n{}'.
format(pprint.pformat(vars(args))))
logging.info('Reading tweet ids file name ...')
with open(args.tweet_ids_filename, encoding='ascii') as fd:
tweet_ids = [line.strip() for line in fd]
logging.info('Connecting to Twitter ...')
app_name = '<your application name>'
consumer_key = '<your application consumer key>'
consumer_secret = '<your application consumer secret>'
twitter_conn = twitter.TwitterReader(app_name,
consumer_key,
consumer_secret,
debug_connection=(args.debug == 2))
twitter_conn.connect()
logging.info('Retrieving tweets ...')
retry = True
while retry:
try:
tweets = twitter_conn.hydrate_tweets(tweet_ids, extended=True)
except Exception as e:
logging.error(
'Error trying to retrieve tweets. Error: {}'.format(e))
traceback.print_exc()
if args.stop_on_error:
logging.error('Exiting on error ...')
twitter_conn.cleanup()
exprs.append(expr)
expr_ids[expr] = len(expr_ids)
with open(os.sep.join([args.destination_dir, 'expression_ids.txt']),
mode='xt',
encoding='utf-8') as fd:
for expr in sorted(expr_ids.keys()):
fd.write('{} - \'{}\'\n'.format(expr, expr_ids[expr]))
with open(os.sep.join([args.destination_dir, 'expression_ids.json']),
mode='xt',
encoding='utf-8') as fd:
json.dump(expr_ids, fd, sort_keys=True)
logging.info('Connecting to Twitter ...')
twitter_conn = twitter.TwitterReader(
credentials['app_name'],
credentials['consumer_key'],
credentials['consumer_secret'],
debug_connection=(args.debug == 2),
)
twitter_conn.connect()
logging.info('Retrieving tweets ...')
for expr in exprs:
retry = True
while retry:
logging.debug(''.join([
'\tSearching tweets by expression \'', expr, '\' , id = ',
str(expr_ids[expr]), '...'
]))
filename = os.path.join(args.destination_dir,
str(expr_ids[expr]) + '.json.gz')
if os.path.exists(filename):
#!/usr/bin/env python3
import sys # if you include the directory where twitter.py is located in your path you don't need this block
sys.path.append('..')
import twitter
twitter_conn = twitter.TwitterReader(
'<your Twitter application name>', # App name
'<application consumer key>', # consumer key
'<application secret key>' # secret key
)
twitter_conn.connect()
users = twitter_conn.search_users('about', max_results=100)
user_id = list(users)[0]
user = twitter_conn.get_user_info(user_id)
tweets = twitter_conn.get_user_timeline(user_id)
import json
print(json.dumps(user, indent=4, sort_keys=True))
print()
print(json.dumps(tweets[0], indent=4, sort_keys=True))
app = flask.Flask(__name__)
# flask_cors.CORS(app)
# database_service.MongoSession().configure_instance(
# app, kwargs['raw_data_database_name'])
# auth_service.AuthSession(constants['salt'],
# private_key, public_key)
# get_and_save_tweets(**kwargs)
add_routes(app)
return app
# logging.basicConfig(level=logging.INFO)
app = create_app()
app.logger.info("In app.py")
settings = read_settings()
# Set up communication from TwitterReader to MongoWriter
tweet_queue = queue.Queue()
mongo = mongo_lib.MongoWriter(app, tweet_queue,
settings['raw_data_database_name'])
twitter = twitter_lib.TwitterReader(app, tweet_queue, settings,
mongo.latest_tweet)
#if __name__ == "__main__":
# app.logger.info("Starting server")
# app.run()
'Destination directory ', args.destination_directory,
' already exists. Quitting ...'
]))
sys.exit(1)
os.makedirs(args.destination_directory)
logging.info('Reading credentials data ...')
with open(args.credentials_filename, mode='rt', encoding='ascii') as fd:
credentials = json.load(fd)
logging.info('Connecting to Twitter ...')
twitter_conn = twitter.TwitterReader(
credentials['app_name'],
credentials['consumer_key'],
credentials['consumer_secret'],
user_auth=True,
access_token=credentials['access_token'],
access_token_secret=credentials['access_token_secret'],
debug_connection=(args.debug == 2),
)
twitter_conn.connect()
logging.info('Reading user IDs ...')
user_ids = []
with open(args.user_ids_filename, mode='rt', encoding='ascii') as fd:
for line in fd:
user_ids.append(line.strip())
logging.info('Retrieving Twitter data ...')
user_location_map = {}
step_count = 0