def test_get_user_id(self):
from tweetf0rm.twitterapi.twitter_api import TwitterAPI
from tweetf0rm.handler.inmemory_handler import InMemoryHandler
apikeys = self.config["apikeys"]["i0mf0rmer03"]
#inmemoryhandler = InMemoryHandler()
twitter_api = TwitterAPI(apikeys=apikeys)
userIds = twitter_api.get_user_ids_by_screen_names(["AmericanCance"])
logger.info(userIds)
def test_get_user_id(self): from tweetf0rm.twitterapi.twitter_api import TwitterAPI from tweetf0rm.handler.inmemory_handler import InMemoryHandler apikeys = self.config["apikeys"]["i0mf0rmer03"] #inmemoryhandler = InMemoryHandler() twitter_api = TwitterAPI(apikeys=apikeys) userIds = twitter_api.get_user_ids_by_screen_names(["AmericanCance"]) logger.info(userIds)
def test_search(self): from tweetf0rm.twitterapi.twitter_api import TwitterAPI from tweetf0rm.handler.inmemory_handler import InMemoryHandler apikeys = self.config["apikeys"]["i0mf0rmer03"] #inmemoryhandler = InMemoryHandler() twitter_api = TwitterAPI(apikeys=apikeys) tweets = twitter_api.search_by_query(query="transmasculine OR transman OR transmale") #tweets = twitter_api.search(q="twitter", geocode=None, lang=None, count=100) logger.info(tweets)
def test_search(self):
from tweetf0rm.twitterapi.twitter_api import TwitterAPI
from tweetf0rm.handler.inmemory_handler import InMemoryHandler
apikeys = self.config["apikeys"]["i0mf0rmer03"]
#inmemoryhandler = InMemoryHandler()
twitter_api = TwitterAPI(apikeys=apikeys)
tweets = twitter_api.search_by_query(
query="transmasculine OR transman OR transmale")
#tweets = twitter_api.search(q="twitter", geocode=None, lang=None, count=100)
logger.info(tweets)
def init_twitter_api(self): # this will throw StopIteration if all proxies have been tried...
if (self.proxies):
try:
self.client_args['proxies'] = next(self.proxies)['proxy_dict'] # this will throw out
#logger.info("client_args: %s"%json.dumps(self.client_args))
except StopIteration as exc:
raise
except Exception as exc:
self.init_twitter_api()
if (self.twitter_api):
del self.twitter_api
#crawler_id=self.crawler_id,
self.twitter_api = TwitterAPI(apikeys=self.apikeys, client_args=self.client_args)
def cmd(config, args):
if (args.command not in avaliable_cmds):
raise Exception("not a valid command...")
nid = args.node_id
logger.info("node_id: %s"%(nid))
node_queue = NodeQueue(nid, redis_config=config['redis_config'])
node_coordinator = NodeCoordinator(config['redis_config'])
# this can be done locally without sending the command to the servers...
if (args.command == 'GET_UIDS_FROM_SCREEN_NAMES'):
apikeys = config["apikeys"].values()[0]
if (not os.path.exists(args.json)):
raise Exception("doesn't exist... ")
with open(os.path.abspath(args.json), 'rb') as f, open(os.path.abspath(args.output), 'wb') as o_f:
screen_names = json.load(f)
twitter_api = TwitterAPI(apikeys=apikeys)
user_ids = twitter_api.get_user_ids_by_screen_names(screen_names)
json.dump(list(user_ids), o_f)
elif (args.command == 'GET_USERS_FROM_IDS'):
apikeys = config["apikeys"].values()[0]
if (not os.path.exists(args.json)):
raise Exception("doesn't exist... ")
with open(os.path.abspath(args.json), 'rb') as f, open(os.path.abspath(args.output), 'wb') as o_f:
user_ids = json.load(f)
twitter_api = TwitterAPI(apikeys=apikeys)
users = twitter_api.get_users(user_ids)
json.dump(list(users), o_f)
elif (args.command.startswith('BATCH_')):
new_command = args.command.replace('BATCH_', '')
args_dict = copy.copy(args.__dict__)
if (not os.path.exists(args.json)):
raise Exception("doesn't exist... ")
with open(os.path.abspath(args.json), 'rb') as f:
if ( args.command == 'BATCH_CRAWL_TWEET' ):
tweet_ids = json.load(f)
for tweet_id in tweet_ids:
print "Loading Tweet ID: ", tweet_id
args_dict['tweet_id'] = tweet_id
cmd = new_cmd(new_command, args_dict)
node_queue.put(cmd)
else:
user_ids = json.load(f)
for user_id in user_ids:
args_dict['user_id'] = user_id
cmd = new_cmd(new_command, args_dict)
node_queue.put(cmd)
elif (args.command == 'LIST_NODES'):
pp.pprint(node_coordinator.list_nodes())
elif (args.command == 'NODE_QSIZES'):
raise NotImplemented("NotImplemented yet...")
#pp.pprint(node_coordinator.list_nodes())
elif (args.command == 'SHUTDOWN_NODE'):
#node_coordinator.remove_node(nid)
#pp.pprint(node_coordinator.list_nodes())
raise NotImplemented("NotImplemented yet...")
elif (args.command == 'CLEAR_NODE_QUEUES'):
node_queue.clear_all_queues()
else:
args_dict = copy.copy(args.__dict__)
cmd = new_cmd(args.command, args_dict)
node_queue.put(cmd)
logger.info('sent [%s]'%(cmd))
def cmd(config, args):
if (args.command not in avaliable_cmds):
raise Exception("not a valid command...")
nid = args.node_id
logger.info("node_id: %s" % (nid))
node_queue = NodeQueue(nid, redis_config=config['redis_config'])
node_coordinator = NodeCoordinator(config['redis_config'])
# this can be done locally without sending the command to the servers...
if (args.command == 'GET_UIDS_FROM_SCREEN_NAMES'):
apikeys = config["apikeys"].values()[0]
if (not os.path.exists(args.json)):
raise Exception("doesn't exist... ")
with open(os.path.abspath(args.json),
'rb') as f, open(os.path.abspath(args.output), 'wb') as o_f:
screen_names = json.load(f)
twitter_api = TwitterAPI(apikeys=apikeys)
user_ids = twitter_api.get_user_ids_by_screen_names(screen_names)
json.dump(list(user_ids), o_f)
elif (args.command == 'GET_USERS_FROM_IDS'):
apikeys = config["apikeys"].values()[0]
if (not os.path.exists(args.json)):
raise Exception("doesn't exist... ")
with open(os.path.abspath(args.json),
'rb') as f, open(os.path.abspath(args.output), 'wb') as o_f:
user_ids = json.load(f)
twitter_api = TwitterAPI(apikeys=apikeys)
users = twitter_api.get_users(user_ids)
json.dump(list(users), o_f)
elif (args.command.startswith('BATCH_')):
new_command = args.command.replace('BATCH_', '')
args_dict = copy.copy(args.__dict__)
if (not os.path.exists(args.json)):
raise Exception("doesn't exist... ")
with open(os.path.abspath(args.json), 'rb') as f:
if (args.command == 'BATCH_CRAWL_TWEET'):
tweet_ids = json.load(f)
for tweet_id in tweet_ids:
print "Loading Tweet ID: ", tweet_id
args_dict['tweet_id'] = tweet_id
cmd = new_cmd(new_command, args_dict)
node_queue.put(cmd)
else:
user_ids = json.load(f)
for user_id in user_ids:
args_dict['user_id'] = user_id
cmd = new_cmd(new_command, args_dict)
node_queue.put(cmd)
elif (args.command == 'LIST_NODES'):
pp.pprint(node_coordinator.list_nodes())
elif (args.command == 'NODE_QSIZES'):
raise NotImplemented("NotImplemented yet...")
#pp.pprint(node_coordinator.list_nodes())
elif (args.command == 'SHUTDOWN_NODE'):
#node_coordinator.remove_node(nid)
#pp.pprint(node_coordinator.list_nodes())
raise NotImplemented("NotImplemented yet...")
elif (args.command == 'CLEAR_NODE_QUEUES'):
node_queue.clear_all_queues()
else:
args_dict = copy.copy(args.__dict__)
cmd = new_cmd(args.command, args_dict)
node_queue.put(cmd)
logger.info('sent [%s]' % (cmd))
def call_user_api(apikeys, client_args): twitter_api = TwitterAPI(apikeys=apikeys, client_args=client_args) twitter_api.find_all_friend_ids(53039176, [Handler()])
required=False,
choices=list_of_choices)
args = parser.parse_args()
with open(os.path.abspath(args.apikeys), 'rb') as apikeys_f:
apikeys_config = json.load(apikeys_f)
# apikeys = apikeys_config.get(args.crawler, None)
apikeys = apikeys_config.get(apikeys_config.keys()[randint(0, len(apikeys_config) - 1)], None)
if not apikeys:
raise Exception("what's the point? Make sure you have all the api keys set in the config file...")
language = args.language
while True:
from tweetf0rm.twitterapi.twitter_api import TwitterAPI
twitter_api = TwitterAPI(apikeys=apikeys)
keywords = []
if language == 'ar':
# Algiers 1253079 ==> town
# Algeria 23424740 ==> country
town_trend = twitter_api.get_place_trends(id="1253079")
country_trend = twitter_api.get_place_trends(id="23424740")
print(json.dumps(town_trend))
print(json.dumps(country_trend))
for trend in town_trend[0]['trends']:
keywords.append(trend['name'])
for trend in country_trend[0]['trends']:
keywords.append(trend['name'])
if language is None:
worldwide_trend = twitter_api.get_place_trends(id="1")