for f in user.friend_ids:
sns_list.add(f)
return sns_list
if sys.argv[4] == 'mentions':
snowball_func = get_mentions_2012
else:
snowball_func = get_friends
processes = []
for h in handles:
p = UserDataWorker(queue=request_queue,
api_hook=h,
out_dir=OUTPUT_DIRECTORY,
step_count=step_count,
populate_friends=True,
gets_user_id=True,
populate_followers=False,
add_users_to_queue_function= snowball_func)
p.start()
processes.append(p)
try:
for p in processes:
p.join()
except KeyboardInterrupt:
print 'keyboard interrupt'
##put data on the queue
request_queue = multiprocess_setup.load_request_queue(user_ids, len(handles))
tweet_count_file_dir = None
if gen_tweet_counts_file == 'y':
tweet_count_file_dir = "tweet_count" + str(datetime.now()).split(" ")[0]
mkdir_no_err(os.path.join(out_dir, tweet_count_file_dir))
processes = []
for i in range(len(handles)):
p = UserDataWorker(request_queue,
handles[i],
out_dir,
always_pickle=True,
gets_user_id=is_ids,
populate_lists=False,
populate_friends=collect_friends == 'y',
add_to_file=True,
populate_followers=collect_followers == 'y',
tweet_count_file_dir=tweet_count_file_dir)
p.start()
processes.append(p)
try:
for p in processes:
print p.join(100000000)
except KeyboardInterrupt:
print 'keyboard interrupt'
if gen_tweet_counts_file:
for p in processes:
sns_list = set()
for f in user.friend_ids:
sns_list.add(f)
return sns_list
if sys.argv[4] == 'mentions':
snowball_func = get_mentions_2012
else:
snowball_func = get_friends
processes = []
for h in handles:
p = UserDataWorker(queue=request_queue,
api_hook=h,
out_dir=OUTPUT_DIRECTORY,
step_count=step_count,
populate_friends=True,
gets_user_id=True,
populate_followers=False,
add_users_to_queue_function=snowball_func)
p.start()
processes.append(p)
try:
for p in processes:
p.join()
except KeyboardInterrupt:
print 'keyboard interrupt'
general_utils.mkdir_no_err(os.path.join(out_dir, "json"))
multiprocess_setup.init_good_sync_manager()
# already_done = set([os.path.basename(f) for f in glob.glob(out_dir+"/*")])
print "len already done:", 0
# user_screennames = [u for u in user_screennames if u not in already_done]
##put data on the queue
request_queue = multiprocess_setup.load_request_queue(user_ids, len(handles))
processes = []
for i in range(len(handles)):
p = UserDataWorker(
request_queue,
handles[i],
out_dir,
always_pickle=True,
gets_user_id=False,
populate_lists=False,
populate_friends=True,
populate_followers=True,
)
p.start()
processes.append(p)
try:
for p in processes:
print p.join(100000000)
except KeyboardInterrupt:
print "keyboard interrupt"
general_utils.mkdir_no_err(os.path.join(out_dir, "obj"))
general_utils.mkdir_no_err(os.path.join(out_dir, "json"))
multiprocess_setup.init_good_sync_manager()
#already_done = set([os.path.basename(f) for f in glob.glob(out_dir+"/*")])
print 'len already done:', 0
#user_screennames = [u for u in user_screennames if u not in already_done]
##put data on the queue
request_queue = multiprocess_setup.load_request_queue(
user_ids, len(handles))
processes = []
for i in range(len(handles)):
p = UserDataWorker(request_queue,
handles[i],
out_dir,
always_pickle=True,
gets_user_id=False,
populate_lists=False,
populate_friends=True,
populate_followers=True)
p.start()
processes.append(p)
try:
for p in processes:
print p.join(100000000)
except KeyboardInterrupt:
print 'keyboard interrupt'
