for f in user.friend_ids: sns_list.add(f) return sns_list if sys.argv[4] == 'mentions': snowball_func = get_mentions_2012 else: snowball_func = get_friends processes = [] for h in handles: p = UserDataWorker(queue=request_queue, api_hook=h, out_dir=OUTPUT_DIRECTORY, step_count=step_count, populate_friends=True, gets_user_id=True, populate_followers=False, add_users_to_queue_function= snowball_func) p.start() processes.append(p) try: for p in processes: p.join() except KeyboardInterrupt: print 'keyboard interrupt'
##put data on the queue request_queue = multiprocess_setup.load_request_queue(user_ids, len(handles)) tweet_count_file_dir = None if gen_tweet_counts_file == 'y': tweet_count_file_dir = "tweet_count" + str(datetime.now()).split(" ")[0] mkdir_no_err(os.path.join(out_dir, tweet_count_file_dir)) processes = [] for i in range(len(handles)): p = UserDataWorker(request_queue, handles[i], out_dir, always_pickle=True, gets_user_id=is_ids, populate_lists=False, populate_friends=collect_friends == 'y', add_to_file=True, populate_followers=collect_followers == 'y', tweet_count_file_dir=tweet_count_file_dir) p.start() processes.append(p) try: for p in processes: print p.join(100000000) except KeyboardInterrupt: print 'keyboard interrupt' if gen_tweet_counts_file: for p in processes:
sns_list = set() for f in user.friend_ids: sns_list.add(f) return sns_list if sys.argv[4] == 'mentions': snowball_func = get_mentions_2012 else: snowball_func = get_friends processes = [] for h in handles: p = UserDataWorker(queue=request_queue, api_hook=h, out_dir=OUTPUT_DIRECTORY, step_count=step_count, populate_friends=True, gets_user_id=True, populate_followers=False, add_users_to_queue_function=snowball_func) p.start() processes.append(p) try: for p in processes: p.join() except KeyboardInterrupt: print 'keyboard interrupt'
general_utils.mkdir_no_err(os.path.join(out_dir, "json")) multiprocess_setup.init_good_sync_manager() # already_done = set([os.path.basename(f) for f in glob.glob(out_dir+"/*")]) print "len already done:", 0 # user_screennames = [u for u in user_screennames if u not in already_done] ##put data on the queue request_queue = multiprocess_setup.load_request_queue(user_ids, len(handles)) processes = [] for i in range(len(handles)): p = UserDataWorker( request_queue, handles[i], out_dir, always_pickle=True, gets_user_id=False, populate_lists=False, populate_friends=True, populate_followers=True, ) p.start() processes.append(p) try: for p in processes: print p.join(100000000) except KeyboardInterrupt: print "keyboard interrupt"
general_utils.mkdir_no_err(os.path.join(out_dir, "obj")) general_utils.mkdir_no_err(os.path.join(out_dir, "json")) multiprocess_setup.init_good_sync_manager() #already_done = set([os.path.basename(f) for f in glob.glob(out_dir+"/*")]) print 'len already done:', 0 #user_screennames = [u for u in user_screennames if u not in already_done] ##put data on the queue request_queue = multiprocess_setup.load_request_queue( user_ids, len(handles)) processes = [] for i in range(len(handles)): p = UserDataWorker(request_queue, handles[i], out_dir, always_pickle=True, gets_user_id=False, populate_lists=False, populate_friends=True, populate_followers=True) p.start() processes.append(p) try: for p in processes: print p.join(100000000) except KeyboardInterrupt: print 'keyboard interrupt'