def collect_user_relatinoships_by_user_ids(call, user_ids_config_filepath, output_folder, config):
'''
user_ids_config = {"current_ix": 0, "users": ["2969995619"]}
'''
apikeys = list(config['apikeys'].values()).pop()
user_ids_config = {}
with open(os.path.abspath(user_ids_config_filepath), 'r') as user_ids_config_rf:
user_ids_config = json.load(user_ids_config_rf)
current_ix = user_ids_config['current_ix'] if ('current_ix' in user_ids_config) else 0
user_ids = user_ids_config['users'][current_ix:]
total = len(user_ids)
for user_id in user_ids:
try:
twitterCralwer = TwitterCrawler(apikeys=apikeys, client_args=CLIENT_ARGS, output_folder = output_folder)
twitterCralwer.fetch_user_relationships(call=call, user_id=user_id)
current_ix += 1 # one at a time... no choice
except Exception as exc:
logger.error(exc)
logger.error(util.full_stack())
pass
user_ids_config['current_ix'] = current_ix
flash_cmd_config(user_ids_config, user_ids_config_filepath, output_folder)
logger.info('COMPLETED -> (current_ix: [%d/%d])'%(current_ix, total))
logger.info('PAUSE %ds to CONTINUE...'%WAIT_TIME)
time.sleep(WAIT_TIME)
else:
logger.info('[%s] ALL COMPLETED'%(call))
def collect_user_relatinoships_by_user_ids(call, user_ids_config_filepath, output_folder, config):
'''
user_ids_config = {"current_ix": 0, "users": ["2969995619"]}
'''
apikeys = list(config['apikeys'].values()).pop()
user_ids_config = {}
with open(os.path.abspath(user_ids_config_filepath), 'r') as user_ids_config_rf:
user_ids_config = json.load(user_ids_config_rf)
current_ix = user_ids_config['current_ix'] if ('current_ix' in user_ids_config) else 0
user_ids = user_ids_config['users'][current_ix:]
total = len(user_ids)
for user_id in user_ids:
try:
twitterCralwer = TwitterCrawler(apikeys=apikeys, client_args=CLIENT_ARGS, output_folder = output_folder)
twitterCralwer.fetch_user_relationships(call=call, user_id=user_id)
current_ix += 1 # one at a time... no choice
except Exception as exc:
logger.error(exc)
logger.error(util.full_stack())
pass
user_ids_config['current_ix'] = current_ix
flash_cmd_config(user_ids_config, user_ids_config_filepath, output_folder)
logger.info('COMPLETED -> (current_ix: [%d/%d])'%(current_ix, total))
logger.info('PAUSE %ds to CONTINUE...'%WAIT_TIME)
time.sleep(WAIT_TIME)
else:
logger.info('[%s] ALL COMPLETED'%(call))
def collect_users_all(user_screen_names_ids,deep, output_folder, config):
apikeys = list(config['apikeys'].values()).pop()
if user_screen_names_ids.isdigit():
call = 'user_id'
else:
call = 'screen_name'
try:
users = [user_screen_names_ids.strip('@')]
twitterCralwer = TwitterCrawler(apikeys=apikeys, client_args=CLIENT_ARGS)
ret = twitterCralwer.fetch_users(call=call, users=users, output_folder=output_folder)
if ret == 'ok':
since_id = 1
ret, since_id, remove = twitterCralwer.fetch_user_timeline(user_screen_names_ids, since_id=since_id)
if ret == 'ok':
call = '/friends/ids'
ret = twitterCralwer.fetch_user_relationships(call=call, user_id=user_screen_names_ids.strip('@'),deep=deep)
except Exception as exc:
logger.error(exc)
return 'error'
logger.info('PAUSE %ds to CONTINUE...' % WAIT_TIME)
time.sleep(WAIT_TIME)
return ret
def collect_user_relatinoships_by_user_ids(call, user_id, deep,output_folder, config):
apikeys = list(config['apikeys'].values()).pop()
try:
twitterCralwer = TwitterCrawler(apikeys=apikeys, client_args=CLIENT_ARGS, output_folder=output_folder)
ret = twitterCralwer.fetch_user_relationships(call=call, user_id=user_id.strip('@'),deep=deep)
except Exception as exc:
logger.error(exc)
return 'error'
pass
logger.info('PAUSE %ds to CONTINUE...' % WAIT_TIME_RELATIONS)
time.sleep(WAIT_TIME_RELATIONS)
return ret