def dump_messages(**options):
logger.info("Updating Messages. Currently have {} messages in DB.".format(
messages.count()))
params = {}
if options.get('after'):
params.update({'after': options.get('after')})
elif options.get('resume'):
if meta.find_one({'endpoint': 'messages'}) is not None:
params.update({
'after':
meta.find_one({
'endpoint': 'messages'
}).get('updated_on')
})
# call messages API
messages_list = get_api_data('/messages.json', **params)
update_collection(collection=messages, data=messages_list, id_field='id')
# loop through potential next pages
while messages_list.get('next'):
messages_list = get_api_data(messages_list.get('next'))
update_collection(collection=messages,
data=messages_list,
id_field='id')
logger.info(
"Updated Messages completed. Now have {} messages in DB.".format(
messages.count()))
def dump_runs(**options):
logger.info("Updating Runs. Currently have {} runs in DB.".format(
runs.count()))
params = {}
if options.get('after'):
params.update({'after': options.get('after')})
elif options.get('resume'):
if meta.find_one({'endpoint': 'runs'}) is not None:
params.update({
'after':
meta.find_one({
'endpoint': 'runs'
}).get('updated_on')
})
# call runs API
runs_list = get_api_data('/runs.json', **params)
update_collection(collection=runs, data=runs_list, id_field='run')
# loop through potential next pages
while runs_list.get('next'):
runs_list = get_api_data(runs_list.get('next'))
update_collection(collection=runs, data=runs_list, id_field='run')
logger.info("Updated Runs completed. Now have {} runs in DB.".format(
runs.count()))
def dump_fields(**options):
logger.info("Updating Fields. Currently have {} fields in DB.".format(
fields.count()))
# call fields API
fields_list = get_api_data('/fields.json')
update_collection(collection=fields, data=fields_list, id_field='key')
# loop through potential next pages
while fields_list.get('next'):
fields_list = get_api_data(fields_list.get('next'))
update_collection(collection=fields, data=fields_list, id_field='key')
logger.info("Updated Fields completed. Now have {} fields in DB.".format(
fields.count()))
def dump_contacts(**options):
logger.info("Updating Contacts. Currently have {} contacts in DB.".format(
contacts.count()))
# call contacts API
contacts_list = get_api_data('/contacts.json')
update_collection(collection=contacts, data=contacts_list, id_field='uuid')
# loop through potential next pages
while contacts_list.get('next'):
contacts_list = get_api_data(contacts_list.get('next'))
update_collection(collection=contacts,
data=contacts_list,
id_field='uuid')
logger.info(
"Updated Contacts completed. Now have {} contacts in DB.".format(
contacts.count()))
def dump_relayers(**options):
logger.info("Updating Relayers. Currently have {} relayers in DB.".format(
relayers.count()))
# call relayers API
relayers_list = get_api_data('/relayers.json')
update_collection(collection=relayers,
data=relayers_list,
id_field='relayer')
# loop through potential next pages
while relayers_list.get('next'):
relayers_list = get_api_data(relayers_list.get('next'))
update_collection(collection=relayers,
data=relayers_list,
id_field='relayer')
logger.info(
"Updated Relayers completed. Now have {} relayers in DB.".format(
relayers.count()))
def dump_flows(**options):
logger.info("Updating Flows. Currently have {} flows in DB.".format(
flows.count()))
params = {}
if options.get('after'):
params.update({'after': options.get('after')})
elif options.get('resume'):
if meta.find_one({'endpoint': 'flows'}) is not None:
params.update({
'after':
meta.find_one({
'endpoint': 'flows'
}).get('updated_on')
})
# call flows API
flows_list = get_api_data('/flows.json', **params)
update_collection(collection=flows, data=flows_list, id_field='uuid')
# loop through potential next pages
while flows_list.get('next'):
flows_list = get_api_data(flows_list.get('next'))
update_collection(collection=flows, data=flows_list, id_field='uuid')