def pull_monitor(model_url, callback, delay=dict(minutes=1),
page_url=None, floor=0, pull_priority=5, job_priority=5):
"""Used to look for instances that need to be pulled.
This only works with models who use an auto-incremented primary key.
"""
if not page_url:
model = get_model(model_url)
instances_url = model._operations['instances']
else:
instances_url = page_url
_, json = get(instances_url or page_url)
latest, highest = None, floor
for item in json['page']:
highest = max(item['pk'], highest)
latest = item['pk']
if latest > floor:
schedule(callback, args=[urljoin(instances_url, item['data'])], priority=job_priority)
if json.has_key('next_page') and latest > floor:
schedule('pubsubpull.async.pull_monitor', args=[model_url, callback],
kwargs=dict(delay=delay, floor=floor,
page_url=urljoin(instances_url, json['next_page']),
pull_priority=pull_priority, job_priority=job_priority),
priority=pull_priority)
print("Got another page to process", json['next_page'], floor)
if not page_url:
run_after = timezone.now() + timedelta(**delay)
schedule('pubsubpull.async.pull_monitor', run_after=run_after,
args=[model_url, callback], kwargs=dict(delay=delay, floor=highest,
pull_priority=pull_priority, job_priority=job_priority),
priority=pull_priority)
print("Looking for new instances above", highest)
def pull_monitor(model_url,
callback,
delay=None,
page_url=None,
floor=0,
pull_priority=5,
job_priority=5,
callback_kwargs=None):
"""Used to look for instances that need to be pulled.
This only works with models who use an auto-incremented primary key.
"""
if callback_kwargs is None:
callback_kwargs = {}
if delay is None:
delay = dict(minutes=1)
if not page_url:
model = get_model(model_url)
instances_url = model._operations['instances']
else:
instances_url = page_url
_, json = get(instances_url or page_url)
latest, highest = None, floor
for item in json['page']:
highest = max(item['pk'], highest)
latest = item['pk']
if latest > floor:
schedule(callback,
args=[urljoin(instances_url, item['data'])],
kwargs=callback_kwargs,
priority=job_priority)
if 'next_page' in json and latest > floor:
schedule('pubsubpull.async.pull_monitor',
args=[model_url, callback],
kwargs=dict(callback_kwargs=callback_kwargs,
delay=delay,
floor=floor,
job_priority=job_priority,
page_url=urljoin(instances_url,
json['next_page']),
pull_priority=pull_priority),
priority=pull_priority)
print("Got another page to process", json['next_page'], floor)
if not page_url and delay:
run_after = timezone.now() + timedelta(**delay)
schedule('pubsubpull.async.pull_monitor',
args=[model_url, callback],
run_after=run_after,
priority=pull_priority,
kwargs=dict(callback_kwargs=callback_kwargs,
delay=delay,
floor=highest,
job_priority=job_priority,
pull_priority=pull_priority))
print("Looking for new instances above", highest)
def pull_up(model, callback, callback_kwargs=None, **kwargs):
"""Start a job monitoring new instance from latest instance.
"""
if callback_kwargs is None:
callback_kwargs = {}
kwargs['callback_kwargs'] = callback_kwargs
model_instance = get_model(model)
instance_url = model_instance._operations['instances']
_, json_data = get(instance_url)
kwargs['floor'] = json_data['page'][0]['pk'] if json_data['page'] else 0
schedule('pubsubpull.async.pull_monitor', args=[model, callback], kwargs=kwargs)
def pull_up(model, callback, callback_kwargs=None, **kwargs):
"""Start a job monitoring new instance from latest instance.
"""
if callback_kwargs is None:
callback_kwargs = {}
kwargs['callback_kwargs'] = callback_kwargs
model_instance = get_model(model)
instance_url = model_instance._operations['instances']
_, json_data = get(instance_url)
kwargs['floor'] = json_data['page'][0]['pk'] if json_data['page'] else 0
schedule('pubsubpull.async.pull_monitor',
args=[model, callback],
kwargs=kwargs)
def from_json_data(base_url, json):
"""Convert a JSON representation of some data to the right types within
the client.
"""
if json['kind'] == 'object':
if json['data'] is None:
return None
else:
# It's a remote object
from slumber.connector.api import get_instance, get_model
model_url = urljoin(base_url, json['data']['type'])
data_url = urljoin(base_url, json['data']['data'])
display = json['data']['display']
return get_instance(get_model(model_url), data_url, display)
else:
return json['data']
def __getattr__(self, attr_name):
"""Fetch the application list from the Slumber directory on request.
"""
logging.debug("Looking for attribute %s on %s for directory %s",
attr_name, self, self._directory)
if not self._directory:
logging.debug("Raising AttributeError as _directory is falsey")
raise AttributeError(attr_name)
_, json = get(self._directory)
logging.debug(
"Looking for attribute %s on %s resulted in these applications %s",
attr_name, self, json)
# Pylint gets confused by the JSON object
# pylint: disable=E1103
json_apps = json.get('apps', {})
apps = {}
for app in json_apps.keys():
root = apps
for k in app.split('.'):
if not root.has_key(k):
root[k] = {}
root = root[k]
def recurse_apps(loc, this_level, name):
"""Recursively build the application connectors.
"""
current_appname = '.'.join(name)
if json_apps.has_key(current_appname):
loc._directory = urljoin(self._directory,
json_apps[current_appname])
for k, v in this_level.items():
app_cnx = ServiceConnector(None)
setattr(loc, k, app_cnx)
recurse_apps(app_cnx, v, name + [k])
recurse_apps(self, apps, [])
models = json.get('models', {})
for model_name, url in models.items():
model_url = urljoin(self._directory, url)
model = get_model(model_url)
setattr(self, model_name, model)
if attr_name in models.keys():
return getattr(self, attr_name)
elif attr_name in apps.keys():
return getattr(self, attr_name)
else:
raise AttributeError(attr_name, json)
def __getattr__(self, attr_name):
"""Fetch the application list from the Slumber directory on request.
"""
logging.debug("Looking for attribute %s on %s for directory %s",
attr_name, self, self._directory)
if not self._directory:
logging.debug("Raising AttributeError as _directory is falsey")
raise AttributeError(attr_name)
_, json = get(self._directory)
logging.debug(
"Looking for attribute %s on %s resulted in these applications %s",
attr_name, self, json)
# Pylint gets confused by the JSON object
# pylint: disable=E1103
json_apps = json.get('apps', {})
apps = {}
for app in json_apps.keys():
root = apps
for k in app.split('.'):
if not root.has_key(k):
root[k] = {}
root = root[k]
def recurse_apps(loc, this_level, name):
"""Recursively build the application connectors.
"""
current_appname = '.'.join(name)
if json_apps.has_key(current_appname):
loc._directory = urljoin(self._directory,
json_apps[current_appname])
for k, v in this_level.items():
app_cnx = ServiceConnector(None)
setattr(loc, k, app_cnx)
recurse_apps(app_cnx, v, name + [k])
recurse_apps(self, apps, [])
models = json.get('models', {})
for model_name, url in models.items():
model_url = urljoin(self._directory, url)
model = get_model(model_url)
setattr(self, model_name, model)
if attr_name in models.keys():
return getattr(self, attr_name)
elif attr_name in apps.keys():
return getattr(self, attr_name)
else:
raise AttributeError(attr_name, json)
