def pull_monitor(model_url, callback, delay=dict(minutes=1), page_url=None, floor=0, pull_priority=5, job_priority=5): """Used to look for instances that need to be pulled. This only works with models who use an auto-incremented primary key. """ if not page_url: model = get_model(model_url) instances_url = model._operations['instances'] else: instances_url = page_url _, json = get(instances_url or page_url) latest, highest = None, floor for item in json['page']: highest = max(item['pk'], highest) latest = item['pk'] if latest > floor: schedule(callback, args=[urljoin(instances_url, item['data'])], priority=job_priority) if json.has_key('next_page') and latest > floor: schedule('pubsubpull.async.pull_monitor', args=[model_url, callback], kwargs=dict(delay=delay, floor=floor, page_url=urljoin(instances_url, json['next_page']), pull_priority=pull_priority, job_priority=job_priority), priority=pull_priority) print("Got another page to process", json['next_page'], floor) if not page_url: run_after = timezone.now() + timedelta(**delay) schedule('pubsubpull.async.pull_monitor', run_after=run_after, args=[model_url, callback], kwargs=dict(delay=delay, floor=highest, pull_priority=pull_priority, job_priority=job_priority), priority=pull_priority) print("Looking for new instances above", highest)
def pull_monitor(model_url, callback, delay=None, page_url=None, floor=0, pull_priority=5, job_priority=5, callback_kwargs=None): """Used to look for instances that need to be pulled. This only works with models who use an auto-incremented primary key. """ if callback_kwargs is None: callback_kwargs = {} if delay is None: delay = dict(minutes=1) if not page_url: model = get_model(model_url) instances_url = model._operations['instances'] else: instances_url = page_url _, json = get(instances_url or page_url) latest, highest = None, floor for item in json['page']: highest = max(item['pk'], highest) latest = item['pk'] if latest > floor: schedule(callback, args=[urljoin(instances_url, item['data'])], kwargs=callback_kwargs, priority=job_priority) if 'next_page' in json and latest > floor: schedule('pubsubpull.async.pull_monitor', args=[model_url, callback], kwargs=dict(callback_kwargs=callback_kwargs, delay=delay, floor=floor, job_priority=job_priority, page_url=urljoin(instances_url, json['next_page']), pull_priority=pull_priority), priority=pull_priority) print("Got another page to process", json['next_page'], floor) if not page_url and delay: run_after = timezone.now() + timedelta(**delay) schedule('pubsubpull.async.pull_monitor', args=[model_url, callback], run_after=run_after, priority=pull_priority, kwargs=dict(callback_kwargs=callback_kwargs, delay=delay, floor=highest, job_priority=job_priority, pull_priority=pull_priority)) print("Looking for new instances above", highest)
def pull_up(model, callback, callback_kwargs=None, **kwargs): """Start a job monitoring new instance from latest instance. """ if callback_kwargs is None: callback_kwargs = {} kwargs['callback_kwargs'] = callback_kwargs model_instance = get_model(model) instance_url = model_instance._operations['instances'] _, json_data = get(instance_url) kwargs['floor'] = json_data['page'][0]['pk'] if json_data['page'] else 0 schedule('pubsubpull.async.pull_monitor', args=[model, callback], kwargs=kwargs)
def from_json_data(base_url, json): """Convert a JSON representation of some data to the right types within the client. """ if json['kind'] == 'object': if json['data'] is None: return None else: # It's a remote object from slumber.connector.api import get_instance, get_model model_url = urljoin(base_url, json['data']['type']) data_url = urljoin(base_url, json['data']['data']) display = json['data']['display'] return get_instance(get_model(model_url), data_url, display) else: return json['data']
def __getattr__(self, attr_name): """Fetch the application list from the Slumber directory on request. """ logging.debug("Looking for attribute %s on %s for directory %s", attr_name, self, self._directory) if not self._directory: logging.debug("Raising AttributeError as _directory is falsey") raise AttributeError(attr_name) _, json = get(self._directory) logging.debug( "Looking for attribute %s on %s resulted in these applications %s", attr_name, self, json) # Pylint gets confused by the JSON object # pylint: disable=E1103 json_apps = json.get('apps', {}) apps = {} for app in json_apps.keys(): root = apps for k in app.split('.'): if not root.has_key(k): root[k] = {} root = root[k] def recurse_apps(loc, this_level, name): """Recursively build the application connectors. """ current_appname = '.'.join(name) if json_apps.has_key(current_appname): loc._directory = urljoin(self._directory, json_apps[current_appname]) for k, v in this_level.items(): app_cnx = ServiceConnector(None) setattr(loc, k, app_cnx) recurse_apps(app_cnx, v, name + [k]) recurse_apps(self, apps, []) models = json.get('models', {}) for model_name, url in models.items(): model_url = urljoin(self._directory, url) model = get_model(model_url) setattr(self, model_name, model) if attr_name in models.keys(): return getattr(self, attr_name) elif attr_name in apps.keys(): return getattr(self, attr_name) else: raise AttributeError(attr_name, json)