def _purge(self, start_date, end_date): for phase, sources, targets in self.sequence: for source in sources: try: source.purge(start_date, end_date) except Exception: logger.error('Failed to purge %r' % source.get_id())
def _clear(self, start_date, end_date): source_ids = set() plugins = [] for phase, sources, targets in self.sequence: source_ids.update(set([s.get_id() for s in sources])) plugins.extend(targets) for target in plugins: try: target.clear(start_date, end_date, list(source_ids)) except Exception: logger.error('Failed to clear %r' % target.get_id())
def read_api(self, url, params=None, data=None): """Reads an API, follows pagination and return the resulting objects. :param url: Url to read from. :param params: List of params to pass in the querystring (filters). :param data: Used to handle recursive calls. The data being retrieved. """ if data is None: data = [] if not params: params = {} orig_params = params.copy() while True: resp = self.session.get(url, params=params) if 400 <= resp.status_code <= 499: logger.error('API 4xx Error: %s Url: %s' % (resp.json()['reason'], url)) return data if 500 <= resp.status_code <= 599: logger.error('API 5xx Error: %s Url: %s' % (resp.text, url)) raise ServerError(resp.status_code) res = resp.json() data.extend(res['objects']) # we can have paginated elements, so we need to get them all next_ = None if 'meta' in res: next_ = res['meta'].get('next') if next_: # Update the params to pick up the new offset. params = orig_params.copy() qs = urlparse(next_).query params.update(dict(parse_qsl(qs))) else: return data
def read_api(self, url, params=None, data=None): """Gets transaction data from Marketplace Transaction API. http://firefox-marketplace-api.readthedocs.org/en/latest/topics/transactions.html """ if data is None: data = {} if not params: params = {} resp = self.session.get(url, params=params) if 400 <= resp.status_code <= 499: logger.error('API 4xx Error: {0} Url: {1}'.format( resp.json().get('detail', resp.text), url)) return None if 500 <= resp.status_code <= 599: logger.error('API 5xx Error: {0} Url: {1}'.format(resp.text, url)) raise ServerError(resp.status_code) return resp.json()
def read_api(self, url, params=None, data=None): """Gets transaction data from Marketplace Transaction API. http://firefox-marketplace-api.readthedocs.org/en/latest/topics/transactions.html """ if data is None: data = {} if not params: params = {} resp = self.session.get(url, params=params) if 400 <= resp.status_code <= 499: logger.error('API 4xx Error: {0} Url: {1}'.format( resp.json().get('detail', resp.text), url)) return None if 500 <= resp.status_code <= 599: logger.error('API 5xx Error: {0} Url: {1}'.format( resp.text, url)) raise ServerError(resp.status_code) return resp.json()