def _purge(self, start_date, end_date):
for phase, sources, targets in self.sequence:
for source in sources:
try:
source.purge(start_date, end_date)
except Exception:
logger.error('Failed to purge %r' % source.get_id())
def _clear(self, start_date, end_date):
source_ids = set()
plugins = []
for phase, sources, targets in self.sequence:
source_ids.update(set([s.get_id() for s in sources]))
plugins.extend(targets)
for target in plugins:
try:
target.clear(start_date, end_date, list(source_ids))
except Exception:
logger.error('Failed to clear %r' % target.get_id())
def read_api(self, url, params=None, data=None):
"""Reads an API, follows pagination and return the resulting objects.
:param url: Url to read from.
:param params: List of params to pass in the querystring (filters).
:param data: Used to handle recursive calls. The data being retrieved.
"""
if data is None:
data = []
if not params:
params = {}
orig_params = params.copy()
while True:
resp = self.session.get(url, params=params)
if 400 <= resp.status_code <= 499:
logger.error('API 4xx Error: %s Url: %s' %
(resp.json()['reason'], url))
return data
if 500 <= resp.status_code <= 599:
logger.error('API 5xx Error: %s Url: %s' % (resp.text, url))
raise ServerError(resp.status_code)
res = resp.json()
data.extend(res['objects'])
# we can have paginated elements, so we need to get them all
next_ = None
if 'meta' in res:
next_ = res['meta'].get('next')
if next_:
# Update the params to pick up the new offset.
params = orig_params.copy()
qs = urlparse(next_).query
params.update(dict(parse_qsl(qs)))
else:
return data
def read_api(self, url, params=None, data=None):
"""Reads an API, follows pagination and return the resulting objects.
:param url: Url to read from.
:param params: List of params to pass in the querystring (filters).
:param data: Used to handle recursive calls. The data being retrieved.
"""
if data is None:
data = []
if not params:
params = {}
orig_params = params.copy()
while True:
resp = self.session.get(url, params=params)
if 400 <= resp.status_code <= 499:
logger.error('API 4xx Error: %s Url: %s' %
(resp.json()['reason'], url))
return data
if 500 <= resp.status_code <= 599:
logger.error('API 5xx Error: %s Url: %s' % (resp.text, url))
raise ServerError(resp.status_code)
res = resp.json()
data.extend(res['objects'])
# we can have paginated elements, so we need to get them all
next_ = None
if 'meta' in res:
next_ = res['meta'].get('next')
if next_:
# Update the params to pick up the new offset.
params = orig_params.copy()
qs = urlparse(next_).query
params.update(dict(parse_qsl(qs)))
else:
return data
def read_api(self, url, params=None, data=None):
"""Gets transaction data from Marketplace Transaction API.
http://firefox-marketplace-api.readthedocs.org/en/latest/topics/transactions.html
"""
if data is None:
data = {}
if not params:
params = {}
resp = self.session.get(url, params=params)
if 400 <= resp.status_code <= 499:
logger.error('API 4xx Error: {0} Url: {1}'.format(
resp.json().get('detail', resp.text), url))
return None
if 500 <= resp.status_code <= 599:
logger.error('API 5xx Error: {0} Url: {1}'.format(resp.text, url))
raise ServerError(resp.status_code)
return resp.json()
def read_api(self, url, params=None, data=None):
"""Gets transaction data from Marketplace Transaction API.
http://firefox-marketplace-api.readthedocs.org/en/latest/topics/transactions.html
"""
if data is None:
data = {}
if not params:
params = {}
resp = self.session.get(url, params=params)
if 400 <= resp.status_code <= 499:
logger.error('API 4xx Error: {0} Url: {1}'.format(
resp.json().get('detail', resp.text), url))
return None
if 500 <= resp.status_code <= 599:
logger.error('API 5xx Error: {0} Url: {1}'.format(
resp.text, url))
raise ServerError(resp.status_code)
return resp.json()