def request(self,
method,
endpoint,
data=None,
permissions=None,
payload=None,
**kwargs):
parsed = urlparse(endpoint)
if not parsed.scheme:
actual_url = utils.urljoin(self.server_url, endpoint)
else:
actual_url = endpoint
if self.auth is not None:
kwargs.setdefault('auth', self.auth)
payload = payload or {}
# if data is not None:
payload['data'] = data or {}
if permissions is not None:
if hasattr(permissions, 'as_dict'):
permissions = permissions.as_dict()
payload['permissions'] = permissions
if payload and method not in ('get', 'head'):
payload_kwarg = 'data' if 'files' in kwargs else 'json'
kwargs.setdefault(payload_kwarg, payload)
retry = self.nb_retry
while retry >= 0:
resp = requests.request(method, actual_url, **kwargs)
retry = retry - 1
if not (200 <= resp.status_code < 400):
if resp.status_code >= 500 and retry >= 0:
# Wait and try again.
# If not forced, use retry-after header and wait.
if self.retry_after is None:
retry_after = resp.headers.get("Retry-After", 0)
else:
retry_after = self.retry_after
time.sleep(retry_after)
continue
# Retries exhausted, raise expection.
message = '{0} - {1}'.format(resp.status_code, resp.json())
exception = KintoException(message)
exception.request = resp.request
exception.response = resp
raise exception
if resp.status_code == 304:
body = None
else:
body = resp.json()
# XXX Add the status code.
return body, resp.headers
def send(self):
self._results = []
_exceptions = []
requests = self._build_requests()
id_request = 0
for chunk in utils.chunks(requests, self.batch_max_requests):
kwargs = dict(method="POST",
endpoint=self.endpoints.get("batch"),
payload={"requests": chunk})
resp, headers = self.session.request(**kwargs)
for i, response in enumerate(resp["responses"]):
status_code = response["status"]
level = logging.WARN if status_code < 400 else logging.ERROR
message = response["body"].get("message", "")
logger.log(
level,
"Batch #{}: {} {} - {} {}".format(id_request,
chunk[i]["method"],
chunk[i]["path"],
status_code, message),
)
# Full log in DEBUG mode
logger.debug(
"\nBatch #{}: \n\tRequest: {}\n\tResponse: {}\n".format(
id_request, utils.json_dumps(chunk[i]),
utils.json_dumps(response)))
if not (200 <= status_code < 400):
# One of the server response is an error.
message = "{0} - {1}".format(status_code, response["body"])
exception = KintoException(message)
exception.request = RequestDict(chunk[i])
exception.response = ResponseDict(response)
# Should we ignore 4XX errors?
raise_on_4xx = status_code >= 400 and not self._ignore_4xx_errors
if raise_on_4xx:
_exceptions.append(exception)
if status_code >= 500:
raise exception
id_request += 1
self._results.append((resp, headers))
if _exceptions:
raise KintoBatchException(_exceptions, self._results)
return self._results
def request(self, method, endpoint, data=None, permissions=None,
payload=None, **kwargs):
parsed = urlparse(endpoint)
if not parsed.scheme:
actual_url = utils.urljoin(self.server_url, endpoint)
else:
actual_url = endpoint
if self.auth is not None:
kwargs.setdefault('auth', self.auth)
payload = payload or {}
# if data is not None:
payload['data'] = data or {}
if permissions is not None:
if hasattr(permissions, 'as_dict'):
permissions = permissions.as_dict()
payload['permissions'] = permissions
if payload and method not in ('get', 'head'):
payload_kwarg = 'data' if 'files' in kwargs else 'json'
kwargs.setdefault(payload_kwarg, payload)
retry = self.nb_retry
while retry >= 0:
resp = requests.request(method, actual_url, **kwargs)
retry = retry - 1
if not (200 <= resp.status_code < 400):
if resp.status_code >= 500 and retry >= 0:
# Wait and try again.
# If not forced, use retry-after header and wait.
if self.retry_after is None:
retry_after = resp.headers.get("Retry-After", 0)
else:
retry_after = self.retry_after
time.sleep(retry_after)
continue
# Retries exhausted, raise expection.
message = '{0} - {1}'.format(resp.status_code, resp.json())
exception = KintoException(message)
exception.request = resp.request
exception.response = resp
raise exception
if resp.status_code == 304:
body = None
else:
body = resp.json()
# XXX Add the status code.
return body, resp.headers
def create_group(self,
group,
bucket=None,
data=None,
permissions=None,
safe=True,
if_not_exists=False):
if if_not_exists:
return self._create_if_not_exists('group',
group=group,
bucket=bucket,
data=data,
permissions=permissions,
safe=safe)
headers = DO_NOT_OVERWRITE if safe else None
endpoint = self.get_endpoint('group', bucket=bucket, group=group)
logger.info("Create group %r in bucket %r" % (group, bucket))
try:
resp, _ = self.session.request('put',
endpoint,
data=data,
permissions=permissions,
headers=headers)
except KintoException as e:
if e.response.status_code == 403:
msg = ("Unauthorized. Please check that the bucket exists and "
"that you have the permission to create or write on "
"this group.")
e = KintoException(msg, e)
raise e
return resp
def create_record(self, data, id=None, collection=None, permissions=None,
bucket=None, safe=True, if_not_exists=False):
if if_not_exists:
return self._create_if_not_exists('record',
data=data,
id=id,
collection=collection,
permissions=permissions,
bucket=bucket,
safe=safe)
id = id or data.get('id', None) or str(uuid.uuid4())
# Make sure that no record already exists with this id.
headers = DO_NOT_OVERWRITE if safe else None
endpoint = self.get_endpoint('record', id=id,
bucket=bucket,
collection=collection)
try:
resp, _ = self.session.request('put', endpoint, data=data,
permissions=permissions,
headers=headers)
except KintoException as e:
if e.response.status_code == 403:
msg = ("Unauthorized. Please check that the collection exists "
"and that you have the permission to create or write on"
" this collection record.")
e = KintoException(msg, e)
raise e
return resp
def attach_file(
self,
*,
collection=None,
filePath=None,
fileName="file",
fileContents=None,
mimeType="application/octet-stream",
recordId=None,
):
if not filePath and not fileContents:
raise Exception("Must specify either filePath or fileContents")
if filePath:
files = [("attachment", (fileName, open(filePath, "rb"), mimeType))]
elif fileContents:
files = [("attachment", (fileName, fileContents, mimeType))]
else:
raise Exception("Unexpected state")
attachmentEndpoint = "buckets/{}/collections/{}/records/{}/attachment".format(
self._bucket_name, collection or self._collection_name, recordId
)
response = requests.post(
self.session.server_url + attachmentEndpoint,
files=files,
auth=self.session.auth,
)
if response.status_code > 200:
raise KintoException(
f"Couldn't attach file at endpoint {self.session.server_url}{attachmentEndpoint}: "
+ f"{response.content.decode('utf-8')}"
)
def create_collection(self, collection=None, bucket=None,
data=None, permissions=None, safe=True,
if_not_exists=False):
if if_not_exists:
return self._create_if_not_exists('collection',
collection=collection,
bucket=bucket,
data=data,
permissions=permissions,
safe=safe)
headers = DO_NOT_OVERWRITE if safe else None
endpoint = self.get_endpoint('collection',
bucket=bucket,
collection=collection)
try:
resp, _ = self.session.request('put', endpoint, data=data,
permissions=permissions,
headers=headers)
except KintoException as e:
if e.response.status_code == 403:
msg = ("Unauthorized. Please check that the bucket exists and "
"that you have the permission to create or write on "
"this collection.")
e = KintoException(msg, e)
raise e
return resp
def send(self):
result = []
requests = self._build_requests()
for chunk in utils.chunks(requests, self.batch_max_requests):
kwargs = dict(method='POST',
endpoint=self.endpoints.get('batch'),
payload={'requests': chunk})
resp, headers = self.session.request(**kwargs)
for i, response in enumerate(resp['responses']):
status_code = response['status']
if not (200 <= status_code < 400):
message = '{0} - {1}'.format(status_code, response['body'])
exception = KintoException(message)
exception.request = chunk[i]
exception.response = response
raise exception
result.append((resp, headers))
return result
def send(self):
result = []
requests = self._build_requests()
for chunk in utils.chunks(requests, self.batch_max_requests):
kwargs = dict(method='POST',
endpoint=self.endpoints.get('batch'),
payload={'requests': chunk})
resp, headers = self.session.request(**kwargs)
for i, response in enumerate(resp['responses']):
status_code = response['status']
if not (200 <= status_code < 400):
message = '{0} - {1}'.format(status_code, response['body'])
exception = KintoException(message)
exception.request = chunk[i]
exception.response = response
raise exception
result.append((resp, headers))
return result
def get(self, endpoint, **kwargs):
# Remove nullable values from the kwargs, and slugify the values.
kwargs = dict((k, utils.slugify(v)) for k, v in kwargs.items() if v)
try:
pattern = self.endpoints[endpoint]
return pattern.format(root=self._root, **kwargs)
except KeyError as e:
msg = "Cannot get {endpoint} endpoint, {field} is missing"
raise KintoException(
msg.format(endpoint=endpoint, field=','.join(e.args)))
def send(self):
self._results = []
requests = self._build_requests()
id_request = 0
for chunk in utils.chunks(requests, self.batch_max_requests):
kwargs = dict(method='POST',
endpoint=self.endpoints.get('batch'),
payload={'requests': chunk})
resp, headers = self.session.request(**kwargs)
for i, response in enumerate(resp['responses']):
status_code = response['status']
if not (200 <= status_code < 400):
message = '{0} - {1}'.format(status_code, response['body'])
exception = KintoException(message)
exception.request = chunk[i]
exception.response = response
level = logging.WARN if status_code < 400 else logging.ERROR
message = response["body"].get("message", "")
logger.log(
level,
"Batch #{}: {} {} - {} {}".format(id_request,
chunk[i]["method"],
chunk[i]["path"],
status_code, message))
# Full log in DEBUG mode
logger.debug(
"\nBatch #{}: \n\tRequest: {}\n\tResponse: {}\n".format(
id_request, json.dumps(chunk[i]),
json.dumps(response)))
# Raise in case of a 500
if status_code >= 500:
raise exception
id_request += 1
self._results.append((resp, headers))
return self._results
def test_get_kinto_records_try_to_create_the_collection_and_keep_going_on_403(
):
kinto_client = mock.MagicMock()
Http403 = mock.MagicMock()
Http403.response.status_code = 403
kinto_client.create_collection.side_effect = KintoException(
exception=Http403)
get_kinto_records(kinto_client, mock.sentinel.bucket,
mock.sentinel.collection, mock.sentinel.permissions)
kinto_client.get_records.assert_called_with(
bucket=mock.sentinel.bucket, collection=mock.sentinel.collection)
def create_record(
self,
*,
id=None,
bucket=None,
collection=None,
data=None,
permissions=None,
safe=True,
if_not_exists=False,
):
id = id or data.get("id", None)
if if_not_exists:
return self._create_if_not_exists(
"record",
data=data,
id=id,
collection=collection,
permissions=permissions,
bucket=bucket,
safe=safe,
)
id = id or str(uuid.uuid4())
# Make sure that no record already exists with this id.
headers = DO_NOT_OVERWRITE if safe else None
endpoint = self.get_endpoint("record",
id=id,
bucket=bucket,
collection=collection)
logger.info("Create record with id %r in collection %r in bucket %r" %
(id, collection or self._collection_name, bucket
or self._bucket_name))
try:
resp, _ = self.session.request("put",
endpoint,
data=data,
permissions=permissions,
headers=headers)
except KintoException as e:
if e.response.status_code == 403:
msg = ("Unauthorized. Please check that the collection exists "
"and that you have the permission to create or write on"
" this collection record.")
e = KintoException(msg, e)
raise e
return resp
def send(self):
self._results = []
requests = self._build_requests()
id_request = 0
for chunk in utils.chunks(requests, self.batch_max_requests):
kwargs = dict(method='POST',
endpoint=self.endpoints.get('batch'),
payload={'requests': chunk})
resp, headers = self.session.request(**kwargs)
for i, response in enumerate(resp['responses']):
status_code = response['status']
if not (200 <= status_code < 400):
message = '{0} - {1}'.format(status_code, response['body'])
exception = KintoException(message)
exception.request = chunk[i]
exception.response = response
level = logging.WARN if status_code < 400 else logging.ERROR
message = response["body"].get("message", "")
logger.log(level, "Batch #{}: {} {} - {} {}".format(
id_request, chunk[i]["method"], chunk[i]["path"],
status_code, message))
# Full log in DEBUG mode
logger.debug("\nBatch #{}: \n\tRequest: {}\n\tResponse: {}\n".format(
id_request, json.dumps(chunk[i]), json.dumps(response)))
# Raise in case of a 500
if status_code >= 500:
raise exception
id_request += 1
self._results.append((resp, headers))
return self._results
def sign_collection(self, *, collection=None):
if not self.collection_needs_sign(collection=collection):
log.info("Collection does not require sign. Skipping.")
return
collectionEnd = "buckets/{}/collections/{}".format(
self._bucket_name, collection or self._collection_name
)
response = requests.patch(
self.session.server_url + collectionEnd,
json={"data": {"status": "to-sign"}},
auth=self.session.auth,
)
if response.status_code > 200:
raise KintoException(f"Couldn't sign: {response.content.decode('utf-8')}")
def get_bucket(self, *, id=None):
endpoint = self.get_endpoint('bucket', bucket=id)
logger.info("Get bucket %r" % id or self._bucket_name)
try:
resp, _ = self.session.request('get', endpoint)
except KintoException as e:
error_resp_code = e.response.status_code
if error_resp_code == 401:
msg = ("Unauthorized. Please authenticate or make sure the bucket "
"can be read anonymously.")
e = KintoException(msg, e)
raise e
raise BucketNotFound(id or self._bucket_name, e)
return resp
def create_collection(self,
*,
id=None,
bucket=None,
data=None,
permissions=None,
safe=True,
if_not_exists=False):
if id is None and data:
id = data.get('id', None)
if if_not_exists:
return self._create_if_not_exists('collection',
id=id,
bucket=bucket,
data=data,
permissions=permissions,
safe=safe)
headers = DO_NOT_OVERWRITE if safe else None
endpoint = self.get_endpoint('collection',
bucket=bucket,
collection=id)
logger.info("Create collection %r in bucket %r" %
(id or self._collection_name, bucket or self._bucket_name))
try:
resp, _ = self.session.request('put',
endpoint,
data=data,
permissions=permissions,
headers=headers)
except KintoException as e:
if e.response.status_code == 403:
msg = ("Unauthorized. Please check that the bucket exists and "
"that you have the permission to create or write on "
"this collection.")
e = KintoException(msg, e)
raise e
return resp
def collection_check_state(self, *, collection=None, state):
collectionEnd = "buckets/{}/collections/{}".format(
self._bucket_name, collection or self._collection_name
)
response = requests.get(
self.session.server_url + collectionEnd,
auth=self.session.auth,
)
if response.status_code > 200:
raise KintoException(
f"Couldn't determine review status: {response.content.decode('utf-8')}"
)
status = response.json()["data"]["status"]
log.debug(
f"Collection review status: {status}, expecting {state} ({status==state})"
)
return status == state
def create_group(self,
*,
id=None,
bucket=None,
data=None,
permissions=None,
safe=True,
if_not_exists=False):
if id is None and data:
id = data.get("id", None)
if id is None:
raise KeyError("Please provide a group id")
if if_not_exists:
return self._create_if_not_exists("group",
id=id,
bucket=bucket,
data=data,
permissions=permissions,
safe=safe)
headers = DO_NOT_OVERWRITE if safe else None
endpoint = self.get_endpoint("group", bucket=bucket, group=id)
logger.info("Create group %r in bucket %r" %
(id, bucket or self._bucket_name))
try:
resp, _ = self.session.request("put",
endpoint,
data=data,
permissions=permissions,
headers=headers)
except KintoException as e:
if e.response.status_code == 403:
msg = ("Unauthorized. Please check that the bucket exists and "
"that you have the permission to create or write on "
"this group.")
e = KintoException(msg, e)
raise e
return resp
def publish_intermediates(*, args, ro_client, rw_client):
local_intermediates = {}
remote_intermediates = {}
remote_error_records = []
run_identifiers = workflow.get_run_identifiers(args.filter_bucket)
if not run_identifiers:
log.warning("No run identifiers found")
return
run_id = run_identifiers[-1]
run_id_path = args.download_path / Path(run_id)
run_id_path.mkdir(parents=True, exist_ok=True)
intermediates_path = run_id_path / Path("enrolled.json")
workflow.download_and_retry_from_google_cloud(
args.filter_bucket,
f"{run_id}/enrolled.json",
intermediates_path,
timeout=timedelta(minutes=5),
)
with intermediates_path.open("r") as f:
for entry in json.load(f):
try:
intObj = Intermediate(**entry, debug=args.debug)
if intObj.unique_id() in local_intermediates:
log.warning(
f"[{intObj.unique_id()}] Local collision: {intObj} with "
+ f"{local_intermediates[intObj.unique_id()]}"
)
continue
local_intermediates[intObj.unique_id()] = intObj
except Exception as e:
log.error("Error importing file from {}: {}".format(args.inpath, e))
log.error("Record: {}".format(entry))
raise e
for record in ro_client.get_records(
collection=settings.KINTO_INTERMEDIATES_COLLECTION
):
try:
intObj = Intermediate(**record)
remote_intermediates[intObj.unique_id()] = intObj
except IntermediateRecordError as ire:
log.warning("Skipping broken intermediate record at Kinto: {}".format(ire))
remote_error_records.append(record)
except KeyError as ke:
log.error("Critical error importing Kinto dataset: {}".format(ke))
log.error("Record: {}".format(record))
raise ke
to_delete = set(remote_intermediates.keys()) - set(local_intermediates.keys())
to_upload = set(local_intermediates.keys()) - set(remote_intermediates.keys())
to_update = set()
for i in set(local_intermediates.keys()) & set(remote_intermediates.keys()):
if not local_intermediates[i].equals(remote_record=remote_intermediates[i]):
to_update.add(i)
expired = set()
for i in to_delete:
try:
if remote_intermediates[i].is_expired():
expired.add(i)
except Exception as e:
log.warning(f"Failed to track expiration for {i}: {e}")
to_delete_not_expired = to_delete - expired
delete_pubkeys = {remote_intermediates[i].pubKeyHash for i in to_delete}
upload_pubkeys = {local_intermediates[i].pubKeyHash for i in to_upload}
unenrollments = set()
new_enrollments = set()
update_other_than_enrollment = set()
for i in to_update:
if (
local_intermediates[i].crlite_enrolled
and not remote_intermediates[i].crlite_enrolled
):
new_enrollments.add(i)
elif (
remote_intermediates[i].crlite_enrolled
and not local_intermediates[i].crlite_enrolled
):
unenrollments.add(i)
else:
update_other_than_enrollment.add(i)
log.info(f"Total entries before update: {len(remote_intermediates)}")
log.info(f"To delete: {len(to_delete)} (Deletion enabled: {args.delete})")
log.info(f"- Expired: {len(expired)}")
log.info(f"To add: {len(to_upload)}")
log.info(
f"Certificates updated (without a key change): {len(delete_pubkeys & upload_pubkeys)}"
)
log.info(f"Remote records in an error state: {len(remote_error_records)}")
log.info(f"Total entries updated: {len(to_update)}")
log.info(f"- New enrollments: {len(new_enrollments)}")
log.info(f"- Unenrollments: {len(unenrollments)}")
log.info(f"- Other: {len(update_other_than_enrollment)}")
log.info(f"Total entries after update: {len(local_intermediates)}")
if args.export:
with open(Path(args.export) / Path("to_delete"), "w") as df:
export_intermediates(df, to_delete, remote_intermediates)
with open(Path(args.export) / Path("to_delete_not_expired"), "w") as df:
export_intermediates(df, to_delete_not_expired, remote_intermediates)
with open(Path(args.export) / Path("expired"), "w") as df:
export_intermediates(df, expired, remote_intermediates)
with open(Path(args.export) / Path("to_upload"), "w") as df:
export_intermediates(df, to_upload, local_intermediates)
with open(Path(args.export) / Path("to_update"), "w") as df:
export_intermediates(
df, to_update, local_intermediates, old=remote_intermediates
)
with open(Path(args.export) / Path("unenrollments"), "w") as df:
export_intermediates(
df, unenrollments, local_intermediates, old=remote_intermediates
)
with open(Path(args.export) / Path("new_enrollments"), "w") as df:
export_intermediates(
df, new_enrollments, local_intermediates, old=remote_intermediates
)
with open(Path(args.export) / Path("update_other_than_enrollment"), "w") as df:
export_intermediates(
df,
update_other_than_enrollment,
local_intermediates,
old=remote_intermediates,
)
if args.debug:
print("Variables available:")
print(" local_intermediates")
print(" remote_intermediates")
print(" remote_error_records")
print("")
print(" to_upload")
print(" to_delete")
print(" to_update")
print("")
print(" new_enrollments")
print(" unenrollments")
print("")
print(" delete_pubkeys")
print(" upload_pubkeys")
print(
" delete_pubkeys & upload_pubkeys # certs updated without changing the key"
)
print("")
print(" local_intermediates[to_update.pop()].cert # get cert object")
print("")
print("Use 'continue' to proceed")
print("")
breakpoint()
if args.noop:
log.info(f"Noop flag set, exiting before any intermediate updates")
return
# Don't accidentally use the ro_client beyond this point
ro_client = None
if len(remote_error_records) > 0:
log.info(f"Cleaning {len(remote_error_records)} broken records")
for record in remote_error_records:
try:
rw_client.delete_record(
collection=settings.KINTO_INTERMEDIATES_COLLECTION,
id=record["id"],
)
except KintoException as ke:
log.warning(f"Couldn't delete record id {record['id']}: {ke}")
for unique_id in to_delete:
intermediate = remote_intermediates[unique_id]
log.info(f"Deleting {intermediate} from Kinto (delete={args.delete})")
if args.delete:
try:
intermediate.delete_from_kinto(rw_client=rw_client)
except KintoException as ke:
log.warning(f"Couldn't delete record id {intermediate}: {ke}")
for unique_id in to_upload:
intermediate = local_intermediates[unique_id]
log.debug(f"Uploading {intermediate} to Kinto")
intermediate.add_to_kinto(rw_client=rw_client)
update_error_records = []
for unique_id in to_update:
local_int = local_intermediates[unique_id]
remote_int = remote_intermediates[unique_id]
if not local_int.equals(remote_record=remote_int):
try:
local_int.update_kinto(
rw_client=rw_client,
remote_record=remote_int,
)
except KintoException as ke:
update_error_records.append((local_int, remote_int, ke))
for (local_int, remote_int, ex) in update_error_records:
log.warning(
f"Failed to update local={local_int} remote={remote_int} exception={ex}"
)
log.info("Verifying correctness...")
verified_intermediates = {}
verification_error_records = []
for record in rw_client.get_records(
collection=settings.KINTO_INTERMEDIATES_COLLECTION
):
try:
intObj = Intermediate(**record)
verified_intermediates[intObj.unique_id()] = intObj
except IntermediateRecordError as ire:
log.warning(
"Verification found broken intermediate record at Kinto: {}".format(ire)
)
verification_error_records.append(record)
except KeyError as ke:
log.error("Critical error importing Kinto dataset: {}".format(ke))
log.error("Record: {}".format(record))
raise ke
if len(verification_error_records) > 0:
raise KintoException(
"There were {} broken intermediates. Re-run to fix.".format(
len(verification_error_records)
)
)
log.info(
"{} intermediates locally, {} at Kinto.".format(
len(local_intermediates), len(verified_intermediates)
)
)
if args.delete and set(local_intermediates.keys()) != set(
verified_intermediates.keys()
):
log.error("The verified intermediates do not match the local set. Differences:")
missing_remote = set(local_intermediates.keys()) - set(
verified_intermediates.keys()
)
missing_local = set(verified_intermediates.keys()) - set(
local_intermediates.keys()
)
for d in missing_remote:
log.error("{} does not exist at Kinto".format(d))
for d in missing_local:
log.error(
"{} exists at Kinto but should have been deleted (not in local set)".format(
d
)
)
raise KintoException("Local/Remote Verification Failed")
elif not args.delete and set(local_intermediates.keys()) > set(
verified_intermediates.keys()
):
log.error("The verified intermediates do not match the local set. Differences:")
missing_remote = set(local_intermediates.keys()) - set(
verified_intermediates.keys()
)
for d in missing_remote:
log.error("{} does not exist at Kinto".format(d))
raise KintoException("Local/Remote Verification Failed")
for unique_id in verified_intermediates.keys():
remote_int = verified_intermediates[unique_id]
if unique_id not in local_intermediates and not args.delete:
log.info(
"Remote {} has been deleted locally, but ignoring.".format(remote_int)
)
continue
local_int = local_intermediates[unique_id]
if not local_int.equals(remote_record=remote_int):
if not remote_int.pemAttachment.verify(pemData=local_int.pemData):
log.warning(
"PEM hash mismatch for {}; remote={} != local={}".format(
unique_id, remote_int, local_int
)
)
raise KintoException(
"Local/Remote PEM mismatch for uniqueId={}".format(unique_id)
)
else:
log.warning(
f"Local/Remote metadata mismatch, uniqueID={unique_id}, "
+ f"local={local_int.details()}, remote={remote_int.details()}"
)
raise KintoException(
"Local/Remote metadata mismatch for uniqueId={}".format(unique_id)
)
if to_update or to_upload or to_delete and not args.noop and args.request_review:
log.info(
f"Set for review, {len(to_update)} updates, {len(to_upload)} uploads, "
+ f"{len(to_delete)} deletions."
)
rw_client.request_review_of_collection(
collection=settings.KINTO_INTERMEDIATES_COLLECTION,
)
else:
log.info(f"No updates to do")
def request(self, method, endpoint, data=None, permissions=None,
payload=None, **kwargs):
current_time = time.time()
if self.backoff and self.backoff > current_time:
seconds = int(self.backoff - current_time)
raise BackoffException("Retry after {} seconds".format(seconds), seconds)
parsed = urlparse(endpoint)
if not parsed.scheme:
actual_url = utils.urljoin(self.server_url, endpoint)
else:
actual_url = endpoint
if self.auth is not None:
kwargs.setdefault('auth', self.auth)
payload = payload or {}
if data is not None:
payload['data'] = data
if permissions is not None:
if hasattr(permissions, 'as_dict'):
permissions = permissions.as_dict()
payload['permissions'] = permissions
if method not in ('get', 'head'):
payload_kwarg = 'data' if 'files' in kwargs else 'json'
kwargs.setdefault(payload_kwarg, payload)
# Set the default User-Agent if not already defined.
if not isinstance(kwargs.get('headers'), dict):
kwargs['headers'] = {}
kwargs['headers'].setdefault('User-Agent', USER_AGENT)
retry = self.nb_retry
while retry >= 0:
resp = requests.request(method, actual_url, **kwargs)
backoff_seconds = resp.headers.get("Backoff")
if backoff_seconds:
self.backoff = time.time() + int(backoff_seconds)
else:
self.backoff = None
retry = retry - 1
if 200 <= resp.status_code < 400:
# Success
break
else:
if resp.status_code >= 500 and retry >= 0:
# Wait and try again.
# If not forced, use retry-after header and wait.
if self.retry_after is None:
retry_after = int(resp.headers.get("Retry-After", 0))
else:
retry_after = self.retry_after
time.sleep(retry_after)
continue
# Retries exhausted, raise expection.
try:
message = '{0} - {1}'.format(resp.status_code, resp.json())
except ValueError:
# In case the response is not JSON, fallback to text.
message = '{0} - {1}'.format(resp.status_code, resp.text)
exception = KintoException(message)
exception.request = resp.request
exception.response = resp
raise exception
if resp.status_code == 304 or method == 'head':
body = None
else:
body = resp.json()
return body, resp.headers
def request(self, method, endpoint, data=None, permissions=None, payload=None, **kwargs):
current_time = time.time()
if self.backoff and self.backoff > current_time:
seconds = int(self.backoff - current_time)
raise BackoffException("Retry after {} seconds".format(seconds), seconds)
parsed = urlparse(endpoint)
if not parsed.scheme:
actual_url = utils.urljoin(self.server_url, endpoint)
else:
actual_url = endpoint
if self.timeout is not False:
kwargs.setdefault("timeout", self.timeout)
if self.auth is not None:
kwargs.setdefault("auth", self.auth)
if kwargs.get("params") is not None:
params = dict()
for key, value in kwargs["params"].items():
if key.startswith("in_") or key.startswith("exclude_"):
params[key] = ",".join(value)
elif isinstance(value, str):
params[key] = value
else:
params[key] = json.dumps(value)
kwargs["params"] = params
overridden_headers = kwargs.get("headers") or {}
# Set the default User-Agent if not already defined.
kwargs["headers"] = {"User-Agent": USER_AGENT, **self.headers, **overridden_headers}
payload = payload or {}
if data is not None:
payload["data"] = data
if permissions is not None:
if hasattr(permissions, "as_dict"):
permissions = permissions.as_dict()
payload["permissions"] = permissions
if method not in ("get", "head"):
if "files" in kwargs:
kwargs.setdefault("data", payload)
else:
kwargs.setdefault("data", utils.json_dumps(payload))
kwargs["headers"].setdefault("Content-Type", "application/json")
retry = self.nb_retry
while retry >= 0:
resp = requests.request(method, actual_url, **kwargs)
backoff_seconds = resp.headers.get("Backoff")
if backoff_seconds:
self.backoff = time.time() + int(backoff_seconds)
else:
self.backoff = None
retry = retry - 1
if 200 <= resp.status_code < 400:
# Success
break
else:
if retry >= 0 and (resp.status_code >= 500 or resp.status_code == 409):
# Wait and try again.
# If not forced, use retry-after header and wait.
if self.retry_after is None:
retry_after = int(resp.headers.get("Retry-After", 0))
else:
retry_after = self.retry_after
time.sleep(retry_after)
continue
# Retries exhausted, raise expection.
try:
message = "{0} - {1}".format(resp.status_code, resp.json())
except ValueError:
# In case the response is not JSON, fallback to text.
message = "{0} - {1}".format(resp.status_code, resp.text)
exception = KintoException(message)
exception.request = resp.request
exception.response = resp
raise exception
if resp.status_code == 204 or resp.status_code == 304 or method == "head":
body = None
else:
body = resp.json()
return body, resp.headers
def request(self,
method,
endpoint,
data=None,
permissions=None,
payload=None,
**kwargs):
current_time = time.time()
if self.backoff and self.backoff > current_time:
seconds = int(self.backoff - current_time)
raise BackoffException("Retry after {} seconds".format(seconds),
seconds)
parsed = urlparse(endpoint)
if not parsed.scheme:
actual_url = utils.urljoin(self.server_url, endpoint)
else:
actual_url = endpoint
if self.auth is not None:
kwargs.setdefault('auth', self.auth)
if kwargs.get('headers') is None:
kwargs['headers'] = dict()
if kwargs.get('params') is not None:
params = dict()
for key, value in kwargs["params"].items():
if key.startswith('in_') or key.startswith('exclude_'):
params[key] = ','.join(value)
elif isinstance(value, str):
params[key] = value
else:
params[key] = json.dumps(value)
kwargs['params'] = params
if not isinstance(kwargs['headers'], dict):
raise TypeError("headers must be a dict (got {})".format(
kwargs['headers']))
# Set the default User-Agent if not already defined.
# In the meantime, clone the header dict to avoid changing the
# user header dict when adding information.
kwargs['headers'] = {"User-Agent": USER_AGENT, **kwargs["headers"]}
payload = payload or {}
if data is not None:
payload['data'] = data
if permissions is not None:
if hasattr(permissions, 'as_dict'):
permissions = permissions.as_dict()
payload['permissions'] = permissions
if method not in ('get', 'head'):
if 'files' in kwargs:
kwargs.setdefault('data', payload)
else:
kwargs.setdefault('data', utils.json_dumps(payload))
kwargs['headers'].setdefault('Content-Type',
'application/json')
retry = self.nb_retry
while retry >= 0:
resp = requests.request(method, actual_url, **kwargs)
backoff_seconds = resp.headers.get("Backoff")
if backoff_seconds:
self.backoff = time.time() + int(backoff_seconds)
else:
self.backoff = None
retry = retry - 1
if 200 <= resp.status_code < 400:
# Success
break
else:
if retry >= 0 and (resp.status_code >= 500
or resp.status_code == 409):
# Wait and try again.
# If not forced, use retry-after header and wait.
if self.retry_after is None:
retry_after = int(resp.headers.get("Retry-After", 0))
else:
retry_after = self.retry_after
time.sleep(retry_after)
continue
# Retries exhausted, raise expection.
try:
message = '{0} - {1}'.format(resp.status_code, resp.json())
except ValueError:
# In case the response is not JSON, fallback to text.
message = '{0} - {1}'.format(resp.status_code, resp.text)
exception = KintoException(message)
exception.request = resp.request
exception.response = resp
raise exception
if resp.status_code == 304 or method == 'head':
body = None
else:
body = resp.json()
return body, resp.headers
def get_http_error(status):
exception = KintoException()
exception.response = mock.MagicMock()
exception.response.status_code = status
exception.request = mock.sentinel.request
return exception
def get_http_error(status):
exception = KintoException()
exception.response = mock.MagicMock()
exception.response.status_code = status
exception.request = mock.sentinel.request
return exception