def send_request(self, requests_callable, *args, **kwargs):
# this is the timeout of requests module
requests_timeout = kwargs.pop('timeout', self.requests_timeout)
files = kwargs.pop('files', None)
if files:
for key, f in files.items():
# file can be a tuple
# if so, the fileobj is in second position
if isinstance(f, (tuple, list)):
f = f[1]
if f is None or isinstance(
f, (string_types, bytes, bytearray, int, float, bool)):
continue
error = "Unsupported file object type '{}' for key '{}'".format(
type(f), key)
# seek files before each retry, to avoid silently retrying with different input
if hasattr(f, 'seek'):
if hasattr(f, 'seekable') and not f.seekable():
raise DeepomaticException(
"{}: not seekable".format(error))
f.seek(0)
continue
raise DeepomaticException(
"{}: not a scalar or seekable.".format(error))
return requests_callable(*args,
files=files,
timeout=requests_timeout,
verify=self.verify_ssl,
**kwargs)
def update(self,
replace=False,
content_type='application/json',
files=None,
**kwargs):
assert (self._pk is not None)
if self._helper.check_query_parameters:
for arg_name in kwargs:
if arg_name not in self.object_template:
raise DeepomaticException("Unexpected keyword argument: " +
arg_name)
for arg_name, arg in self.object_template.items():
if not arg._mutable and arg_name in kwargs:
raise DeepomaticException("Immutable keyword argument: " +
arg_name)
if replace:
self._data = self._helper.put(self._uri(pk=self._pk),
data=kwargs,
content_type=content_type,
files=files)
else:
self._data = self._helper.patch(self._uri(pk=self._pk),
data=kwargs,
content_type=content_type,
files=files)
def create(self, content_type='application/json', files=None, **kwargs):
post_kwargs = {}
# TODO: this is a hack, kwargs shouldn't be the data to post
# it should be the requests kwargs
# the fix will be a breaking change, so for now we just pop them
for key in ['http_retry', 'timeout']:
try:
post_kwargs[key] = kwargs.pop(key)
except KeyError:
pass
if self._helper.check_query_parameters:
for arg_name in kwargs:
if arg_name not in self.object_template or self.object_template[
arg_name]._shoud_be_present_when_adding is False:
raise DeepomaticException("Unexpected keyword argument: " +
arg_name)
for arg_name, arg in self.object_template.items():
if arg._shoud_be_present_when_adding and arg_name not in kwargs:
raise DeepomaticException("Missing keyword argument: " +
arg_name)
if files is not None:
content_type = 'multipart/mixed'
data = self._helper.post(self._uri(),
data=kwargs,
content_type=content_type,
files=files,
**post_kwargs)
return self.__class__(self._helper, pk=data['id'], data=data)
def _refresh_tasks_status(self, pending_tasks, success_tasks, error_tasks,
positions):
logger.debug("Refreshing batch of Task {}".format(pending_tasks))
task_ids = [task.pk for idx, task in pending_tasks]
functor = functools.partial(self.list, task_ids=task_ids)
refreshed_tasks = warn_on_http_retry_error(
functor,
suffix="Retrying until Task.batch_wait timeouts.",
reraise=True)
pending_tasks[:] = [] # clear the list (we have to keep the reference)
for task in refreshed_tasks:
status = task['status']
pos = positions[task.pk]
if is_pending_status(status):
pending_tasks.append((pos, task))
elif is_error_status(status):
error_tasks.append((pos, task))
elif is_success_status(status):
success_tasks.append((pos, task))
else:
raise DeepomaticException("Unknown task status %s" % status)
return pending_tasks
def __init__(self, source, encoding=None):
is_file = hasattr(source, 'read')
is_raw = False
if not is_file:
is_raw = (
(sys.version_info >=
(3, 0) and isinstance(source, bytes)) or not any(
[source.startswith(p) for p in self.supported_protocols]))
if is_raw:
if encoding is None:
raise DeepomaticException(
"You must specify 'encoding' when passing raw data")
if encoding not in self.supported_encodings:
raise DeepomaticException("Unknown 'encoding' type.")
# Send binary directly to minimize load
if encoding == 'base64':
source = base64.b64decode(source)
encoding = 'binary'
self._source = source
self._need_multipart = is_file or (is_raw and encoding == 'binary')
def recursive_json_dump(prefix, obj, data_dict, omit_dot=False):
if isinstance(obj, dict):
if not omit_dot: # see comment below
prefix += '.'
for key, value in obj.items():
recursive_json_dump(prefix + key, value, data_dict)
elif isinstance(obj, list):
for i, value in enumerate(obj):
# omit_dot is True as DRF parses list of dictionnaries like this:
# {"parent": [{"subfield": 0}]} would be:
# 'parent[0]subfield': 0
recursive_json_dump(prefix + '[{}]'.format(i),
value,
data_dict,
omit_dot=True)
else:
if prefix in data_dict:
raise DeepomaticException("Duplicate key: " + prefix)
data_dict[prefix] = obj
def get_base_uri(self, pk, **kwargs):
raise DeepomaticException('Unimplemented')
def make_request(self,
func,
resource,
params=None,
data=None,
content_type='application/json',
files=None,
stream=False,
*args,
**kwargs):
if content_type is not None:
if content_type.strip() == 'application/json':
if data is not None:
data = json.dumps(data)
elif content_type.strip() == 'multipart/mixed':
# If no files are provided, requests will default to form-urlencoded content type
# But the API doesn't support it.
if not files:
raise DeepomaticException(
"Cannot send the request as multipart without files provided."
)
# requests will build the good multipart content types with the boundaries
content_type = None
data = self.dump_json_for_multipart(data)
files = self.dump_json_for_multipart(files)
else:
raise DeepomaticException("Unsupported Content-Type")
headers = self.setup_headers(content_type=content_type)
params = self.format_params(params)
opened_files = []
if files is not None:
new_files = {}
for key, file in files.items():
if isinstance(file, string_types):
try:
# this may raise if file is a string containing bytes
if os.path.exists(file):
file = open(file, 'rb')
opened_files.append(file)
except TypeError:
pass
elif hasattr(file, 'seek'):
file.seek(0)
if hasattr(file, 'read') or \
isinstance(file, bytes) or \
(isinstance(file, tuple) and len(file) == 3):
new_files[key] = file
else:
new_files[key] = (None, file, 'application/json')
files = new_files
if not resource.startswith('http'):
resource = self.resource_prefix + resource
response = self.maybe_retry_send_request(func,
resource,
*args,
params=params,
data=data,
files=files,
headers=headers,
stream=stream,
**kwargs)
# Close opened files
for file in opened_files:
file.close()
status_code = response.status_code
if status_code == 204: # delete
return None
if status_code < 200 or status_code >= 300:
if status_code >= 400 and status_code < 500:
raise ClientError(response)
elif status_code >= 500 and status_code < 600:
raise ServerError(response)
else:
raise BadStatus(response)
if stream:
# we asked for a stream, we let the user download it as he wants or it will load everything in RAM
# not good for big files
return response
elif 'application/json' in response.headers.get('Content-Type', ''):
return response.json()
else:
return response.content