def _download_model(self, location: dict):
if not location:
return None
model = ModelWrapper.unwrap(location, self.context.model_serializer())
if model.blob:
#Embedded model
model = model.blob
else:
xsum = model.wrapping['xsum']
if not xsum:
raise fflabc.MalformedResponseException(f"Malformed wrapping (xsum): {model.wrapping}")
#Download from bin store
url = model.wrapping.get('url', None)
if not url:
raise fflabc.MalformedResponseException(f"Malformed wrapping: {model.wrapping}")
#Download from bin store
if self.context.download_models():
with rabbitmq.RabbitHeartbeat([self.subscriber, self.publisher]):
buff = requests.get(url)
#Now compare checksums
xsum2 = model.xsum(buff.content)
if xsum != xsum2:
raise fflabc.MalformedResponseException(f"Checksum mismatch!")
model = self.context.model_serializer().deserialize(buff.content)
else:
#Let user decide what to do
model = model.wrapping
return model
def _dispatch_model(self, task_name: str = None, model: dict = None) -> dict:
"""
Dispatch a model and determine its download location.
Throws: An exception on failure
:param model: model to be sent
:type model: `dict`
:return: download location information
:rtype: `dict`
"""
wrapper = ModelWrapper.wrap(model, self.context.model_serializer())
if not model:
return wrapper.wrapping
# First, obtain the upload location/keys
if task_name:
message = self.catalog.msg_bin_upload_object(task_name)
else:
if len(wrapper.blob) > self.context.dispatch_threshold():
message = self.catalog.msg_bin_uploader()
else:
#Small model - embed it
return wrapper.wrapping
upload_info = self._invoke_service(message)
if 'key' not in upload_info['fields']:
raise fflabc.MalformedResponseException('Update Error: Malformed URL')
key = upload_info['fields']['key']
try:
with rabbitmq.RabbitHeartbeat(self.subscriber):
# And then perform the upload
response = requests.post(upload_info['url'],
files={'file': wrapper.blob},
data=upload_info['fields'],
headers=None)
response.raise_for_status()
except requests.exceptions.RequestException as err:
raise fflabc.DispatchException(err) from err
except:
raise fflabc.DispatchException(f'General Update Error')
# Now obtain the download location/keys
if task_name:
message = self.catalog.msg_bin_download_object(key)
else:
message = self.catalog.msg_bin_downloader(key)
download_info = self._invoke_service(message)
wrapper = ModelWrapper.wrap({'url': download_info, 'key': key})
return wrapper.wrapping
def _dispatch_model(self, task_name: str = None, model: dict = None) -> dict:
"""
Dispatch a model and determine its download location.
Throws: An exception on failure
:param model: model to be sent
:type model: `dict`
:return: download location information
:rtype: `dict`
"""
if not model:
return {}
blob = self.context.serializer().serialize(model)
# First, obtain the upload location/keys
if task_name:
message = self.catalog.msg_bin_upload_object(task_name)
else:
if len(blob) > self.context.dispatch_threshold():
message = self.catalog.msg_bin_uploader()
else:
return model
upload_info = self._invoke_service(message)
if 'key' not in upload_info['fields']:
raise Exception('Update Error: Malformed URL.')
try:
with rabbitmq.RabbitHeartbeat(self.subscriber):
# And then perform the upload
response = requests.post(upload_info['url'],
files={'file': blob},
data=upload_info['fields'],
headers=None)
response.raise_for_status()
except requests.exceptions.RequestException as err:
raise Exception(f'Update Error: {err.response.status_code}')
except:
raise Exception(f'General Update Error')
# Now obtain the download location/keys
if task_name:
message = self.catalog.msg_bin_download_object(upload_info['fields']['key'])
else:
message = self.catalog.msg_bin_downloader(upload_info['fields']['key'])
download_info = self._invoke_service(message)
return {'url': download_info, 'key': upload_info['fields']['key']}
def _dispatch_model(self, task_name: str = None, model: dict = None) -> dict:
"""
Dispatch a model and determine its download location.
Throws: An exception on failure
:param model: model to be sent
:type model: `dict`
:return: download location information
:rtype: `dict`
"""
wrapper = ModelWrapper.wrap(model, self.context.model_serializer())
if not model:
return wrapper.wrapping
# First, obtain the upload location/keys
if task_name:
message = self.catalog.msg_bin_upload_object(task_name)
else:
if len(wrapper.blob) > self.context.dispatch_threshold():
message = self.catalog.msg_bin_uploader()
else:
#Small model - embed it
return wrapper.wrapping
upload_info = self._invoke_service(message)
if 'key' not in upload_info['fields']:
raise fflabc.MalformedResponseException('Update Error: Malformed URL')
key = upload_info['fields']['key']
try:
with rabbitmq.RabbitHeartbeat([self.subscriber, self.publisher]):
# And then perform the upload
if len(wrapper.blob) > self.context.model_threshold():
self._post_big_model(wrapper.blob, upload_info)
else:
self._post_model(wrapper.blob, upload_info)
except (requests.exceptions.RequestException, Exception) as err:
raise fflabc.DispatchException(err) from err
#Now add the checksum to the inline message
if isinstance(model, bytes):
xsum = model.xsum(model)
else:
xsum = wrapper.xsum()
wrapper = ModelWrapper.wrap({'key': key, 'xsum': xsum})
return wrapper.wrapping
def _dispatch_model(self, model: dict = None) -> dict:
"""
Dispatch a model and determine its download location.
Throws: An exception on failure
:param model: model to be sent
:type model: `dict`
:return: download location information
:rtype: `dict`
"""
if not model:
return {}
# First, obtain the upload location/keys
message = self.catalog.msg_bin_uploader()
upload_info = self._invoke_service(message)
if 'key' not in upload_info['fields']:
raise Exception('Update Error: Malformed URL.')
try:
with rabbitmq.RabbitHeartbeat(self.subscriber):
# And then perform the upload
response = requests.post(upload_info['url'],
files={'file': json.dumps(model)},
data=upload_info['fields'],
headers=None)
response.raise_for_status()
except requests.exceptions.RequestException as err:
raise Exception(f'Update Error: {err.response.status_code}')
except:
raise Exception(f'General Update Error')
# Now obtain the download location/keys
message = self.catalog.msg_bin_downloader(upload_info['fields']['key'])
download_info = self._invoke_service(message)
return {'url': download_info}