def send_request(input_data, url, headers, kwargs):
"""
Use the requests library to send of an HTTP call to the indico servers
"""
data = {}
if input_data != None:
data['data'] = input_data
data.update(**kwargs)
json_data = json.dumps(data)
response = requests.post(url, data=json_data, headers=headers)
warning = response.headers.get('x-warning')
if warning:
warnings.warn(warning)
cloud = urlparse(url).hostname
if response.status_code == 503 and not cloud.endswith('.indico.io'):
raise IndicoError("Private cloud '%s' does not include api '%s'" %
(cloud, api))
json_results = response.json()
results = json_results.get('results', False)
if results is False:
error = json_results.get('error')
raise IndicoError(error)
return results
def api_handler(arg, cloud, api, url_params=None, **kwargs):
"""
Sends finalized request data to ML server and receives response.
"""
if type(arg) == bytes:
arg = arg.decode('utf-8')
if type(arg) == list:
arg = [a.decode('utf-8') if type(arg) == bytes else a for a in arg]
data = {'data': arg}
data.update(**kwargs)
json_data = json.dumps(data)
cloud = cloud or config.cloud
host = "%s.indico.domains" % cloud if cloud else config.PUBLIC_API_HOST
url = create_url(host, api, dict(kwargs, **url_params))
response = requests.post(url, data=json_data, headers=JSON_HEADERS)
if response.status_code == 503 and cloud != None:
raise IndicoError("Private cloud '%s' does not include api '%s'" %
(cloud, api))
json_results = response.json()
results = json_results.get('results', False)
if results is False:
error = json_results.get('error')
raise IndicoError(error)
return results
def data_preprocess(data, size=None, min_axis=None, batch=False):
"""
Takes data and prepares it for sending to the api including
resizing and image data/structure standardizing.
"""
if batch:
return [
data_preprocess(el, size=size, min_axis=min_axis, batch=False)
for el in data
]
if isinstance(data, string_types):
if file_exists(data):
# probably a path to an image
preprocessed = Image.open(data)
else:
# base 64 encoded image data, a url, or raw content
# send raw data to the server and let the server infer type
b64_or_url = re.sub('^data:image/.+;base64,', '', data)
return b64_or_url
elif isinstance(data, Image.Image):
# data is image from PIL
preprocessed = data
elif type(data).__name__ == "ndarray":
# data is likely image from numpy/scipy
if "float" in str(data.dtype) and data.min() >= 0 and data.max() <= 1:
data *= 255.
try:
preprocessed = Image.fromarray(data.astype("uint8"))
except TypeError as e:
raise IndicoError(
"Please ensure the numpy array is in a format by PIL. "
"Values must be between 0 and 1 or between 0 and 255 in greyscale, rgb, or rgba format."
)
else:
# at this point we are unsure of the type -- it could be malformatted text or image data.
raise IndicoError(
"Invalid input datatype: `{}`. "
"Ensure input data is one of the following types: "
"`str`, `unicode`, `PIL.Image`, `np.ndarray`.".format(
data.__class__.__name__))
#
if size or min_axis:
preprocessed = resize_image(preprocessed, size, min_axis)
# standardize on b64 encoding for sending image data over the wire
temp_output = BytesIO()
preprocessed.save(temp_output, format='PNG')
temp_output.seek(0)
output_s = temp_output.read()
return base64.b64encode(output_s).decode(
'utf-8') if PY3 else base64.b64encode(output_s)
def image_preprocess(image, size=None, min_axis=None, batch=False):
"""
Takes an image and prepares it for sending to the api including
resizing and image data/structure standardizing.
"""
if batch:
return [image_preprocess(img, batch=False) for img in image]
if isinstance(image, string_types):
b64_str = re.sub('^data:image/.+;base64,', '', image)
if os.path.isfile(image):
# check type of element
out_image = Image.open(image)
elif B64_PATTERN.match(b64_str) is not None:
return b64_str
else:
raise IndicoError(
"String provided must be a valid filepath or base64 encoded string"
)
elif isinstance(image, Image.Image):
out_image = image
elif type(image).__name__ == "ndarray": # image is from numpy/scipy
if "float" in str(
image.dtype) and image.min() >= 0 and image.max() <= 1:
image *= 255.
try:
out_image = Image.fromarray(image.astype("uint8"))
except TypeError as e:
raise IndicoError(
"Please ensure the numpy array is acceptable by PIL. Values must be between 0 and 1 or between 0 and 255 in greyscale, rgb, or rgba format."
)
else:
raise IndicoError(
"Image must be a filepath, base64 encoded string, or a numpy array"
)
if size or min_axis:
out_image = resize_image(out_image, size, min_axis)
# convert to base64
temp_output = BytesIO()
out_image.save(temp_output, format='PNG')
temp_output.seek(0)
output_s = temp_output.read()
return base64.b64encode(output_s).decode(
'utf-8') if PY3 else base64.b64encode(output_s)
def collect_api_results(input_data, url, headers, api, batch_size, kwargs):
"""
Optionally split up a single request into a series of requests
to ensure timely HTTP responses.
Could eventually speed up the time required to receive a response by
sending batches to the indico API concurrently
"""
if batch_size:
results = []
for batch in batched(input_data, size=batch_size):
try:
results.extend(send_request(batch, url, headers, kwargs))
except IndicoError as e:
# Log results so far to file
timestamp = datetime.datetime.now().strftime(
'%Y-%m-%d-%H:%M:%S')
filename = "indico-{api}-{timestamp}.json".format(
api=api, timestamp=timestamp)
json.dump(results, open(filename, 'wb'))
raise IndicoError(
"The following error occurred while processing your data: `{err}` "
"Partial results have been saved to {filename}".format(
err=e, filename=os.path.abspath(filename)))
return results
else:
return send_request(input_data, url, headers, kwargs)
def multi(data, datatype, apis, batch=False, **kwargs):
"""
Helper to make multi requests of different types.
:param data: Data to be sent in API request
:param datatype: String type of API request
:param apis: List of apis to use.
:param batch: Is this a batch request?
:rtype: Dictionary of api responses
"""
# Client side api name checking - strictly only accept func name api
available = AVAILABLE_APIS.get(datatype)
invalid_apis = [api for api in apis if api not in available]
if invalid_apis:
raise IndicoError(
"%s are not valid %s APIs. Please reference the available APIs below:\n%s"
% (", ".join(invalid_apis), datatype, ", ".join(available)))
# Convert client api names to server names before sending request
cloud = kwargs.pop("cloud", None)
api_key = kwargs.pop('api_key', None)
result = api_handler(data,
cloud=cloud,
api='apis/multiapi',
url_params={
"apis": apis,
"batch": batch,
"api_key": api_key
},
**kwargs)
return handle_response(result)
def parsed_response(api, response):
result = response.get('results', False)
if result != False:
return result
raise IndicoError(
"Sorry, the %s API returned an unexpected response.\n\t%s" %
(api, response.get('error', "")))
def get(self, api_key=None, user_id=None, cloud=None):
if self.block:
start_time = time.time()
while True:
status = self.status(api_key=api_key, user_id=user_id)
if status in {"SUCCESS", "FAILURE", "REVOKED"}:
break
if self.timeout and time.time() - start_time > self.timeout:
raise IndicoError(
"JobResult didn't finish in provided timeout {timeout}"
.format(timeout=self.timeout))
time.sleep(2)
user_id = user_id or self.user_id
api_key = api_key or self.api_key
return api_handler(
None,
cloud=cloud,
api="async",
url_params={
"method": self.task_id,
"api_key": api_key
},
user_id=user_id,
)
def api_handler(arg, cloud, api, url_params=None, **kwargs):
"""
Sends finalized request data to ML server and receives response.
"""
url_params = url_params or {}
if type(arg) == bytes:
arg = arg.decode('utf-8')
if type(arg) == list:
arg = [a.decode('utf-8') if type(arg) == bytes else a for a in arg]
cloud = cloud or config.cloud
host = "%s.indico.domains" % cloud if cloud else config.host
# LOCAL DEPLOYMENTS
if not (host.endswith('indico.domains') or host.endswith('indico.io')):
url_protocol = "http"
else:
url_protocol = config.url_protocol
headers = dict(JSON_HEADERS)
headers["X-ApiKey"] = url_params.get("api_key") or config.api_key
url = create_url(url_protocol, host, api, dict(kwargs, **url_params))
data = {}
if arg != None:
data['data'] = arg
data.update(**kwargs)
json_data = json.dumps(data)
response = requests.post(url, data=json_data, headers=headers)
warning = response.headers.get('x-warning')
if warning:
warnings.warn(warning)
if response.status_code == 503 and cloud != None:
raise IndicoError("Private cloud '%s' does not include api '%s'" %
(cloud, api))
json_results = response.json()
results = json_results.get('results', False)
if results is False:
error = json_results.get('error')
raise IndicoError(error)
return results
def wait(self, interval=1):
"""
Block until the collection's model is completed training
"""
while True:
status = self.info().get('status')
if status == "ready":
break
if status != "training":
raise IndicoError(
"Collection status failed with: {0}".format(status))
time.sleep(interval)
def intersections(data, apis=None, **kwargs):
"""
Helper to make multi requests of different types.
:param data: Data to be sent in API request
:param type: String type of API request
:rtype: Dictionary of api responses
"""
# Client side api name checking
for api in apis:
assert api not in MULTIAPI_NOT_SUPPORTED
# remove auto-inserted batch param
kwargs.pop('batch', None)
if not isinstance(apis, list) or len(apis) != 2:
raise IndicoError("Argument 'apis' must be of length 2")
if isinstance(data, list) and len(data) < 3:
raise IndicoError(
"At least 3 examples are required to use the intersections API")
api_types = list(map(API_TYPES.get, apis))
if api_types[0] != api_types[1]:
raise IndicoError(
"Both `apis` must accept the same kind of input to use the intersections API"
)
cloud = kwargs.pop("cloud", None)
url_params = {
'batch': False,
'api_key': kwargs.pop('api_key', None),
'apis': apis
}
return api_handler(data,
cloud=cloud,
api="apis/intersections",
url_params=url_params,
**kwargs)
def test_assess_message_score_raise_exception(self, mock_indicoio,
mock_rough_score):
mock_indicoio.sentiment.side_effect = IndicoError('Boom!')
mock_rough_score.return_value = 0.67890
# make sure update_id has not been used in previous tests,
# as Redis dict is a persistent in memory dict
update = TelegramUpdates.TEXT_OK_ID_OK_TEXT
update['update_id'] -= 10
parsed_update = parse_update(update)
result = assess_message_score.delay(parsed_update).get(timeout=5)
self.assertNotEqual(result['score'], 0.87654321)
self.assertEqual(result['score'], 0.67890)
self.assertEqual(mock_indicoio.config.api_key,
current_app.config['INDICO_TOKEN'])
mock_indicoio.sentiment.assert_called_with(parsed_update['text'],
language='latin')