def _process_input(multidict):
kwargs2 = {}
for (arg, value) in multidict.iterlists():
if len(value) > 1:
raise WebException('Cannot specify a parameter more than once: %r' % (arg, ))
value = str(value[0])
if ',' in value and '[' not in value and ']' not in value:
value = '[%s]' % (value, )
if value in ['True', 'False']:
value = value.lower()
try:
converted = ut.from_json(value)
except Exception:
# try making string and try again...
value = '"%s"' % (value, )
converted = ut.from_json(value)
if arg.endswith('_list') and not isinstance(converted, (list, tuple)):
if isinstance(converted, str) and ',' in converted:
converted = converted.strip().split(',')
else:
converted = [converted]
# Allow JSON formatted strings to be placed into note fields
if ((arg.endswith('note_list') or arg.endswith('notes_list')) and
isinstance(converted, (list, tuple))):
type_ = type(converted)
temp_list = []
for _ in converted:
if isinstance(_, dict):
temp_list.append('%s' % (_, ))
else:
temp_list.append(_)
converted = type_(temp_list)
kwargs2[arg] = converted
return kwargs2
def _process_input(multidict=None):
if multidict is None:
return {}
if isinstance(multidict, dict):
from werkzeug.datastructures import ImmutableMultiDict
multidict = ImmutableMultiDict([item for item in multidict.items()])
kwargs2 = {}
for (arg, value) in multidict.lists():
if len(value) > 1:
raise WebException('Cannot specify a parameter more than once: %r' % (arg,))
# value = str(value[0])
value = value[0]
if (
',' in value
and '[' not in value
and ']' not in value
and '{' not in value
and '}' not in value
):
value = '[%s]' % (value,)
if value in ['True', 'False']:
value = value.lower()
try:
converted = ut.from_json(value)
except Exception:
# try making string and try again...
try:
value_ = '"%s"' % (value,)
converted = ut.from_json(value_)
except Exception as ex:
logger.info('FAILED TO JSON CONVERT: %s' % (ex,))
logger.info(ut.repr3(value))
converted = value
if arg.endswith('_list') and not isinstance(converted, (list, tuple)):
if isinstance(converted, str) and ',' in converted:
converted = converted.strip().split(',')
else:
converted = [converted]
# Allow JSON formatted strings to be placed into note fields
if (arg.endswith('note_list') or arg.endswith('notes_list')) and isinstance(
converted, (list, tuple)
):
type_ = type(converted)
temp_list = []
for _ in converted:
if isinstance(_, dict):
temp_list.append('%s' % (_,))
else:
temp_list.append(_)
converted = type_(temp_list)
kwargs2[arg] = converted
return kwargs2
def send_ibeis_request(suffix, type_='post', **kwargs):
"""
Posts a request to a url suffix
"""
import requests
import utool as ut
payload = ut.map_dict_vals(ut.to_json, kwargs)
if type_ == 'post':
resp = requests.post(baseurl + suffix, data=payload)
content = ut.from_json(resp._content)
elif type_ == 'get':
resp = requests.get(baseurl + suffix, data=payload)
content = ut.from_json(resp.content)
response = content['response']
return response
def get_flukebook_image_uuids(ibs):
from datetime import datetime
import uuid
import pytz
PST = pytz.timezone('US/Pacific')
url = 'https://www.flukebook.org/acmIdSync.jsp'
now = datetime.now(tz=PST)
timestamp = now.strftime('%Y-%m-%d-%H-00-00')
filename = 'flukebook.image.admid.%s.json' % (timestamp, )
filepath = ut.grab_file_url(url, appname='wbia', fname=filename)
with open(filepath, 'r') as file:
file_content = file.read()
file_json = ut.from_json(file_content)
logger.info('Loaded %d Image ACM string UUIDs from Flukebook' %
(len(file_json), ))
uuid_list = []
for uuid_str in file_json:
try:
uuid_ = uuid.UUID(uuid_str)
uuid_list.append(uuid_)
except ValueError:
continue
logger.info('Validated %d Image UUIDs from Flukebook' % (len(uuid_list), ))
flukebook_image_uuid_list = list(set(uuid_list))
logger.info('Validated %d de-duplicated Image UUIDs from Flukebook' %
(len(uuid_list), ))
return flukebook_image_uuid_list
def send_wbia_request(suffix, type_='post', json=True, **kwargs):
"""
Posts a request to a url suffix
"""
import requests
import utool as ut
if not suffix.endswith('/'):
raise Exception('YOU PROBABLY WANT A / AT THE END OF YOUR URL')
payload = ut.map_dict_vals(ut.to_json, kwargs)
if type_ == 'post':
resp = requests.post(baseurl + suffix, data=payload)
content = resp._content
elif type_ == 'get':
resp = requests.get(baseurl + suffix, data=payload)
content = resp.content
if json:
try:
content = ut.from_json(content)
except ValueError:
raise Exception('Expected JSON string but got content=%r' %
(content, ))
else:
# logger.info('content = %r' % (content,))
if content['status']['code'] != 200:
logger.info(content['status']['message'])
raise Exception(content['status']['message'])
content = content['response']
return content
def translated_call(**kwargs):
#debug = {'kwargs': kwargs}
try:
# Pipe web input into Python web call
kwargs2 = _process_input(flask.request.args)
kwargs3 = _process_input(flask.request.form)
try:
# kwargs4 = _process_input(flask.request.get_json())
kwargs4 = ut.from_json(flask.request.data)
except Exception:
kwargs4 = {}
kwargs.update(kwargs2)
kwargs.update(kwargs3)
kwargs.update(kwargs4)
jQuery_callback = None
if 'callback' in kwargs and 'jQuery' in kwargs['callback']:
jQuery_callback = str(kwargs.pop('callback', None))
kwargs.pop('_', None)
args = ()
print('Processing: %r with args: %r and kwargs: %r' % (func, args, kwargs, ))
result = func(**kwargs)
except Exception as ex:
rawreturn = str(traceback.format_exc())
success = False
code = 400
message = (
'Route error, Python Exception thrown: %r' %
(str(ex), ))
jQuery_callback = None
result = translate_ibeis_webreturn(rawreturn, success,
code, message,
jQuery_callback)
return result
def send_ibeis_request(suffix, type_='post', **kwargs):
"""
Posts a request to a url suffix
"""
import requests
import utool as ut
if not suffix.endswith('/'):
raise Exception('YOU PROBABLY WANT A / AT THE END OF YOUR URL')
payload = ut.map_dict_vals(ut.to_json, kwargs)
if type_ == 'post':
resp = requests.post(baseurl + suffix, data=payload)
json_content = resp._content
elif type_ == 'get':
resp = requests.get(baseurl + suffix, data=payload)
json_content = resp.content
try:
content = ut.from_json(json_content)
except ValueError:
raise Exception('Expected JSON string but got json_content=%r' % (json_content,))
else:
# print('content = %r' % (content,))
if content['status']['code'] != 200:
print(content['status']['message'])
raise Exception(content['status']['message'])
request_response = content['response']
return request_response
def api_remote_wbia(remote_wbia_url, remote_api_func, remote_wbia_port=5001, **kwargs):
import requests
if GLOBAL_APP_ENABLED and GLOBAL_APP is None:
raise ValueError('Flask has not been initialized')
api_name = remote_api_func.__name__
route_list = list(GLOBAL_APP.url_map.iter_rules(api_name))
assert len(route_list) == 1, 'More than one route resolved'
route = route_list[0]
api_route = route.rule
assert api_route.startswith('/api/'), 'Must be an API route'
method_list = sorted(list(route.methods - set(['HEAD', 'OPTIONS'])))
remote_api_method = method_list[0].upper()
assert api_route is not None, 'Route could not be found'
args = (remote_wbia_url, remote_wbia_port, api_route)
remote_api_url = 'http://%s:%s%s' % args
headers = {'Authorization': get_url_authorization(remote_api_url)}
for key in kwargs.keys():
value = kwargs[key]
if isinstance(value, (tuple, list, set)):
value = str(list(value))
kwargs[key] = value
logger.info('[REMOTE] %s' % ('-' * 80,))
logger.info('[REMOTE] Calling remote IBEIS API: %r' % (remote_api_url,))
logger.info('[REMOTE] \tMethod: %r' % (remote_api_method,))
if ut.DEBUG2 or ut.VERBOSE:
logger.info('[REMOTE] \tHeaders: %s' % (ut.repr2(headers),))
logger.info('[REMOTE] \tKWArgs: %s' % (ut.repr2(kwargs),))
# Make request to server
try:
if remote_api_method == 'GET':
req = requests.get(remote_api_url, headers=headers, data=kwargs, verify=False)
elif remote_api_method == 'POST':
req = requests.post(
remote_api_url, headers=headers, data=kwargs, verify=False
)
elif remote_api_method == 'PUT':
req = requests.put(remote_api_url, headers=headers, data=kwargs, verify=False)
elif remote_api_method == 'DELETE':
req = requests.delete(
remote_api_url, headers=headers, data=kwargs, verify=False
)
else:
message = '_api_result got unsupported method=%r' % (remote_api_method,)
raise KeyError(message)
except requests.exceptions.ConnectionError as ex:
message = '_api_result could not connect to server %s' % (ex,)
raise IOError(message)
response = req.text
converted = ut.from_json(value)
response = converted.get('response', None)
logger.info('[REMOTE] got response')
if ut.DEBUG2:
logger.info('response = %s' % (response,))
return response
def get_web_port_via_scan(ibs,
url_base='127.0.0.1',
port_base=5000,
scan_limit=100,
verbose=True):
import requests
api_rule = CORE_DB_UUID_INIT_API_RULE
target_uuid = ibs.get_db_init_uuid()
for candidate_port in range(port_base, port_base + scan_limit + 1):
candidate_url = 'http://%s:%s%s' % (url_base, candidate_port,
api_rule)
try:
response = requests.get(candidate_url)
except (requests.ConnectionError):
if verbose:
logger.info('Failed to find IA server at %s' %
(candidate_url, ))
continue
logger.info('Found IA server at %s' % (candidate_url, ))
try:
response = ut.from_json(response.text)
candidate_uuid = response.get('response')
assert candidate_uuid == target_uuid
return candidate_port
except (AssertionError):
if verbose:
logger.info('Invalid response from IA server at %s' %
(candidate_url, ))
continue
return None
def api_remote_ibeis(remote_ibeis_url, remote_api_func, remote_ibeis_port=5001,
**kwargs):
import requests
if GLOBAL_APP_ENABLED and GLOBAL_APP is None:
raise ValueError('Flask has not been initialized')
api_name = remote_api_func.__name__
route_list = list(GLOBAL_APP.url_map.iter_rules(api_name))
assert len(route_list) == 1, 'More than one route resolved'
route = route_list[0]
api_route = route.rule
assert api_route.startswith('/api/'), 'Must be an API route'
method_list = sorted(list(route.methods - set(['HEAD', 'OPTIONS'])))
remote_api_method = method_list[0].upper()
assert api_route is not None, 'Route could not be found'
args = (remote_ibeis_url, remote_ibeis_port, api_route)
remote_api_url = 'http://%s:%s%s' % args
headers = {
'Authorization': get_url_authorization(remote_api_url)
}
for key in kwargs.keys():
value = kwargs[key]
if isinstance(value, (tuple, list, set)):
value = str(list(value))
kwargs[key] = value
print('[REMOTE] %s' % ('-' * 80, ))
print('[REMOTE] Calling remote IBEIS API: %r' % (remote_api_url, ))
print('[REMOTE] \tMethod: %r' % (remote_api_method, ))
print('[REMOTE] \tHeaders: %s' % (ut.dict_str(headers), ))
print('[REMOTE] \tKWArgs: %s' % (ut.dict_str(kwargs), ))
# Make request to server
try:
if remote_api_method == 'GET':
req = requests.get(remote_api_url, headers=headers, data=kwargs,
verify=False)
elif remote_api_method == 'POST':
req = requests.post(remote_api_url, headers=headers, data=kwargs,
verify=False)
elif remote_api_method == 'PUT':
req = requests.put(remote_api_url, headers=headers, data=kwargs,
verify=False)
elif remote_api_method == 'DELETE':
req = requests.delete(remote_api_url, headers=headers, data=kwargs,
verify=False)
else:
message = '_api_result got unsupported method=%r' % (remote_api_method, )
raise KeyError(message)
except requests.exceptions.ConnectionError as ex:
message = '_api_result could not connect to server %s' % (ex, )
raise IOError(message)
response = req.text
converted = ut.from_json(value)
response = converted.get('response', None)
print('response = %s' % (response,))
return response
def _verify_response_ibs(ibs, response):
try:
response_dict = ut.from_json(response.text)
except ValueError:
raise AssertionError('Could not get valid JSON response from server')
status = response_dict.get('status', {})
assert status.get('success', False)
response = response_dict.get('response', None)
return response
def get_unpacked_result(jobiface, jobid):
reply = jobiface.get_job_result(jobid)
json_result = reply['json_result']
try:
result = ut.from_json(json_result)
except TypeError as ex:
ut.printex(ex, keys=['json_result'], iswarning=True)
result = json_result
# raise
#print('Job %r result = %s' % (jobid, ut.repr2(result, truncate=True),))
return result
def get_unpacked_result(jobiface, jobid):
reply = jobiface.get_job_result(jobid)
json_result = reply['json_result']
try:
result = ut.from_json(json_result)
except TypeError as ex:
ut.printex(ex, keys=['json_result'], iswarning=True)
result = json_result
# raise
#print('Job %r result = %s' % (jobid, ut.repr2(result, truncate=True),))
return result
def rcv_multipart_json(sock, num=2, print=print):
""" helper """
# note that the first two parts will be ['Controller.ROUTER', 'Client.<id_>']
# these are needed for the reply to propagate up to the right client
multi_msg = sock.recv_multipart()
if VERBOSE_JOBS:
print('----')
print('RCV Json: %s' % (ut.repr2(multi_msg, truncate=True),))
idents = multi_msg[:num]
request_json = multi_msg[num]
request = ut.from_json(request_json)
return idents, request
def rcv_multipart_json(sock, num=2, print=print):
""" helper """
# note that the first two parts will be ['Controller.ROUTER', 'Client.<id_>']
# these are needed for the reply to propagate up to the right client
multi_msg = sock.recv_multipart()
if VERBOSE_JOBS:
print('----')
print('RCV Json: %s' % (ut.repr2(multi_msg, truncate=True), ))
idents = multi_msg[:num]
request_json = multi_msg[num]
request = ut.from_json(request_json)
return idents, request
def log_detections(ibs, aid_list, fallback=True):
import time
import os
json_log_path = ibs.get_logdir_local()
json_log_filename = 'detections.json'
json_log_filepath = os.path.join(json_log_path, json_log_filename)
logger.info('Logging detections added to: %r' % (json_log_filepath,))
try:
# Log has never been made, create one
if not os.path.exists(json_log_filepath):
json_dict = {
'updates': [],
}
json_str = ut.to_json(json_dict, pretty=True)
with open(json_log_filepath, 'w') as json_log_file:
json_log_file.write(json_str)
# Get current log state
with open(json_log_filepath, 'r') as json_log_file:
json_str = json_log_file.read()
json_dict = ut.from_json(json_str)
# Get values
db_name = ibs.get_db_name()
db_init_uuid = ibs.get_db_init_uuid()
# Zip all the updates together and write to updates list in dictionary
gid_list = ibs.get_annot_gids(aid_list)
bbox_list = ibs.get_annot_bboxes(aid_list)
theta_list = ibs.get_annot_thetas(aid_list)
zipped = list(zip(aid_list, gid_list, bbox_list, theta_list))
for aid, gid, bbox, theta in zipped:
json_dict['updates'].append(
{
'time_unixtime': time.time(),
'db_name': db_name,
'db_init_uuid': db_init_uuid,
'image_rowid': gid,
'annot_rowid': aid,
'annot_bbox': bbox,
'annot_theta': theta,
}
)
# Write new log state
json_str = ut.to_json(json_dict, pretty=True)
with open(json_log_filepath, 'w') as json_log_file:
json_log_file.write(json_str)
except Exception:
if fallback:
logger.info('WRITE DETECTION.JSON FAILED - ATTEMPTING FALLBACK')
ut.delete(json_log_filepath)
ibs.log_detections(aid_list, fallback=False)
else:
logger.info('WRITE DETECTION.JSON FAILED - FALLBACK FAILED')
def translated_call(**kwargs):
# debug = {'kwargs': kwargs}
try:
# Pipe web input into Python web call
kwargs2 = _process_input(flask.request.args)
kwargs3 = _process_input(flask.request.form)
try:
# kwargs4 = _process_input(flask.request.get_json())
kwargs4 = ut.from_json(flask.request.data)
except Exception:
kwargs4 = {}
kwargs.update(kwargs2)
kwargs.update(kwargs3)
kwargs.update(kwargs4)
jQuery_callback = None
if 'callback' in kwargs and 'jQuery' in kwargs['callback']:
jQuery_callback = str(kwargs.pop('callback', None))
kwargs.pop('_', None)
args = ()
logger.info(
'Processing: %r with args: %r and kwargs: %r'
% (func, args, kwargs)
)
from wbia.web.app import PROMETHEUS
if PROMETHEUS:
ibs = flask.current_app.ibs
tag = request.url_rule.rule
ibs.prometheus_increment_route(tag)
result = func(**kwargs)
except Exception as ex:
rawreturn = str(traceback.format_exc())
success = False
code = 400
message = 'Route error, Python Exception thrown: %r' % (str(ex),)
jQuery_callback = None
result = translate_wbia_webreturn(
rawreturn,
success,
code,
message,
jQuery_callback,
__skip_microsoft_validation__=True,
)
return result
def get_part_staged_metadata(ibs, part_rowid_list, return_raw=False):
r"""
Returns:
list_ (list): part metadata dictionary
RESTful:
Method: GET
URL: /api/part/staged/metadata/
"""
metadata_str_list = ibs.db.get(const.PART_TABLE, ('part_staged_metadata_json',), part_rowid_list)
metadata_list = []
for metadata_str in metadata_str_list:
if metadata_str in [None, '']:
metadata_dict = {}
else:
metadata_dict = metadata_str if return_raw else ut.from_json(metadata_str)
metadata_list.append(metadata_dict)
return metadata_list
def get_review_metadata(ibs, review_rowid_list, return_raw=False):
r"""
Returns:
list_ (list): review metadata dictionary
RESTful:
Method: GET
URL: /api/review/metadata/
"""
metadata_str_list = ibs.staging.get(const.REVIEW_TABLE, ('review_metadata_json',), review_rowid_list)
metadata_list = []
for metadata_str in metadata_str_list:
if metadata_str in [None, '']:
metadata_dict = {}
else:
metadata_dict = metadata_str if return_raw else ut.from_json(metadata_str)
metadata_list.append(metadata_dict)
return metadata_list
def get_part_contour(ibs, part_rowid_list, return_raw=False):
r"""
Returns:
list_ (list): part contour dictionary
RESTful:
Method: GET
URL: /api/part/contour/
"""
contour_str_list = ibs.db.get(const.PART_TABLE, ('part_contour_json',), part_rowid_list)
contour_list = []
for contour_str in contour_str_list:
if contour_str in [None, '']:
contour_dict = {}
else:
contour_dict = contour_str if return_raw else ut.from_json(contour_str)
contour_list.append(contour_dict)
return contour_list
def _read_dict_from_sqlite3(blob):
return ut.from_json(blob)
def load_json(fpath):
import utool as ut
json_data = ut.load_text(fpath)
data = ut.from_json(json_data)
return data
def get_unpacked_result(jobiface, jobid):
reply = jobiface.get_job_result(jobid)
json_result = reply['json_result']
result = ut.from_json(json_result)
#print('Job %r result = %s' % (jobid, ut.repr2(result, truncate=True),))
return result
def get_wildbook_annot_uuids(ibs, filter_match_against_on=True):
from datetime import datetime
import uuid
import pytz
PST = pytz.timezone('US/Pacific')
assert WILDBOOK_TARGET is not None
url = 'https://%s/acmIdSync.jsp?annotations' % (WILDBOOK_TARGET, )
now = datetime.now(tz=PST)
timestamp = now.strftime('%Y-%m-%d-%H-00-00')
filename = 'wildbook.annot.admid.%s.json' % (timestamp, )
filepath = ut.grab_file_url(url, appname='wbia', fname=filename)
with open(filepath, 'r') as file:
file_content = file.read()
file_json = ut.from_json(file_content)
print('Loaded %d Annot ACM string UUIDs from Wildbook: %r' % (
len(file_json),
url,
))
# KEY = 'species'
SPECIES_KEY = 'iaClass'
bad_mapping = {
'None,whale_fluke': 'whale_fluke',
'dolphin_bottlenose+fin_dorsal,tursiops_truncatus':
'dolphin_bottlenose+fin_dorsal',
'dolphin_bottlenose_fin,tursiops_truncatus': 'dolphin_bottlenose_fin',
'dolphin_spotted,stenella_frontalis': 'dolphin_spotted',
'tursiops_truncatus,whale_orca+fin_dorsal': 'whale_orca+fin_dorsal',
'turtle_hawksbill,turtle_hawksbill+head': 'turtle_hawksbill+head',
}
bad_list = []
uuid_list = []
species_list = []
for uuid_str in file_json:
content = file_json[uuid_str]
try:
uuid_ = uuid.UUID(uuid_str)
except ValueError:
continue
match = content.get('match', None)
species = content.get(SPECIES_KEY, None)
assert None not in [match, species]
assert len(match) == len(species)
if len(match) > 1:
match_ = set(match)
species_ = set(species)
if len(match_) == 1 and len(species_) == 1:
match = list(match_)
species = list(species_)
assert len(match) == len(species)
if len(match) > 1:
match_ = set(match)
species_ = set(species)
if len(species_) == 1:
match = [any(match_)]
species = list(species_)
if len(match) == 1 and len(species) == 1:
match = match[0]
species = species[0]
if filter_match_against_on is None or match == filter_match_against_on:
uuid_list.append(uuid_)
species_list.append(species)
else:
species_ = ['None' if val is None else val for val in species]
species_ = set(species_)
species_ = sorted(species_)
bad_species = ','.join(species_)
fixed_species = bad_mapping.get(bad_species, None)
if fixed_species is not None:
if filter_match_against_on is None or match == filter_match_against_on:
uuid_list.append(uuid_)
species_list.append(fixed_species)
else:
print(match, species)
bad_list.append(bad_species)
if len(bad_list) > 0:
print('Found bad species: %r' % (set(bad_list), ))
assert len(uuid_list) == len(species_list)
assert len(uuid_list) == len(set(uuid_list))
print('Validated %d Annotation UUIDs from Wildbook' % (len(uuid_list), ))
wildbook_annot_uuid_list = uuid_list
wildbook_annot_species_list = species_list
return wildbook_annot_uuid_list, wildbook_annot_species_list
def _read_dict_from_sqlite3(blob):
return ut.from_json(blob)
def translated_call(**kwargs):
def html_newlines(text):
r = '<br />\n'
text = text.replace(' ', ' ')
text = (
text.replace('\r\n', r)
.replace('\n\r', r)
.replace('\r', r)
.replace('\n', r)
)
return text
__format__ = False # Default __format__ value
ignore_cookie_set = False
try:
# logger.info('Processing: %r with args: %r and kwargs: %r' % (func, args, kwargs, ))
# Pipe web input into Python web call
kwargs2 = _process_input(flask.request.args)
kwargs3 = _process_input(flask.request.form)
try:
# kwargs4 = _process_input(flask.request.get_json())
kwargs4 = ut.from_json(flask.request.data)
except Exception:
kwargs4 = {}
kwargs.update(kwargs2)
kwargs.update(kwargs3)
kwargs.update(kwargs4)
# Update the request object to include the final rectified inputs for possible future reference
flask.request.processed = ut.to_json(kwargs)
jQuery_callback = None
if 'callback' in kwargs and 'jQuery' in kwargs['callback']:
jQuery_callback = str(kwargs.pop('callback', None))
kwargs.pop('_', None)
# logger.info('KWARGS: %s' % (kwargs, ))
# logger.info('COOKIES: %s' % (request.cookies, ))
__format__ = request.cookies.get('__format__', None)
__format__ = kwargs.pop('__format__', __format__)
if __format__ is not None:
__format__ = str(__format__).lower()
ignore_cookie_set = __format__ in ['onetime', 'true']
__format__ = __format__ in ['true', 'enabled', 'enable']
from wbia.web.app import PROMETHEUS
if PROMETHEUS:
exclude_tag_list = [
'/api/test/heartbeat/',
'/v0.1/wildbook/status/',
'/v0.1/vulcan/status/',
]
tag = request.url_rule.rule
if tag not in exclude_tag_list:
ibs = flask.current_app.ibs
ibs.prometheus_increment_api(tag)
resp_tup = translate_wbia_webcall(func, **kwargs)
rawreturn, success, code, message = resp_tup
except WebException as webex:
# ut.printex(webex)
logger.info('CAUGHT2: %r' % (webex,))
rawreturn = webex.get_rawreturn(
DEBUG_PYTHON_STACK_TRACE_JSON_RESPONSE
)
success = False
code = webex.code
message = webex.message
jQuery_callback = None
except Exception as ex:
logger.info('CAUGHT2: %r' % (ex,))
# ut.printex(ex)
rawreturn = None
if DEBUG_PYTHON_STACK_TRACE_JSON_RESPONSE:
rawreturn = str(traceback.format_exc())
success = False
code = 500
message = str(ex)
# errmsg = str(ex)
# message = 'API error, Python Exception thrown: %s' % (errmsg)
if "'int' object is not iterable" in message:
rawreturn = """
HINT: the input for this call is most likely expected to be a list.
Try adding a comma at the end of the input (to cast the conversion into a list) or encapsulate the input with [].
"""
jQuery_callback = None
# logger.info('RECEIVED FORMAT: %r' % (__format__, ))
if __format__:
# Hack for readable error messages
webreturn = translate_wbia_webreturn(
rawreturn, success, code, message, jQuery_callback
)
webreturn = ut.repr3(ut.from_json(webreturn), strvals=True)
try:
from ansi2html import Ansi2HTMLConverter
conv = Ansi2HTMLConverter()
webreturn = conv.convert(webreturn)
except ImportError as ex:
ut.printex(ex, 'pip install ansi2html', iswarning=True)
webreturn = ut.strip_ansi(webreturn)
webreturn = (
'<p><samp>\n' + html_newlines(webreturn) + '\n</samp></p>'
)
webreturn = (
'<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-8">\n'
+ webreturn
)
def get_func_href(funcname):
url = (
'http://'
+ request.environ['HTTP_HOST']
+ flask.url_for(funcname)
+ '?__format__=True'
)
return '<a href="{url}">{url}</a>'.format(url=url)
if not success:
webreturn += (
'<pre>See logs for details: %s</pre>'
% get_func_href('get_current_log_text')
)
webreturn += (
'<pre>Might also look into db_info: %s</pre>'
% get_func_href('get_dbinfo')
)
else:
webreturn = translate_wbia_webreturn(
rawreturn, success, code, message, jQuery_callback
)
webreturn = ut.strip_ansi(webreturn)
resp = flask.make_response(webreturn, code)
resp.status_code = code
if not __format__:
resp.headers['Content-Type'] = 'application/json; charset=utf-8'
resp.headers['mimetype'] = 'application/json'
if not ignore_cookie_set:
if __format__:
resp.set_cookie('__format__', 'enabled')
else:
resp.set_cookie('__format__', '', expires=0)
return resp
def translated_call(**kwargs):
def html_newlines(text):
r = '<br />\n'
text = text.replace(' ', ' ')
text = text.replace('\r\n', r).replace('\n\r', r).replace('\r', r).replace('\n', r)
return text
__format__ = False # Default __format__ value
ignore_cookie_set = False
try:
#print('Processing: %r with args: %r and kwargs: %r' % (func, args, kwargs, ))
# Pipe web input into Python web call
kwargs2 = _process_input(flask.request.args)
kwargs3 = _process_input(flask.request.form)
try:
# kwargs4 = _process_input(flask.request.get_json())
kwargs4 = ut.from_json(flask.request.data)
except:
kwargs4 = {}
kwargs.update(kwargs2)
kwargs.update(kwargs3)
kwargs.update(kwargs4)
jQuery_callback = None
if 'callback' in kwargs and 'jQuery' in kwargs['callback']:
jQuery_callback = str(kwargs.pop('callback', None))
kwargs.pop('_', None)
#print('KWARGS: %s' % (kwargs, ))
#print('COOKIES: %s' % (request.cookies, ))
__format__ = request.cookies.get('__format__', None)
__format__ = kwargs.pop('__format__', __format__)
if __format__ is not None:
__format__ = str(__format__).lower()
ignore_cookie_set = __format__ in ['onetime', 'true']
__format__ = __format__ in ['true', 'enabled', 'enable']
resp_tup = translate_ibeis_webcall(func, **kwargs)
rawreturn, success, code, message = resp_tup
except WebException as webex:
ut.printex(webex)
rawreturn = webex.get_rawreturn(
DEBUG_PYTHON_STACK_TRACE_JSON_RESPONSE)
success = False
code = webex.code
message = webex.message
jQuery_callback = None
except Exception as ex:
ut.printex(ex)
rawreturn = ''
if DEBUG_PYTHON_STACK_TRACE_JSON_RESPONSE:
rawreturn = str(traceback.format_exc())
success = False
code = 500
errmsg = str(ex)
message = 'API error, Python Exception thrown: %s' % (errmsg)
if "'int' object is not iterable" in message:
rawreturn = (
'HINT: the input for this call is most likely '
'expected to be a list. Try adding a comma at '
'the end of the input (to cast the conversion '
'into a list) or encapsualte the input with '
'[].')
jQuery_callback = None
#print('RECEIVED FORMAT: %r' % (__format__, ))
if __format__:
# Hack for readable error messages
webreturn = translate_ibeis_webreturn(
rawreturn, success, code, message, jQuery_callback)
webreturn = ut.repr3(ut.from_json(webreturn), strvals=True)
try:
from ansi2html import Ansi2HTMLConverter
conv = Ansi2HTMLConverter()
webreturn = conv.convert(webreturn)
except ImportError as ex:
ut.printex(ex, 'pip install ansi2html', iswarning=True)
webreturn = ut.strip_ansi(webreturn)
webreturn = '<p><samp>\n' + html_newlines(webreturn) + '\n</samp></p>'
webreturn = '<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-8">\n' + webreturn
def get_func_href(funcname):
url = 'http://' + request.environ['HTTP_HOST'] + flask.url_for(funcname) + '?__format__=True'
return '<a href="{url}">{url}</a>'.format(url=url)
if not success:
webreturn += '<pre>See logs for details: %s</pre>' % get_func_href('get_current_log_text')
webreturn += '<pre>Might also look into db_info: %s</pre>' % get_func_href('get_dbinfo')
else:
webreturn = translate_ibeis_webreturn(
rawreturn, success, code, message, jQuery_callback)
webreturn = ut.strip_ansi(webreturn)
resp = flask.make_response(webreturn, code)
if not ignore_cookie_set:
if __format__:
resp.set_cookie('__format__', 'enabled')
else:
resp.set_cookie('__format__', '', expires=0)
return resp
def translated_call(**kwargs):
def html_newlines(text):
r = '<br />\n'
text = text.replace(' ', ' ')
text = text.replace('\r\n', r).replace('\n\r', r).replace('\r', r).replace('\n', r)
return text
__format__ = False # Default __format__ value
ignore_cookie_set = False
try:
#print('Processing: %r with args: %r and kwargs: %r' % (func, args, kwargs, ))
# Pipe web input into Python web call
kwargs2 = _process_input(flask.request.args)
kwargs3 = _process_input(flask.request.form)
kwargs.update(kwargs2)
kwargs.update(kwargs3)
jQuery_callback = None
if 'callback' in kwargs and 'jQuery' in kwargs['callback']:
jQuery_callback = str(kwargs.pop('callback', None))
kwargs.pop('_', None)
#print('KWARGS: %s' % (kwargs, ))
#print('COOKIES: %s' % (request.cookies, ))
__format__ = request.cookies.get('__format__', None)
__format__ = kwargs.pop('__format__', __format__)
if __format__ is not None:
__format__ = str(__format__).lower()
ignore_cookie_set = __format__ in ['onetime', 'true']
__format__ = __format__ in ['true', 'enabled', 'enable']
resp_tup = translate_ibeis_webcall(func, **kwargs)
rawreturn, success, code, message = resp_tup
except WebException as webex:
ut.printex(webex)
rawreturn = webex.get_rawreturn(
DEBUG_PYTHON_STACK_TRACE_JSON_RESPONSE)
success = False
code = webex.code
message = webex.message
jQuery_callback = None
except Exception as ex:
ut.printex(ex)
rawreturn = ''
if DEBUG_PYTHON_STACK_TRACE_JSON_RESPONSE:
rawreturn = str(traceback.format_exc())
success = False
code = 500
errmsg = str(ex)
message = 'API error, Python Exception thrown: %s' % (errmsg)
if "'int' object is not iterable" in message:
rawreturn = (
'HINT: the input for this call is most likely '
'expected to be a list. Try adding a comma at '
'the end of the input (to cast the conversion '
'into a list) or encapsualte the input with '
'[].')
jQuery_callback = None
#print('RECEIVED FORMAT: %r' % (__format__, ))
if __format__:
# Hack for readable error messages
webreturn = translate_ibeis_webreturn(
rawreturn, success, code, message, jQuery_callback)
webreturn = ut.repr3(ut.from_json(webreturn), strvals=True)
try:
from ansi2html import Ansi2HTMLConverter
conv = Ansi2HTMLConverter()
webreturn = conv.convert(webreturn)
except ImportError as ex:
ut.printex(ex, 'pip install ansi2html', iswarning=True)
webreturn = ut.strip_ansi(webreturn)
webreturn = '<p><samp>\n' + html_newlines(webreturn) + '\n</samp></p>'
webreturn = '<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-8">\n' + webreturn
def get_func_href(funcname):
url = 'http://' + request.environ['HTTP_HOST'] + flask.url_for(funcname) + '?__format__=True'
return '<a href="{url}">{url}</a>'.format(url=url)
if not success:
webreturn += '<pre>See logs for details: %s</pre>' % get_func_href('get_current_log_text')
webreturn += '<pre>Might also look into db_info: %s</pre>' % get_func_href('get_dbinfo')
else:
webreturn = translate_ibeis_webreturn(
rawreturn, success, code, message, jQuery_callback)
webreturn = ut.strip_ansi(webreturn)
resp = flask.make_response(webreturn, code)
if not ignore_cookie_set:
if __format__:
resp.set_cookie('__format__', 'enabled')
else:
resp.set_cookie('__format__', '', expires=0)
return resp
def get_flukebook_annot_uuids(ibs, filter_match_against_on=True):
from datetime import datetime
import uuid
import pytz
PST = pytz.timezone('US/Pacific')
url = 'https://www.flukebook.org/acmIdSync.jsp?annotations'
now = datetime.now(tz=PST)
timestamp = now.strftime('%Y-%m-%d-%H-00-00')
filename = 'flukebook.annot.admid.%s.json' % (timestamp, )
filepath = ut.grab_file_url(url, appname='wbia', fname=filename)
with open(filepath, 'r') as file:
file_content = file.read()
file_json = ut.from_json(file_content)
logger.info('Loaded %d Annot ACM string UUIDs from Flukebook' %
(len(file_json), ))
uuid_list = []
species_list = []
for uuid_str in file_json:
content = file_json[uuid_str]
try:
uuid_ = uuid.UUID(uuid_str)
except ValueError:
continue
match = content.get('match', None)
species = content.get('species', None)
assert None not in [match, species]
assert len(match) == len(species)
if len(match) > 1:
match_ = set(match)
species_ = set(species)
if len(match_) == 1 and len(species_) == 1:
match = list(match_)
species = list(species_)
assert len(match) == len(species)
if len(match) > 1:
match_ = set(match)
species_ = set(species)
if len(species_) == 1:
match = [any(match_)]
species = list(species_)
if len(match) == 1 and len(species) == 1:
match = match[0]
species = species[0]
if filter_match_against_on is None or match == filter_match_against_on:
uuid_list.append(uuid_)
species_list.append(species)
else:
logger.info(match, species)
assert len(uuid_list) == len(species_list)
assert len(uuid_list) == len(set(uuid_list))
logger.info('Validated %d Annotation UUIDs from Flukebook' %
(len(uuid_list), ))
flukebook_annot_uuid_list = uuid_list
flukebook_annot_species_list = species_list
return flukebook_annot_uuid_list, flukebook_annot_species_list
def load_json(fpath):
import utool as ut
json_data = ut.load_text(fpath)
data = ut.from_json(json_data)
return data
def on_collect_request(collect_request,
collecter_data,
awaiting_data,
shelve_path,
containerized=False):
""" Run whenever the collector recieves a message """
import requests
reply = {}
action = collect_request['action']
if VERBOSE_JOBS:
print('...building action=%r response' % (action, ))
if action == 'notification':
# From the Queue
jobid = collect_request['jobid']
awaiting_data[jobid] = collect_request['text']
# Make waiting lock
lock_filepath = join(shelve_path, '%s.lock' % (jobid, ))
ut.touch(lock_filepath)
elif action == 'store':
# From the Engine
engine_result = collect_request['engine_result']
callback_url = collect_request['callback_url']
callback_method = collect_request['callback_method']
jobid = engine_result['jobid']
if containerized:
callback_url = callback_url.replace('://localhost/',
'://wildbook:8080/')
# OLD METHOD
# collecter_data[jobid] = engine_result
collecter_data[jobid] = engine_result['exec_status']
# NEW METHOD
shelve_filepath = join(shelve_path, '%s.shelve' % (jobid, ))
shelf = shelve.open(shelve_filepath, writeback=True)
try:
shelf[str('result')] = engine_result
finally:
shelf.close()
# Delete the lock
lock_filepath = join(shelve_path, '%s.lock' % (jobid, ))
ut.delete(lock_filepath)
if callback_url is not None:
if callback_method is None:
callback_method = 'post'
else:
callback_method = callback_method.lower()
if VERBOSE_JOBS:
print('calling callback_url using callback_method')
try:
# requests.get(callback_url)
data_dict = {'jobid': jobid}
if callback_method == 'post':
response = requests.post(callback_url, data=data_dict)
elif callback_method == 'get':
response = requests.get(callback_url, params=data_dict)
elif callback_method == 'put':
response = requests.put(callback_url, data=data_dict)
else:
raise ValueError('callback_method %r unsupported' %
(callback_method, ))
try:
text = response.text
except:
text = None
args = (
callback_url,
callback_method,
data_dict,
response,
text,
)
print(
'WILDBOOK CALLBACK TO %r\n\tMETHOD: %r\n\tDATA: %r\n\tRESPONSE: %r\n\tTEXT: %r'
% args)
except Exception as ex:
msg = (
('ERROR in collector. '
'Tried to call callback_url=%r with callback_method=%r') %
(
callback_url,
callback_method,
))
print(msg)
ut.printex(ex, msg)
#requests.post(callback_url)
if VERBOSE_JOBS:
print('stored result')
elif action == 'job_status':
# From a Client
jobid = collect_request['jobid']
if jobid in collecter_data:
reply['jobstatus'] = 'completed'
reply['exec_status'] = collecter_data[jobid]
elif jobid in awaiting_data:
reply['jobstatus'] = 'working'
else:
reply['jobstatus'] = 'unknown'
reply['status'] = 'ok'
reply['jobid'] = jobid
elif action == 'job_id_list':
reply['status'] = 'ok'
reply['jobid_list'] = list(collecter_data.keys())
elif action == 'job_result':
# From a Client
jobid = collect_request['jobid']
try:
# OLD METHOD
# engine_result = collecter_data[jobid]
# NEW METHOD
shelve_filepath = join(shelve_path, '%s.shelve' % (jobid, ))
shelf = shelve.open(shelve_filepath)
try:
engine_result = shelf[str('result')]
finally:
shelf.close()
json_result = engine_result['json_result']
reply['jobid'] = jobid
reply['status'] = 'ok'
# reply['json_result'] = json_result
# We want to parse the JSON result here, since we need to live in
# Python land for the rest of the call until the API wrapper
# converts the Python objcets to JSON before the response is
# generated. This prevents the API from converting a Python string
# of JSON to a JSON string of JSON, which is bad.
reply['json_result'] = ut.from_json(json_result)
except KeyError:
reply['jobid'] = jobid
reply['status'] = 'invalid'
reply['json_result'] = None
else:
# Other
print('...error unknown action=%r' % (action, ))
reply['status'] = 'error'
reply['text'] = 'unknown action'
return reply
def on_collect_request(collect_request, collecter_data, awaiting_data, shelve_path):
""" Run whenever the collector recieves a message """
import requests
reply = {}
action = collect_request['action']
if VERBOSE_JOBS:
print('...building action=%r response' % (action,))
if action == 'notification':
# From the Queue
jobid = collect_request['jobid']
awaiting_data[jobid] = collect_request['text']
# Make waiting lock
lock_filepath = join(shelve_path, '%s.lock' % (jobid, ))
ut.touch(lock_filepath)
elif action == 'store':
# From the Engine
engine_result = collect_request['engine_result']
callback_url = collect_request['callback_url']
callback_method = collect_request['callback_method']
jobid = engine_result['jobid']
# OLD METHOD
# collecter_data[jobid] = engine_result
collecter_data[jobid] = engine_result['exec_status']
# NEW METHOD
shelve_filepath = join(shelve_path, '%s.shelve' % (jobid, ))
shelf = shelve.open(shelve_filepath, writeback=True)
try:
shelf[str('result')] = engine_result
finally:
shelf.close()
# Delete the lock
lock_filepath = join(shelve_path, '%s.lock' % (jobid, ))
ut.delete(lock_filepath)
if callback_url is not None:
if callback_method is None:
callback_method = 'post'
else:
callback_method = callback_method.lower()
if VERBOSE_JOBS:
print('calling callback_url using callback_method')
try:
# requests.get(callback_url)
if callback_method == 'post':
requests.post(callback_url, data={'jobid': jobid})
elif callback_method == 'get':
requests.get(callback_url, params={'jobid': jobid})
elif callback_method == 'put':
requests.put(callback_url, data={'jobid': jobid})
else:
raise ValueError('callback_method %r unsupported' %
(callback_method, ))
except Exception as ex:
msg = (('ERROR in collector. '
'Tried to call callback_url=%r with callback_method=%r')
% (callback_url, callback_method, ))
print(msg)
ut.printex(ex, msg)
#requests.post(callback_url)
if VERBOSE_JOBS:
print('stored result')
elif action == 'job_status':
# From a Client
jobid = collect_request['jobid']
if jobid in collecter_data:
reply['jobstatus'] = 'completed'
reply['exec_status'] = collecter_data[jobid]
elif jobid in awaiting_data:
reply['jobstatus'] = 'working'
else:
reply['jobstatus'] = 'unknown'
reply['status'] = 'ok'
reply['jobid'] = jobid
elif action == 'job_result':
# From a Client
jobid = collect_request['jobid']
try:
# OLD METHOD
# engine_result = collecter_data[jobid]
# NEW METHOD
shelve_filepath = join(shelve_path, '%s.shelve' % (jobid, ))
shelf = shelve.open(shelve_filepath)
try:
engine_result = shelf[str('result')]
finally:
shelf.close()
json_result = engine_result['json_result']
reply['jobid'] = jobid
reply['status'] = 'ok'
# reply['json_result'] = json_result
# We want to parse the JSON result here, since we need to live in
# Python land for the rest of the call until the API wrapper
# converts the Python objcets to JSON before the response is
# generated. This prevents the API from converting a Python string
# of JSON to a JSON string of JSON, which is bad.
reply['json_result'] = ut.from_json(json_result)
except KeyError:
reply['jobid'] = jobid
reply['status'] = 'invalid'
reply['json_result'] = None
else:
# Other
print('...error unknown action=%r' % (action,))
reply['status'] = 'error'
reply['text'] = 'unknown action'
return reply