def post(self):
logger.info('In ClusterText post()')
args = parsers.cluster_parser.parse_args()
if not allowed_file(args['file'].filename, settings.TEXT_ALLOWED_EXTENSIONS):
logger.error('Invalid file name ' + args['file'].filename)
error = BadRequest()
error.data = ['This filetype is not allowed. Please contact admin.']
raise error
n = args['n']
if n < 2 or n > 6:
logger.error('Invalid n-gram size: %d' % n)
error = BadRequest()
error.data = ['n value must be between 2 and 6. Please try again.']
raise error
try:
coverage = True if 'true' in args['coverage'] else False
logger.info('Coverage value is ' + str(coverage))
mem_file = BytesIO()
args['file'].save(mem_file)
logger.info('Finding cluster labels...')
clusters, coverage = find_labels(mem_file.getvalue().decode('UTF-8'), n, coverage)
output = {'coverage': coverage, 'clusters': clusters}
except:
logger.error('Some exception occurred.')
error = InternalServerError()
error.data = ['Some error. Please contact admin.']
raise error
return output, 200
def post(self):
"""Predict audio classes from input data"""
result = {'status': 'error'}
args = input_parser.parse_args()
audio_data = args['audio'].read()
# clean up from earlier runs
if os.path.exists("/audio.wav"):
os.remove("/audio.wav")
if '.wav' in str(args['audio']):
file = open("/audio.wav", "wb")
file.write(audio_data)
file.close()
else:
e = BadRequest()
e.data = {
'status': 'error',
'message': 'Invalid file type/extension'
}
raise e
# Getting the predictions
try:
preds = self.model_wrapper._predict("/audio.wav",
args['start_time'])
except ValueError:
e = BadRequest()
e.data = {
'status': 'error',
'message': 'Invalid start time: value outside audio clip'
}
raise e
# Aligning the predictions to the required API format
label_preds = [{
'label_id': p[0],
'label': p[1],
'probability': p[2]
} for p in preds]
# Filter list
if args['filter'] is not None and any(x.strip() != ''
for x in args['filter']):
label_preds = [
x for x in label_preds if x['label'] in args['filter']
]
result['predictions'] = label_preds
result['status'] = 'ok'
os.remove("/audio.wav")
return result
def post(self):
"""Predict audio classes from input data"""
result = {'status': 'error'}
args = input_parser.parse_args()
if not re.match("audio/.*wav", str(args['audio'].mimetype)):
e = BadRequest()
e.data = {
'status':
'error',
'message':
'Invalid file type/extension: ' + str(args['audio'].mimetype)
}
raise e
audio_data = args['audio'].read()
# sliced_audio = audio_slice.slice(audio_data, 10)
# Getting the predictions
try:
preds = self.model_wrapper._predict(audio_data, args['start_time'])
except ValueError:
e = BadRequest()
e.data = {
'status': 'error',
'message': 'Invalid start time: value outside audio clip'
}
raise e
# Aligning the predictions to the required API format
label_preds = [{
'label_id': p[0],
'label': p[1],
'probability': p[2]
} for p in preds]
# label_preds = [{'label_id': p[0], 'probability': p[2]} for p in preds]
# Filter list
if args['filter'] is not None and any(x.strip() != ''
for x in args['filter']):
label_preds = [
x for x in label_preds if x['label'] in args['filter']
]
result['predictions'] = label_preds
result['status'] = 'ok'
return result
def post(self):
"""Generate audio embedding from input data"""
result = {'status': 'error'}
args = input_parser.parse_args()
audio_data = args['audio'].read()
# clean up from earlier runs
if os.path.exists("/audio.wav"):
os.remove("/audio.wav")
if '.wav' in str(args['audio']):
file = open("/audio.wav", "wb")
file.write(audio_data)
file.close()
else:
e = BadRequest()
e.data = {
'status': 'error',
'message': 'Invalid file type/extension'
}
raise e
# Getting the predictions
preds = self.model_wrapper.predict("/audio.wav")
# Aligning the predictions to the required API format
result['embedding'] = preds.tolist()
result['status'] = 'ok'
os.remove("/audio.wav")
return result
def post(self):
"""Predict audio classes from input data"""
result = {'status': 'error'}
args = audio_parser.parse_args()
audio_data = args['audio'].read()
# clean up from earlier runs
if os.path.exists("/audio.wav"):
os.remove("/audio.wav")
if '.wav' in str(args['audio']):
file = open("/audio.wav", "wb")
file.write(audio_data)
file.close()
else:
e = BadRequest()
e.data = {'status': 'error', 'message': 'Invalid file type/extension'}
raise e
# Getting the predictions
preds = self.mw.predict("/audio.wav")
# Aligning the predictions to the required API format
label_preds = [{'label_id': p[0], 'label': p[1], 'probability': p[2]} for p in preds]
result['predictions'] = label_preds
result['status'] = 'ok'
os.remove("/audio.wav")
return result
def authorizeRequest():
bearer = request.headers.get('Authorization')
e = BadRequest(
'No Authorization provided in headers. Attach a valid bearer token.')
e.data = {'Access': 'Denied', 'CODE': 'Unauthorized'}
if bearer is None or 'Bearer' not in bearer:
raise e
def validate(self, req=None, strict=False):
if req is None:
req = request
result = self.result_class()
# A record of arguments not yet parsed; as each is found among self.args, it will be popped out
req.unparsed_arguments = dict(
self.argument_class('').source(req)) if strict else {}
errors = {}
for arg in self.args:
value, found = arg.parse(req, self.bundle_errors)
if isinstance(value, ValueError):
errors.update(found)
found = None
if found or arg.store_missing:
result[arg.dest or arg.name] = value
if errors:
data = {
"errors": errors,
"message": "Invalid data",
}
bad_request = BadRequest()
bad_request.data = data
raise bad_request
if strict and req.unparsed_arguments:
arguments = ', '.join(req.unparsed_arguments.keys())
msg = 'Unknown arguments: {0}'.format(arguments)
raise BadRequest(msg)
return result
def get(self, filename):
logger.info('In DownloadFile get(), filename:' + filename)
if not allowed_file(filename, settings.EVENT_ALLOWED_EXTENSIONS):
logger.error('Invalid file name:' + filename)
error = BadRequest()
error.data = ['This file type is not allowed. Please contact admin.']
raise error
logger.info('Sending file from directory...')
root_dir = os.getcwd()
return send_from_directory(os.path.join(root_dir, settings.UPLOAD_FOLDER), filename)
def validate_payload(payload, api_model):
# check if any reqd fields are missing in payload
for key in api_model:
if api_model[key].required and key not in payload:
e = BadRequest('My custom message')
e.data = {'custom': 'Required field \'%s\' missing' % key}
raise e
# check payload
for key in payload:
field = api_model[key]
if isinstance(field, fields.List):
field = field.container
data = payload[key]
else:
data = [payload[key]]
if isinstance(field, custom_fields.CustomField) and hasattr(field, 'validate'):
for i in data:
if not field.validate(i):
e = BadRequest('My custom message')
e.data = {'custom': 'Validation of \'%s\' field failed' % key}
raise e
def raise_validation_error(message):
"""
Raises validation error
Args:
message (str): error message
Raises:
(ValidationError): raise an exception
"""
error = BadRequest()
error.data = {'status': 'error', 'message': message}
raise error
def read_image(self, image_data):
try:
image = Image.open(io.BytesIO(image_data))
except IOError as img_exception:
logger.error(str(img_exception))
e = BadRequest()
e.data = {
'status': 'error',
'message': 'The provided input is not a valid image.'
}
raise e
return image
def post(self):
data = request.json
holder = data.get('holder')
number = data.get('number')
cvv = data.get('cvv')
try:
decrypted_data = decrypt_card_data(holder, number, cvv)
except ValueError as error:
error_dict = ast.literal_eval(str(error))
err = BadRequest(error_dict.get("message", "Internal Error"))
err.data = error_dict
raise err
return decrypted_data, 200
def post(self):
logger.info('In ClusterEvent post()')
args = parsers.event_parser.parse_args()
if not allowed_file(args['file'].filename, settings.EVENT_ALLOWED_EXTENSIONS):
logger.error('Invalid file name ' + args['file'].filename)
error = BadRequest()
error.data = ['This file type is not allowed. Please contact admin.']
raise error
try:
comments_field = args['comments']
action_field = args['action']
filename = secure_filename(args['file'].filename)
filename = get_uuid_filename(filename)
file_path = os.path.join(settings.UPLOAD_FOLDER, filename)
if not os.path.exists(settings.UPLOAD_FOLDER):
logger.info('Creating upload folder: ' + settings.UPLOAD_FOLDER)
os.makedirs(settings.UPLOAD_FOLDER)
logger.info('Saving file to ' + file_path)
args['file'].save(file_path)
logger.info('Running event_cluster.run_e2e()')
derived_filename = utils.event_cluster.run_e2e(file_path, comments_field, action_field)
logger.info('Created new file at %s' % derived_filename)
output = {'download_url': url_for('download_file', filename=os.path.basename(derived_filename))}
except:
logger.error('Some error occurred.\n' + traceback.format_exc())
error = InternalServerError()
error.data = ['Some error. Please contact admin.']
raise error
return output, 200
def raise_bad_request_error(errors):
"""
Raises bad request error
Args:
errors (list): list of errors
Raises:
(ValidationError): raise an exception
"""
error = BadRequest()
error.data = {
'status': 'error',
'errors': errors
}
raise error
def get(self):
parse = reqparse.RequestParser(bundle_errors=True)
parse.add_argument('file',
type=werkzeug.datastructures.FileStorage,
required=True,
location='files',
help="File must be image.")
parse.add_argument(
'candidates',
type=int,
default=0,
choices=list(range(11)),
help='Number of candidate captions ranges from 0 to 10.')
args = parse.parse_args()
app.logger.info((f"Processing /caption request with "
f"{args['candidates']} candidates"))
file = args['file']
if file.filename == '':
app.logger.debug("File not selected in /caption request")
elif not str(file.filename).split('.')[-1] in ALLOWED_EXTENSIONS:
app.logger.debug(
f"File with invalid extension"
f"{str(file.filename).split('.')[-1]} in /caption request")
e = BadRequest('Input payload validation failed.')
e.data = {
'errors': {
'file':
f"File has invalid extension ."
f"{str(file.filename).split('.')[-1]}",
},
}
raise e
else:
image_name = f"{uuid.uuid4()}.jpg"
image_path = os.path.join('static', image_name)
file.save(image_path)
captions = model_driver.get_prediction(image_path, mt.ckpt,
args['candidates'])
result = {'captions': captions}
return result
def get(self, id):
'''Get user item from database'''
try:
my_user = User.fetch_by_id(id)
user = user_schema.dump(my_user)
if len(user) == 0:
e = BadRequest('User item does not exist')
e.data = {'status': '404'}
raise e
return {'status': 'Match retrieved', 'user': user}, 200
except KeyError as e:
api.abort(500,
e.__doc__,
status="Could not retrieve information",
statusCode="500")
except Exception as e:
api.abort(404,
e.__doc__,
status="User item does not exist",
statusCode="404")
def get(self, id):
'''Get todo item from database'''
try:
my_todo = Todo.fetch_by_id(id)
todo = todo_schema.dump(my_todo)
if len(todo) == 0:
e = BadRequest('Todo item does not exist')
e.data = {'status': '404'}
raise e
return {'status': 'Match retrieved', 'todo': todo}, 200
except KeyError as e:
api.abort(500,
e.__doc__,
status="Could not retrieve information",
statusCode="500")
except Exception as e:
api.abort(404,
e.__doc__,
status="Todo item does not exist",
statusCode="404")
def delete(self, id):
'''Delete user item from database'''
try:
my_user = User.fetch_by_id(id)
user = user_schema.dump(my_user)
if len(user) == 0:
e = BadRequest('User item does not exist')
e.data = {'status': '404'}
raise e
User.delete_by_id(id)
return {'status': 'User item has been deleted'}
except KeyError as e:
api.abort(500,
e.__doc__,
status="Could not perform this action",
statusCode="500")
except Exception as e:
api.abort(400,
e.__doc__,
status="Could perform this action",
statusCode="400")
def get_verified_data(request, schema):
"""
:param
request: request object
schema: json schema to validate against
:return: dict
data is verified against a JSON schema and errors are raised accordingly
data is decoded from JSON to a dict and returned
"""
try:
data = json.loads(request.data.decode('utf-8'))
except ValueError:
raise BadRequest("Unable to read JSON payload")
try:
validate(data, schema, format_checker=FormatChecker())
except ValidationError as e:
xcpt = BadRequest(e.message)
xcpt.data = {"source": {"path": schema_validation_error_path(e)}}
raise xcpt
return data
def post(self):
"""Generate audio embedding from input data"""
result = {'status': 'error'}
args = input_parser.parse_args()
if not re.match("audio/.*wav", str(args['audio'].mimetype)):
e = BadRequest()
e.data = {
'status': 'error',
'message': 'Invalid file type/extension'
}
raise e
audio_data = args['audio'].read()
# Getting the predictions
preds = self.model_wrapper.predict(audio_data)
# Aligning the predictions to the required API format
result['embedding'] = preds.tolist()
result['status'] = 'ok'
return result
def handle_error(error, error_code):
print(error)
bad_request = BadRequest()
bad_request.data = {'detail': str(error), 'error_code': error_code}
raise bad_request
file_parser = token_parser.copy()
file_parser.add_argument('xml_file',
required=True,
help=upload_file_help,
type=werkzeug.datastructures.FileStorage,
location='files')
page_parser = token_parser.copy()
page_parser.add_argument('limit', help=limit_help, type=int, location='args')
page_parser.add_argument('offset', help=offset_help, type=int, location='args')
date_parser = token_parser.copy()
date_parser.add_argument('from_date',
type=str,
help=date_help,
location='args')
date_parser.add_argument('to_date', type=str, help=date_help, location='args')
e401 = BadRequest('Unauthorized')
e401.code = 401
e401.data = {
'message': 'Autentication token is not authorized to do this action'
}
e404 = BadRequest('Not Found')
e404.code = 404
e500 = BadRequest('Internal Server Error')
e500.code = 500
def bad_request(message):
e = BadRequest(message)
e.data = {'error': message}
return e
def failure(errors=None, message=None):
from werkzeug.exceptions import BadRequest
e = BadRequest('My custom message')
e.data = ({'errors': errors, 'message': message, 'status': 'Failed'})
raise e
def getLastLocation(token_auth, visitor_id):
try:
config = {
'user': app.config.MATOMO_DATABASE_USER,
'password': app.config.MATOMO_DATABASE_PASS,
'host': app.config.MATOMO_DATABASE_HOST,
'port': app.config.MATOMO_DATABASE_PORT,
'database': app.config.MATOMO_DATABASE_NAME,
'connection_timeout': 5,
}
query = 'SELECT ' \
'lv.visit_last_action_time, ' \
'lv.idsite, inet_ntoa(conv(hex(location_ip), 16, 10)) AS location_ip, ' \
'lv.location_browser_lang, lv.location_country, lv.location_region, lv.location_city, ' \
'lv.location_latitude, lv.location_longitude ' \
'FROM `' + app.config.MATOMO_DATABASE_NAME + '`.`log_visit` lv ' \
'WHERE lv.idvisitor = UNHEX(\''+visitor_id+'\') ' \
'AND idsite IN (SELECT idsite FROM ' \
'`' + app.config.MATOMO_DATABASE_NAME + '`.`access` da, ' \
'`' + app.config.MATOMO_DATABASE_NAME + '`.`user` du ' \
' WHERE da.login = du.login ' \
'AND du.superuser_access = 0 AND du.token_auth=\''+token_auth+'\') ' \
'ORDER BY lv.idvisit'
cnx = mysql.connector.connect(**config)
cur = cnx.cursor(buffered=True)
cur.execute(query)
# fetch all of the rows from the query
data = cur.fetchall()
value = {
'visitor_id': visitor_id,
'site_id': None,
'location_ip': None,
'location_browser_lang': None,
'location_country': None,
'location_region': None,
'location_city': None,
'location_latitude': None,
'location_longitude': None,
'visit_last_action_time': None
}
if len(data) > 0:
for row in data:
value = {
'visitor_id': visitor_id,
'site_id': str(row[1]),
'location_ip': str(row[2]),
'location_browser_lang': str(row[3]),
'location_country': str(row[4]),
'location_region': str(row[5]),
'location_city': str(row[6]),
'location_latitude': str(row[7]),
'location_longitude': str(row[8]),
'visit_last_action_time': str(row[0])
}
cur.close()
cnx.close()
rtrn = json.dumps(value)
return json.loads(rtrn)
except mysql.connector.Error:
e = BadRequest('Connection Error')
e.code = 503
e.data = {'visitorid': visitor_id, 'value': 'Connection Error'}
raise e
except:
e = BadRequest('Service Unavailable')
e.code = 503
e.data = {'id': visitor_id, 'value': 'Error'}
raise e
def bad_request(message):
e = BadRequest(message)
e.data = {'error': message}
return e
def throw(server_error, status_code):
# abort(status_code, server_error=server_error)
b = BadRequest()
b.data = {'server_error': server_error}
raise b
def post(self):
"""Generate audio embedding from input data"""
result = {'status': 'error'}
true_start = time.time()
args = input_parser.parse_args()
if args['audio'] is None and args['url'] is None:
e = BadRequest()
e.data = {
'status': 'error',
'message': 'Need to provide either an audio or url argument'
}
raise e
audio_data = {}
uuid_map = {}
if args['url'] is not None:
url_splt = args['url'].split(',')
for url in url_splt:
audio_data[url] = urllib.request.urlopen(url).read()
else:
audio_data[args['audio'].filename] = args['audio'].read()
print(f"audio_data: {audio_data.keys()}")
for filestring in audio_data.keys():
uuid_map[filestring] = uuid.uuid1()
if 'mp3' in filestring:
print(f"Creating file: /{uuid_map[filestring]}.mp3")
file = open(f"/{uuid_map[filestring]}.mp3", "wb+")
file.write(audio_data[filestring])
file.close()
elif 'wav' in filestring:
print(f"Creating file: /{uuid_map[filestring]}.wav")
file = open(f"/{uuid_map[filestring]}.wav", "wb+")
file.write(audio_data[filestring])
file.close()
else:
e = BadRequest()
e.data = {
'status': 'error',
'message': 'Invalid file type/extension'
}
raise e
start = time.time()
commands = [
f"ffmpeg -i /{uuid_map[x]}.mp3 /{uuid_map[x]}.wav"
if 'mp3' in x else "" for x in uuid_map.keys()
]
threads = []
for command in commands:
if command != "":
print(f" Running command: {command}")
threads.append(
threading.Thread(target=run_sys, args=(command, )))
for thread in threads:
thread.start()
for thread in threads:
thread.join()
print(f'Converted mp3 files in {time.time() - start}s')
start = time.time()
for filestring in uuid_map.keys():
audio_data[filestring] = open(f"/{uuid_map[filestring]}.wav",
"rb").read()
os.remove(f"/{uuid_map[filestring]}.wav")
if 'mp3' in filestring:
os.remove(f"/{uuid_map[filestring]}.mp3")
print(f'Deleted files in {time.time() - start}s')
# Getting the predictions
res = {}
threads = []
for filestring in audio_data.keys():
threads.append(
threading.Thread(target=run_model,
args=(self.model_wrapper.predict, filestring,
audio_data[filestring], res)))
for thread in threads:
thread.start()
for thread in threads:
thread.join()
# Aligning the predictions to the required API format
result['embedding'] = res
result['status'] = 'ok'
print(f'Completed processing in {time.time() - true_start}s')
return result
