def create_annotation_group():
data = flask.request.get_json()
if 'user' not in data:
return flask.json.jsonify({'error': 'User not specified'})
if 'name' not in data:
return flask.json.jsonify({'error': 'Annotation name not specified'})
if 'type' not in data:
return flask.json.jsonify({'error': 'Annotation type not specified'})
db = get_db(db='annotation_groups')
try:
# Save user-file relationship in MongoDB
dataset = {
"user": data['user'],
"name": data['name'],
"type": data['type']
}
db.replace_one({'user': data['user'], 'name': data['name']},
dataset, upsert=True)
return flask.json.jsonify({'status': 'success'})
except Exception as e:
return flask.json.jsonify({'error': e})
def get_annotation_type():
user = flask.request.args.get('user')
group = flask.request.args.get('group')
logging.info(group)
db = get_db(db='annotation_groups')
results = db.find_one({'user': user, 'name': group})
data = {'type': results['type']}
return flask.json.jsonify({'data': data})
def annotate():
data = flask.request.get_json()
# Error checking
if 'user' not in data:
return flask.json.jsonify({'error': 'User not specified'})
if 'group' not in data:
return flask.json.jsonify({'error': 'Annotation group not specified'})
if 'id' not in data:
return flask.json.jsonify({'error': 'ID not specified'})
if 'type' not in data:
return flask.json.jsonify({'error': 'Annotation type not specified'})
if 'data' not in data:
return flask.json.jsonify({'error': 'Data not specified'})
if data['type'] == 'label' and ('class' not in data['data']):
return flask.json.jsonify({'error': 'Class label not specified'})
if data['type'] == 'bbox':
for each in ['x1', 'x2', 'y1', 'y2', 'z1', 'z2']:
if each not in data['data']:
return flask.json.jsonify(
{'error': 'Bounding box not specified'}
)
db = get_db(db='annotations')
try:
if data['type'] == 'label':
dataset = {
'user': data['user'],
'group': data['group'],
'id': data['id'],
'type': data['type'],
'class': data['data']['class']
}
elif data['type'] == 'bbox':
dataset = {
'user': data['user'],
'group': data['group'],
'id': data['id'],
'type': data['type'],
'x1': data['data']['x1'],
'x2': data['data']['x2'],
'y1': data['data']['y1'],
'y2': data['data']['y2'],
'z1': data['data']['z1'],
'z2': data['data']['z2']
}
else:
return flask.json.jsonify({'error': 'Invalid annotation type'})
db.replace_one({'user': data['user'], 'group': data['group'],
'id': data['id']}, dataset, upsert=True)
return flask.json.jsonify({'status': 'success'})
except Exception as e:
return flask.json.jsonify({'error': e})
def get_annotation_list():
user = flask.request.args.get('user')
db = get_db(db='annotation_groups')
results = db.find({'user': user})
data = []
for doc in results:
data.append({'user': doc['user'],
'name': doc['name'],
'type': doc['type']})
return flask.json.jsonify({'data': data})
def upload_dataset_from_dropbox():
data = flask.request.get_json()
if 'user' not in data:
return flask.json.jsonify({'error': 'User not specified'})
if 'path' not in data:
return flask.json.jsonify({'error': 'Path not specified'})
if 'token' not in data:
return flask.json.jsonify({'error': 'Token not specified'})
db = get_db()
dbx_client = dbx.authenticate(data['token'])
try:
dbx_client.users_get_current_account()
except Exception:
return flask.json.jsonify({'error': 'Invalid token'})
try:
files = dbx.get_cab_from_folder(dbx_client, data['path'])
except Exception:
return flask.json.jsonify({'error': 'Invalid path'})
current_date = str(datetime.datetime.now())
for file in files:
# Save user-file relationship in MongoDB
dataset = {
"user": data['user'],
"name": file,
"date": current_date,
"mip": False,
"shape": "None",
"dataset": 'default',
"gcs_url": '{}/default/{}.npy'.format(data['user'], file),
"status": "running",
"message": "Please wait for the file to finish loading."
}
db.replace_one({
'name': file,
'dataset': 'default'
},
dataset,
upsert=True)
p = Process(target=__process_dbx_file,
args=(files, data['user'], current_date, data['path'],
data['token']))
p.start()
return flask.json.jsonify({'status': 'success'})
def make_preprocessing_job():
data = flask.request.get_json()
# Get list of images to preprocess
db = get_db()
image_list = []
results = db.find({'user': data['user'], 'dataset': 'default'})
for doc in results:
image_list.append({'name': doc['name'], 'gcs_url': doc['gcs_url']})
# Save new file relationship in MongoDB
current_date = str(datetime.datetime.now())
for image in image_list:
dataset = {
"user":
data['user'],
"name":
image['name'],
"mip":
data['mip'],
"shape":
"None",
"date":
current_date,
"dataset":
data['name'],
"gcs_url":
'{}/{}/{}.npy'.format(data['user'], data['name'], image['name']),
"status":
"running",
"message":
"Currently processing this file."
}
db.replace_one({
'name': image['name'],
'dataset': data['name']
},
dataset,
upsert=True)
p = Process(target=__processing_job_helper,
args=(image_list, data, current_date))
p.start()
return flask.json.jsonify({'status': 'success'})
def get_annotations():
user = flask.request.args.get('user')
group = flask.request.args.get('group')
db = get_db(db='annotations')
results = db.find({'user': user, 'group': group})
data = {}
for doc in results:
if doc['type'] == 'label':
data[doc['id']] = {
'class': doc['class']
}
else:
data[doc['id']] = {
'x1': doc['x1'],
'x2': doc['x2'],
'y1': doc['y1'],
'y2': doc['y2'],
'z1': doc['z1'],
'z2': doc['z2']
}
return flask.json.jsonify({'data': data})
def __process_dbx_file(files, user, date, path, token):
db = get_db()
dbx_client = dbx.authenticate(token)
client = gcs.authenticate()
bucket = client.get_bucket('blueno-ml-files')
for file in files:
try:
dbx.download_from_dropbox(dbx_client, '/tmp/cab_files/', path,
file)
npy = preprocess.process_cab(file, '/tmp/cab_files/')
preprocess.generate_images(npy, user, 'default', file, bucket,
'/tmp/')
gcs.upload_npy_to_gcs(npy, file, user, 'default', bucket)
dataset = {
"user": user,
"name": file,
"date": date,
"mip": False,
"shape": npy.shape,
"dataset": 'default',
"gcs_url": '{}/default/{}.npy'.format(user, file),
"status": "loaded",
"message": "Successfully loaded."
}
db.replace_one({'name': file, 'dataset': 'default'}, dataset)
except Exception as e:
dataset = {
"user": user,
"name": file,
"date": date,
"mip": False,
"shape": "None",
"dataset": 'default',
"gcs_url": '{}/default/{}.npy'.format(user, file),
"status": "failed",
"message": "Failed: {}".format(e)
}
db.replace_one({'name': file, 'dataset': 'default'}, dataset)
def __processing_job_helper(image_list, data, current_date):
client = gcs.authenticate()
bucket = client.get_bucket('blueno-ml-files')
db = get_db()
for image in image_list:
try:
logging.info(data)
arr = gcs.download_array(image['gcs_url'], bucket)
arr = preprocess.transform_array(arr, data)
if data['mip']:
preprocess.generate_mip_images(arr, data['user'], data['name'],
image['name'], bucket, '/tmp/')
else:
preprocess.generate_images(arr, data['user'], data['name'],
image['name'], bucket, '/tmp/')
gcs.upload_npy_to_gcs(arr, image['name'], data['user'],
data['name'], bucket)
dataset = {
"user":
data['user'],
"name":
image['name'],
"mip":
data['mip'],
"shape":
arr.shape,
"date":
current_date,
"dataset":
data['name'],
"gcs_url":
'{}/{}/{}.npy'.format(data['user'], data['name'],
image['name']),
"status":
"loaded",
"message":
"Successfully loaded."
}
db.replace_one({
'name': image['name'],
'dataset': data['name']
}, dataset)
logging.info(arr.shape)
except Exception as e:
dataset = {
"user":
data['user'],
"name":
image['name'],
"mip":
data['mip'],
"shape":
"None",
"date":
current_date,
"dataset":
data['name'],
"gcs_url":
'{}/{}/{}.npy'.format(data['user'], data['name'],
image['name']),
"status":
"failed",
"message":
"Failed: {}".format(e)
}
db.replace_one({
'name': image['name'],
'dataset': data['name']
}, dataset)
def get_datasets_from_user():
user = flask.request.args.get('user')
db = get_db()
results = db.find({'user': user}).distinct('dataset')
return flask.json.jsonify({'data': results})