def get_dataset_image():
user = flask.request.args.get('user')
dataset = flask.request.args.get('dataset')
data_type = flask.request.args.get('type')
data_name = flask.request.args.get('name')
client = gcs.authenticate()
bucket = client.get_bucket('blueno-ml-files')
image = gcs.download_image(user, dataset, data_type, data_name, bucket)
return flask.send_file(image, mimetype='image/jpg')
def upload_dataset():
data = flask.request.form
files = flask.request.files
if 'user' not in data:
return flask.json.jsonify({'error': 'User not specified'})
if 'file' not in files:
logging.debug("File not found")
return flask.json.jsonify({'error': 'No file'})
files = files.getlist('file')
logging.debug(files)
logging.debug(str(datetime.datetime.now()))
client = pymongo.MongoClient(
"mongodb://*****:*****@104.196.51.205/elvo")
datasets = client.elvo.datasets
client = gcs.authenticate()
bucket = client.get_bucket('blueno-ml-files')
# TODO: Make multiprocessing work; it doesn't work because multiple
# consecutive requests to GCS seems to throw an error.
current_date = str(datetime.datetime.now())
for file in files:
# Save user-file relationship in MongoDB
dataset = {
"user": data['user'],
"name": file.filename,
"date": current_date,
"mip": False,
"shape": "None",
"dataset": 'default',
"gcs_url": '{}/default/{}.npy'.format(data['user'], file.filename),
"status": "running",
"message": "Please wait for the file to finish loading."
}
datasets.replace_one({
'name': file.filename,
'dataset': 'default'
},
dataset,
upsert=True)
p = Process(target=__process_file,
args=(files, data['user'], current_date, bucket))
p.start()
return flask.json.jsonify({'status': 'success'})
def _retrieve_arr(user, patient_id):
cached_arr = cache.get('{}-{}'.format(user, patient_id))
if cached_arr is not None:
logging.debug('loading {} from cache'.format(patient_id))
return cached_arr
client = gcs.authenticate()
bucket = client.get_bucket('blueno-ml-files')
logging.debug('Downloading {} from GCS'.format(patient_id))
arr = gcs.download_array('{}/default/{}.npy'.format(user, patient_id),
bucket)
if arr is None:
raise ValueError(
'Blob with patient id {} does not exist'.format(patient_id))
cache.set('{}-{}'.format(user, patient_id), arr)
return arr
def __process_dbx_file(files, user, date, path, token):
db = get_db()
dbx_client = dbx.authenticate(token)
client = gcs.authenticate()
bucket = client.get_bucket('blueno-ml-files')
for file in files:
try:
dbx.download_from_dropbox(dbx_client, '/tmp/cab_files/', path,
file)
npy = preprocess.process_cab(file, '/tmp/cab_files/')
preprocess.generate_images(npy, user, 'default', file, bucket,
'/tmp/')
gcs.upload_npy_to_gcs(npy, file, user, 'default', bucket)
dataset = {
"user": user,
"name": file,
"date": date,
"mip": False,
"shape": npy.shape,
"dataset": 'default',
"gcs_url": '{}/default/{}.npy'.format(user, file),
"status": "loaded",
"message": "Successfully loaded."
}
db.replace_one({'name': file, 'dataset': 'default'}, dataset)
except Exception as e:
dataset = {
"user": user,
"name": file,
"date": date,
"mip": False,
"shape": "None",
"dataset": 'default',
"gcs_url": '{}/default/{}.npy'.format(user, file),
"status": "failed",
"message": "Failed: {}".format(e)
}
db.replace_one({'name': file, 'dataset': 'default'}, dataset)
def __processing_job_helper(image_list, data, current_date):
client = gcs.authenticate()
bucket = client.get_bucket('blueno-ml-files')
db = get_db()
for image in image_list:
try:
logging.info(data)
arr = gcs.download_array(image['gcs_url'], bucket)
arr = preprocess.transform_array(arr, data)
if data['mip']:
preprocess.generate_mip_images(arr, data['user'], data['name'],
image['name'], bucket, '/tmp/')
else:
preprocess.generate_images(arr, data['user'], data['name'],
image['name'], bucket, '/tmp/')
gcs.upload_npy_to_gcs(arr, image['name'], data['user'],
data['name'], bucket)
dataset = {
"user":
data['user'],
"name":
image['name'],
"mip":
data['mip'],
"shape":
arr.shape,
"date":
current_date,
"dataset":
data['name'],
"gcs_url":
'{}/{}/{}.npy'.format(data['user'], data['name'],
image['name']),
"status":
"loaded",
"message":
"Successfully loaded."
}
db.replace_one({
'name': image['name'],
'dataset': data['name']
}, dataset)
logging.info(arr.shape)
except Exception as e:
dataset = {
"user":
data['user'],
"name":
image['name'],
"mip":
data['mip'],
"shape":
"None",
"date":
current_date,
"dataset":
data['name'],
"gcs_url":
'{}/{}/{}.npy'.format(data['user'], data['name'],
image['name']),
"status":
"failed",
"message":
"Failed: {}".format(e)
}
db.replace_one({
'name': image['name'],
'dataset': data['name']
}, dataset)