def reef_upload(toktok):
if pp.token_test(toktok) == False:
return 'Invalid token'
if request.method != 'POST':
return 'Invalid, no file submitted'
file = request.files['file']
# Avoids errors with users with no sandbox assigned
try:
os.listdir('/root/project/api/sandbox_files/DIR_' + str(toktok))
except:
return 'User sandbox is not set-up, create a sandbox first'
# No user can have more than 2 Gb
assigned_allocation = float(r.get(toktok).decode('UTF-8'))
if pp.user_sandbox_size(str(toktok)) > (assigned_allocation * 1073741824):
return 'User has exceded asssigned allocation. Max. allocation is ' + str(
assigned_allocation) + ' GB'
# Avoids empty filenames and those with commas
if file.filename == '':
return 'Invalid, no file uploaded'
if ',' in file.filename:
return "ERROR: No ',' allowed in filenames"
# Ensures no commands within the file
new_name = secure_filename(file.filename)
file.save(os.path.join(UPLOAD_FOLDER + '/DIR_' + str(toktok), new_name))
return 'File succesfully uploaded to Reef'
def allocation_status():
if request.method != 'POST':
return "INVALID, no data provided"
if request.form['token'] == '':
return "INVALID, no token provided"
toktok = request.form["token"]
if pp.token_test(toktok) == False:
return 'Invalid token'
used_space = pp.user_sandbox_size(str(toktok)) / 1073741824
assigned_allocation = r_alloc.get(toktok).decode('UTF-8')
all_info = {
'Max. allocation':
assigned_allocation + ' GB',
'Used space':
str(used_space) + ' GB',
'Space available left':
str((1 - used_space / float(assigned_allocation)) * 100) +
'% allocation available'
}
return jsonify(all_info)
def reef_allocation_status(toktok):
if pp.token_test(toktok) == False:
return 'Invalid token'
used_space = pp.user_sandbox_size(str(toktok))/1073741824
assigned_allocation = r.get(toktok).decode('UTF-8')
all_info = {'Max. allocation': assigned_allocation+' GB',
'Used space': str(used_space)+' GB',
'Space available left': str((1 - used_space/float(assigned_allocation))*100)+'% allocation available'}
return jsonify(all_info)
def reef_upload(toktok):
if pp.token_test(toktok) == False:
return 'INVALID token'
if request.method != 'POST':
return 'INVALID, no file submitted'
file = request.files['file']
assigned_allocation = float(r.get(toktok).decode('UTF-8'))
if pp.user_sandbox_size(str(toktok)) > (assigned_allocation*1073741824):
return 'INVALID, User has exceded asssigned allocation. Max. available allocation is '+str(assigned_allocation)+' GB'
# Avoids empty filenames and those with commas
if file.filename == '':
return 'INVALID, no file uploaded'
if ',' in file.filename:
return "INVALID, no ',' allowed in filenames"
resp = requests.post('http://'+os.environ['Reef_IP']+':2000/reef/upload/'+os.environ['Reef_Key']+'/'+toktok,
data={"filename":secure_filename(file.filename)}, files={"file": file.read()})
return resp.content
def midas(toktok):
if pp.token_test(toktok) == False:
return 'Invalid token'
if request.method != 'POST':
return 'Invalid, no file submitted'
file = request.files['file']
try:
ALL_USER_DATA = os.listdir(
'/home/boincadm/project/api/sandbox_files/DIR_' + str(toktok))
except:
return 'User sandbox is not set-up, create a sandbox first'
# If no app is provided, it will assume BOINC
try:
boapp = request.form["app"].lower()
if (boapp != "boinc2docker") and (boapp != "adtdp"):
return "INVALID app"
except:
boapp = "boinc2docker"
# No user can submit jobs with a full allocation
assigned_allocation = float(r.get(toktok).decode('UTF-8'))
if pp.user_sandbox_size(str(toktok)) > (assigned_allocation * 1073741824):
return 'User has exceded asssigned allocation. Current available allocation is ' + str(
assigned_allocation) + ' GB'
if file.filename == '':
return 'Invalid, no file uploaded'
if ',' in file.filename:
return "ERROR: No ',' allowed in filenames"
if ('.tar.gz' not in file.filename) and ('.tgz' not in file.filename):
return 'ERROR: Compression file not accepted, file must be .tgz or .tar.gz'
new_name = secure_filename(file.filename)
file.save(os.path.join(UPLOAD_FOLDER + '/DIR_' + str(toktok), new_name))
try:
TAR = tarfile.open(UPLOAD_FOLDER + '/DIR_' + str(toktok) + '/' +
new_name)
except:
return 'ERROR: python cannot open tar file'
if not any('README.txt' in str(f) for f in TAR.getmembers()):
os.remove(os.path.join(UPLOAD_FOLDER + '/DIR_' + str(toktok),
new_name))
return 'ERROR: tar file does not contain mandatory README.txt'
# Creates a new MIDAS directory with a new name
while True:
new_MID = 'MID_' + pp.random_dir_name()
if new_MID not in ALL_USER_DATA: break
# Checks the README for all necessary inputs
TAR_PATH = UPLOAD_FOLDER + '/DIR_' + str(toktok) + '/' + new_MID
os.makedirs(TAR_PATH)
TAR.extractall(TAR_PATH)
if not mdr.valid_README(TAR_PATH + '/README.txt'):
shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' +
str(toktok) + '/' + new_MID)
return 'ERROR: README is not valid'
if not mdr.valid_OS(TAR_PATH + '/README.txt'):
shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' +
str(toktok) + '/' + new_MID)
return 'ERROR: OS is not accepted'
if not mdr.present_input_files(TAR_PATH):
shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' +
str(toktok) + '/' + new_MID)
return 'ERROR: Not all input files for commands are present'
if not mdr.valid_language(TAR_PATH + '/README.txt'):
shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' +
str(toktok) + '/' + new_MID)
return 'ERROR: Language is not accepted'
# Avoids cases where no libraries are needed at all
if (not mdr.install_libraries(TAR_PATH + '/README.txt')) and (
mdr.install_libraries(TAR_PATH + '/README.txt') != []):
shutil.rmtree('/home/boincadm/project/api/sandbox_files/DIR_' +
str(toktok) + '/' + new_MID)
return 'ERROR: Language does not support libraries'
# Creates a redis database with syntax {TOKEN;MID_DIRECTORY:boapp}
r.set(toktok + ';' + new_MID, boapp)
return 'File submitted for processing'
def delete_user_data():
if request.method != 'POST':
return "INVALID, no data provided"
if request.form['token'] == '':
return "INVALID, no token provided"
toktok = request.form["token"]
if pp.token_test(toktok) == False:
return "Invalid token"
# All of the options must be selected with either y, or left blank
# Sets up what to delete (more than one option may be selected):
# - all: Deletes all the data in Reef, including directories
# - basic: Deletes the temporary MIDAS directories, non-accessible to users in regular cases
# - ordinary: Deletes Reef documents, not results, or MIDAS
# - results: Deletes the results
# Write y/Y/yes/YES to delete, all other commands will not delete
delete_all = str(request.form['all']).lower()
delete_basic = str(request.form['basic']).lower()
delete_ordinary = str(request.form['ordinary']).lower()
delete_results = str(request.form['results']).lower()
user_commands = list(
map(pp.y_parser,
[delete_all, delete_basic, delete_ordinary, delete_results]))
# Avoids users who do not want to delete any data
if all((not x) for x in user_commands):
return "INVALID, no files have been marked for erasing"
# Computes the starting user space
used_space = pp.user_sandbox_size(str(toktok)) / 1073741824
# Finds all the user data
USER_PATH = "/root/project/api/sandbox_files/DIR_" + toktok
user_files = os.listdir(USER_PATH)
if user_commands[0]:
# Calls all delete options
user_commands = [True, True, True, True]
if user_commands[1]:
# Deletes all MIDAS directories
ALL_MIDAS = [
USER_PATH + '/' + x for x in user_files if x[:4:] == "MID_"
]
for y in ALL_MIDAS:
shutil.rmtree(y)
if user_commands[2]:
# Deletes regular user files
REGULAR_REEF = [
USER_PATH + '/' + x for x in user_files
if (not os.path.isdir(USER_PATH + '/' + x))
]
for z in REGULAR_REEF:
os.remove(z)
if user_commands[3]:
# Deletes the results
RESULTS = os.listdir(USER_PATH + "/___RESULTS")
for w in RESULTS:
os.remove(USER_PATH + "/___RESULTS/" + w)
# Computes the saved space
now_space = pp.user_sandbox_size(str(toktok)) / 1073741824
recovered_space = used_space - now_space
r_alloc.incrbyfloat(toktok, recovered_space)
new_alloc = r_alloc.get(toktok).decode("UTF-8")
return "Space recovered: " + str(
recovered_space) + " GB; new allocated space: " + str(
new_alloc) + " GB"
