def upload_archive(upload_data: UploadData,
requested_dir_path: str) -> (Path, ErrorCodeAndMessage):
try:
raw_content = base64.decodebytes(upload_data.base64_content.encode())
except Error as e:
return None, ErrorCodeAndMessageFormatter(INVALID_BASE_64, e)
file_name = '{}.zip'.format(requested_dir_path)
try:
with open(file_name, 'wb') as f:
f.write(raw_content)
except OSError:
return None, UNEXPECTED_ERROR
try:
with zipfile.ZipFile(file_name, mode='r') as zf:
zf.extractall(path=requested_dir_path)
except zipfile.BadZipFile as e:
try:
os.remove(file_name)
except:
pass
return None, ErrorCodeAndMessageFormatter(NOT_AN_ARCHIVE, e)
os.remove(file_name)
path = Path.object_from_pathname(requested_dir_path)
return path, None
def put(self, user, execution_identifier):
execution_db = get_execution(execution_identifier, db.session)
if not execution_db:
return ErrorCodeAndMessageFormatter(EXECUTION_NOT_FOUND,
execution_identifier)
if user.role != Role.admin and execution_db.creator_username != user.username:
return UNAUTHORIZED
if execution_db.status != ExecutionStatus.Running:
return ErrorCodeAndMessageFormatter(
CANNOT_KILL_NOT_RUNNING_EXECUTION, execution_db.status.name)
# Look at its running processes
execution_processes = get_execution_processes(execution_identifier,
db.session)
if not execution_processes: # Most probably due to the execution being in termination process
return CANNOT_KILL_FINISHING_EXECUTION
kill_all_execution_processes(execution_processes)
# Mark the execution as "Killed" and delete the execution processes
execution_db.status = ExecutionStatus.Killed
execution_db.end_date = current_milli_time()
for execution_process in execution_processes:
db.session.delete(execution_process)
db.session.commit()
def validate_request_model(model: Execution,
url_root: str) -> (bool, ErrorCodeAndMessage):
if model.identifier:
return False, EXECUTION_IDENTIFIER_MUST_NOT_BE_SET
pipeline = get_pipeline(model.pipeline_identifier)
if not pipeline:
return False, INVALID_PIPELINE_IDENTIFIER
files_exist, error = input_files_exist(model.input_values, pipeline,
url_root)
if not files_exist:
error_code_and_message = ErrorCodeAndMessageFormatter(
INVALID_INPUT_FILE, error)
return False, error_code_and_message
# Timeout validation
min_authorized_execution_timeout = PLATFORM_PROPERTIES.get(
"minAuthorizedExecutionTimeout", 0)
max_authorized_execution_timeout = PLATFORM_PROPERTIES.get(
"maxAuthorizedExecutionTimeout", 0)
if model.timeout and ((max_authorized_execution_timeout > 0 and
model.timeout > max_authorized_execution_timeout) or
(model.timeout < min_authorized_execution_timeout)):
error_code_and_message = ErrorCodeAndMessageFormatter(
INVALID_EXECUTION_TIMEOUT, min_authorized_execution_timeout,
max_authorized_execution_timeout or "(no maximum timeout)")
return False, error_code_and_message
return True, None
def get(self, user, execution_identifier):
execution_db = get_execution(execution_identifier, db.session)
execution, error = get_execution_as_model(user.username, execution_db)
if error:
return ErrorCodeAndMessageFormatter(EXECUTION_NOT_FOUND,
execution_identifier)
return execution
def test_get_invalid_execution(self, test_client):
execution_id = "invalid"
response = test_client.get('/executions/{}'.format(execution_id),
headers={"apiKey": standard_user().api_key})
error = error_from_response(response)
expected_error_code_and_message = ErrorCodeAndMessageFormatter(
EXECUTION_NOT_FOUND, execution_id)
assert error == expected_error_code_and_message
def test_get_execution_std_err_invalid_execution_id(
self, test_client, execution_id, write_std_err):
invalid_execution_id = "NOT_{}".format(execution_id)
response = test_client.get(
'/executions/{}/stderr'.format(invalid_execution_id),
headers={"apiKey": standard_user().api_key})
error = error_from_response(response)
expected_error_code_and_message = ErrorCodeAndMessageFormatter(
EXECUTION_NOT_FOUND, invalid_execution_id)
assert error == expected_error_code_and_message
def get(self, user, execution_identifier):
execution_db = get_execution(execution_identifier, db.session)
if not execution_db:
return ErrorCodeAndMessageFormatter(EXECUTION_NOT_FOUND,
execution_identifier)
if not is_safe_for_get(user, execution_db):
return UNAUTHORIZED
if execution_db.status not in EXECUTION_COMPLETED_STATUSES:
return ErrorCodeAndMessageFormatter(
CANNOT_GET_RESULT_NOT_COMPLETED_EXECUTION,
execution_db.status.name)
# We now know the execution has completed and can retrieve the output files
output_files, error = get_output_files(execution_db.creator_username,
execution_identifier)
if error:
return CORRUPTED_EXECUTION
return output_files
def put(self, user, execution_identifier):
execution_db = get_execution(execution_identifier, db.session)
if not execution_db:
return ErrorCodeAndMessageFormatter(EXECUTION_NOT_FOUND,
execution_identifier)
if execution_db.creator_username != user.username:
return UNAUTHORIZED
if execution_db.status != ExecutionStatus.Initializing:
return ErrorCodeAndMessageFormatter(CANNOT_REPLAY_EXECUTION,
execution_db.status.name)
execution, error = get_execution_as_model(user.username, execution_db)
if error:
return CORRUPTED_EXECUTION
# Get the descriptor path
descriptor_path = get_descriptor_path(user.username,
execution.identifier)
# Get appriopriate descriptor object
descriptor = Descriptor.descriptor_factory_from_type(
execution_db.descriptor)
if not descriptor:
# We don't have any descriptor defined for this pipeline type
logger = logging.getLogger('server-error')
logger.error(
"Unsupported descriptor type extracted from file at {}".format(
descriptor_path))
return ErrorCodeAndMessageFormatter(UNSUPPORTED_DESCRIPTOR_TYPE,
execution_db.descriptor)
modified_inputs_path = get_absolute_path_inputs_path(
user.username, execution.identifier)
if not os.path.isfile(modified_inputs_path):
logger = logging.getLogger('server-error')
logger.error("Absolute path inputs file not found at {}".format(
descriptor_path))
return UNEXPECTED_ERROR
# The execution is valid and we are now ready to start it
start_execution(user, execution, descriptor, modified_inputs_path)
def put(self, model, execution_identifier, user):
if model.identifier:
return ErrorCodeAndMessageFormatter(CANNOT_MODIFY_PARAMETER,
"identifier")
if model.status:
return ErrorCodeAndMessageFormatter(CANNOT_MODIFY_PARAMETER,
"status")
if model.name or model.timeout:
execution_db = get_execution(execution_identifier, db.session)
if not execution_db:
return ErrorCodeAndMessageFormatter(EXECUTION_NOT_FOUND,
execution_identifier)
if model.name:
execution_db.name = model.name
if model.timeout:
execution_db.timeout = model.timeout
db.session.add(execution_db)
db.session.commit()
def delete(self, user, execution_identifier):
execution_db = get_execution(execution_identifier, db.session)
if not execution_db:
return ErrorCodeAndMessageFormatter(EXECUTION_NOT_FOUND,
execution_identifier)
if execution_db.creator_username != user.username:
return UNAUTHORIZED
deleteFiles = request.args.get(
'deleteFiles', default=False, type=inputs.boolean)
# Get all the execution running processes
execution_processes = get_execution_processes(execution_identifier,
db.session)
# Given delete is called to perform only a kill and encounter the same situation as kill, we do not kill the processes
# See execution_kill for more information
if execution_db.status == ExecutionStatus.Running and not execution_processes and not deleteFiles:
return CANNOT_KILL_FINISHING_EXECUTION
if execution_db.status != ExecutionStatus.Running and not deleteFiles:
return ErrorCodeAndMessageFormatter(
CANNOT_KILL_NOT_RUNNING_EXECUTION, execution_db.status.name)
# Kill all the execution processed
kill_all_execution_processes(execution_processes)
for execution_process in execution_processes:
db.session.delete(execution_process)
# If the execution is not in a completed status, we mark it as killed
if execution_db.status == ExecutionStatus.Running:
execution_db.status = ExecutionStatus.Killed
execution_db.end_date = current_milli_time()
db.session.commit()
# Free all resources associated with the execution if delete files is True
if deleteFiles:
execution_dir = get_execution_dir(user.username,
execution_identifier)
delete_execution_directory(execution_dir)
db.session.delete(execution_db)
db.session.commit()
def upload_file(upload_data: UploadData,
requested_file_path: str) -> (Path, ErrorCodeAndMessage):
try:
raw_content = base64.decodebytes(upload_data.base64_content.encode())
except Error as e:
return None, ErrorCodeAndMessageFormatter(INVALID_BASE_64, e)
try:
with open(requested_file_path, 'wb') as f:
f.write(raw_content)
except OSError:
return None, UNEXPECTED_ERROR
path = Path.object_from_pathname(requested_file_path)
return path, None
def put(self, user, complete_path: str = ''):
data = request.data
requested_data_path = make_absolute(complete_path)
if not is_safe_for_put(requested_data_path, user):
return marshal(INVALID_PATH), 401
if request.headers.get(
'Content-Type',
default='').lower() == 'application/carmin+json' and data:
# Request data contains base64 encoding of file or archive
data = request.get_json(force=True, silent=True)
model, error = UploadDataSchema().load(data)
if error:
return marshal(
ErrorCodeAndMessageAdditionalDetails(
INVALID_MODEL_PROVIDED, error)), 400
if model.upload_type == "File":
if os.path.isdir(requested_data_path):
error = ErrorCodeAndMessageFormatter(
PATH_IS_DIRECTORY, complete_path)
return marshal(error), 400
path, error = upload_file(model, requested_data_path)
if error:
return marshal(error), 400
return marshal(path), 201
if model.upload_type == "Archive":
path, error = upload_archive(model, requested_data_path)
if error:
return marshal(error), 400
return marshal(path), 201
if data:
# Content-Type is not 'application/carmin+json',
# request data is taken as raw text
try:
with open(requested_data_path, 'w') as f:
f.write(data.decode('utf-8', errors='ignore'))
return marshal(
PathModel.object_from_pathname(requested_data_path)), 201
except OSError:
return marshal(INVALID_PATH), 400
if not data:
path, error = create_directory(requested_data_path)
if error:
return marshal(error), 400
file_location_header = {'Location': path.platform_path}
string_path = json.dumps(PathSchema().dump(path).data)
return make_response((string_path, 201, file_location_header))
return marshal(INVALID_REQUEST), 400
def post(self, model, user):
already_existing_user = db.session.query(User).filter_by(
username=model.username).first()
if already_existing_user:
return ErrorCodeAndMessageFormatter(USERNAME_ALREADY_EXISTS,
model.username)
result, error = register_user(model.username, model.password,
Role.user, db.session)
if error:
if error.error_code != USERNAME_ALREADY_EXISTS.error_code:
return UNEXPECTED_ERROR
return error
def register_user(username: str, password: str, user_role: Role,
db_session) -> (bool, ErrorCodeAndMessage):
try:
new_user = User(
username=username,
password=generate_password_hash(password),
role=user_role)
db_session.add(new_user)
path, error = create_user_directory(new_user.username)
if error:
db_session.rollback()
return False, error
db_session.commit()
return True, None
except IntegrityError:
db_session.rollback()
return False, ErrorCodeAndMessageFormatter(USERNAME_ALREADY_EXISTS,
username)
def std_file_resource(user, execution_identifier, path_to_file):
execution_db = get_execution(execution_identifier, db.session)
if not execution_db:
error = ErrorCodeAndMessageFormatter(EXECUTION_NOT_FOUND,
execution_identifier)
return marshal(error), 400
if not is_safe_for_get(user, execution_db):
return UNAUTHORIZED
execution, error = get_execution_as_model(execution_db.creator_username,
execution_db)
if error:
return marshal(error), 400
std, error = get_std_file(user.username, execution_identifier,
path_to_file)
if error:
return marshal(error), 400
return Response(std, mimetype='text/plain')
def wrapper(*args, **kwargs):
model = func(*args, **kwargs)
if (isinstance(model, ErrorCodeAndMessage)):
return ErrorCodeAndMessageMarshaller(
model), 500 if model == UNEXPECTED_ERROR else 400
if (schema is None):
return '', 204
json, errors = schema.dump(model)
if errors:
model_dumping_error = ErrorCodeAndMessageFormatter(
MODEL_DUMPING_ERROR,
type(model).__name__)
model_dumping_error = ErrorCodeAndMessageAdditionalDetails(
model_dumping_error, errors)
return ErrorCodeAndMessageMarshaller(model_dumping_error), 500
return json
def post(self, model, user):
if model.password:
if user.role == Role.admin and model.username:
# Logged in user is an admin and wants to change
# another user's password
edit_user = db.session.query(User).filter_by(
username=model.username).first()
if not edit_user:
return ErrorCodeAndMessageFormatter(
USER_DOES_NOT_EXIST, model.username)
else:
# Logged in user is not an admin
if model.username and model.username != user.username:
return UNAUTHORIZED
edit_user = db.session.query(User).filter_by(
username=user.username).first()
edit_user.password = generate_password_hash(model.password)
db.session.commit()
else:
# Password was not provided
return ErrorCodeAndMessageAdditionalDetails(
INVALID_MODEL_PROVIDED, "'password' is required")