def clone(ws_id: int, url: str, name: str = None):
"""
Clones a repository by url into given workspace
:param name: Optional name of the local repository name, otherwise the remote name is taken
:param user_data: Session data to get access token for GitHub
:param ws_id: Destination workspace to clone
:param url: URL of the source repository
:return: True if successful, otherwise NameConflict is thrown
"""
workspace = get_workspace(ws_id)
url_decode = parse.urlparse(url)
if is_github(url_decode.netloc):
# Take the suffix of url as first name candidate
github_project_name = name
if github_project_name is None:
github_project_name = _repo_name_from_url(url_decode)
dbsession = db_session()
pj = dbsession.query(Project).join(Workspace)\
.filter(Workspace.id == workspace.id).filter(
Project.name == github_project_name).first()
dbsession.commit()
# Error when the project name in given workspace already exists
if pj is not None:
raise NameConflict('A project with name {} already exists'.format(github_project_name))
project_target_path = os.path.join(workspace.path, PROJECT_REL_PATH, github_project_name)
logger.info('Cloning from github repo...')
# If url in GitHub domain, access by token
url_with_token = _get_repo_url(url_decode)
out, err, exitcode = git_command(['clone', url_with_token, project_target_path])
if exitcode is 0:
setup_git_user_email(project_target_path)
# Check if the project is a valid son project
check_son_validity(project_target_path)
# Create project and scan it.
dbsession = db_session()
try:
pj = Project(github_project_name, github_project_name, workspace)
pj.repo_url = url
sync_project_descriptor(pj)
dbsession.add(pj)
scan_project_dir(project_target_path, pj)
dbsession.commit()
# Check if the project is valid
result = create_info_dict(out=out)
result["id"] = pj.id
return result
except:
dbsession.rollback()
shutil.rmtree(project_target_path)
raise Exception("Scan project failed")
else:
return create_info_dict(err=err, exitcode=exitcode)
raise NotImplemented("Cloning from other is not implemented yet. Only github is supported for now.")
def delete_image_file(ws_id, project_id, vnf_id, filename):
"""
Deletes the image file with the given name
:param ws_id: The workspace ID
:param project_id: The project ID
:param vnf_id: The VNF ID
:param filename: The name of the file to delete
:return: A success message
:raises NotFound: if the image file could not be located
"""
session = db_session()
function = session.query(Function). \
join(Project). \
join(Workspace). \
filter(Workspace.id == ws_id). \
filter(Project.id == project_id). \
filter(Function.id == vnf_id).first()
if function:
save_name = secure_filename(filename)
if not save_name == get_file_name(function):
file_path = get_file_path("vnf", function).replace(get_file_name(function), save_name)
if os.path.exists(file_path):
os.remove(file_path)
return "File {} was deleted".format(save_name)
else:
raise NotFound("File {} not found!".format(save_name))
else:
raise NotFound("Function with id " + vnf_id + " does not exist")
def update_project(project_data, project_id):
"""
Update the Project
:param project_data:
:param project_id:
:return:
"""
session = db_session()
project = session.query(Project).filter(Project.id == project_id).first()
if project is None:
raise NotFound("Project with id {} could not be found".format(project_id))
# Update name
if 'name' in project_data and project_data['name'] != project.name:
if os.path.exists(get_project_path(project.workspace.path, project.rel_path)):
new_name = shlex.quote(project_data['name'])
old_path = get_project_path(project.workspace.path, project.rel_path)
new_path = rreplace(old_path, project.name, new_name, 1)
if os.path.exists(new_path):
raise NameConflict("Invalid name parameter, workspace '{}' already exists".format(new_name))
# Do not allow move directories outside of the workspaces_dir
if not new_path.startswith(WORKSPACES_DIR):
raise Exception("Invalid path parameter, you are not allowed to break out of {}".format(WORKSPACES_DIR))
else:
# Move the directory
shutil.move(old_path, new_path)
project.name = new_name
project.rel_path = new_name
set_data(project, project_data)
sync_project_descriptor(project)
db_session.commit()
return project.as_dict()
def get_image_files(ws_id, project_id, function_id):
"""
Returns a list of image file names located in the vnf folder
:param ws_id: The Workspace ID
:param project_id: The project ID
:param function_id: The function ID
:return: A List of image file names for this VNF
"""
session = db_session()
function = session.query(Function). \
join(Project). \
join(Workspace). \
filter(Workspace.id == ws_id). \
filter(Project.id == project_id). \
filter(Function.id == function_id).first()
if function:
folder_path = get_file_path("vnf",
function).replace(get_file_name(function),
"")
image_files = []
for filename in os.listdir(folder_path):
if not Path(os.path.join(folder_path, filename)).is_dir():
if not filename == get_file_name(function):
image_files.append(filename)
return image_files
else:
raise NotFound("Function with id " + function_id + " does not exist")
def delete_image_file(ws_id, project_id, vnf_id, filename):
"""
Deletes the image file with the given name
:param ws_id: The workspace ID
:param project_id: The project ID
:param vnf_id: The VNF ID
:param filename: The name of the file to delete
:return: A success message
:raises NotFound: if the image file could not be located
"""
session = db_session()
function = session.query(Function). \
join(Project). \
join(Workspace). \
filter(Workspace.id == ws_id). \
filter(Project.id == project_id). \
filter(Function.id == vnf_id).first()
if function:
save_name = secure_filename(filename)
if not save_name == get_file_name(function):
file_path = get_file_path("vnf", function).replace(
get_file_name(function), save_name)
if os.path.exists(file_path):
os.remove(file_path)
return "File {} was deleted".format(save_name)
else:
raise NotFound("File {} not found!".format(save_name))
else:
raise NotFound("Function with id " + vnf_id + " does not exist")
def get_image_files(ws_id, project_id, function_id):
"""
Returns a list of image file names located in the vnf folder
:param ws_id: The Workspace ID
:param project_id: The project ID
:param function_id: The function ID
:return: A List of image file names for this VNF
"""
session = db_session()
function = session.query(Function). \
join(Project). \
join(Workspace). \
filter(Workspace.id == ws_id). \
filter(Project.id == project_id). \
filter(Function.id == function_id).first()
if function:
folder_path = get_file_path("vnf", function).replace(get_file_name(function), "")
image_files = []
for filename in os.listdir(folder_path):
if not Path(os.path.join(folder_path, filename)).is_dir():
if not filename == get_file_name(function):
image_files.append(filename)
return image_files
else:
raise NotFound("Function with id " + function_id + " does not exist")
def query_private_nsfs(ws_id, vendor, name, version, is_vnf):
"""
Finds a function in the private catalogue
:param ws_id:
:param is_vnf:
:param vendor:
:param name:
:param version:
:return:
"""
session = db_session()
if is_vnf:
descriptor = session.query(PrivateFunction).filter(
PrivateFunction.name == name and PrivateFunction.vendor == vendor
and PrivateFunction.version == version
and PrivateFunction.workspace.id == ws_id
and PrivateFunction.workspace.owner == get_user(
session['user_data'])).first()
else:
descriptor = session.query(PrivateService).filter(
PrivateService.name == name and PrivateService.vendor == vendor
and PrivateService.version == version
and PrivateFunction.workspace.id == ws_id
and PrivateFunction.workspace.owner == get_user(
session['user_data'])).first()
return descriptor
def create_platform(workspace_id: int, platform_data) -> dict:
"""
Create a new platform entry
:param workspace_id:
:param platform_data:
:return:
"""
platform_name = shlex.quote(platform_data['name'])
platform_url = shlex.quote(platform_data['url'])
session = db_session()
workspace = session.query(Workspace).filter(
Workspace.id == workspace_id).first()
if workspace is None:
raise NotFound(
"workspace with id {} could not be found".format(workspace_id))
existing_platforms = session.query(Platform). \
filter(Platform.workspace == workspace). \
filter(Platform.name == platform_data['name']). \
all()
if len(existing_platforms) > 0:
raise NameConflict("Platform with name {} already exists".format(
platform_data['name']))
platform = Platform(name=platform_name,
url=platform_url,
workspace=workspace)
session.add(platform)
update_workspace_descriptor(platform.workspace)
session.commit()
return platform.as_dict()
def delete_service(project_id: int, service_id: int) -> dict:
"""
Deletes the service from the Database and from the disk
:param project_id: The Projects ID
:param service_id: The Services ID
:return: The descriptor of the deleted service
"""
session = db_session()
project = session.query(Project).filter(Project.id == project_id).first()
if project is None:
raise NotFound("Could not delete service: project with id {} not found".format(service_id))
service = session.query(Service). \
filter(Service.id == service_id). \
filter(Service.project == project). \
first()
if service is None:
raise NotFound("Could not delete service: service with id {} not found".format(service_id))
refs = get_references(service, session)
if refs:
session.rollback()
ref_str = ",\n".join(ref.uid for ref in refs)
raise StillReferenced("Could not delete service because it is still referenced by \n" + ref_str)
session.delete(service)
try:
os.remove(get_file_path("nsd", service))
except:
session.rollback()
logger.exception("Could not delete service:")
raise
session.commit()
return service.as_dict()
def query_private_nsfs(ws_id, vendor, name, version, is_vnf):
"""
Finds a function in the private catalogue
:param ws_id:
:param is_vnf:
:param vendor:
:param name:
:param version:
:return:
"""
session = db_session()
if is_vnf:
descriptor = session.query(PrivateFunction).filter(PrivateFunction.name == name and
PrivateFunction.vendor == vendor and
PrivateFunction.version == version and
PrivateFunction.workspace.id == ws_id and
PrivateFunction.workspace.owner == get_user(
session['user_data'])).first()
else:
descriptor = session.query(PrivateService).filter(
PrivateService.name == name and
PrivateService.vendor == vendor and
PrivateService.version == version and
PrivateFunction.workspace.id == ws_id and
PrivateFunction.workspace.owner == get_user(session['user_data'])).first()
return descriptor
def delete_service(project_id: int, service_id: int) -> dict:
"""
Deletes the service from the Database and from the disk
:param project_id: The Projects ID
:param service_id: The Services ID
:return: The descriptor of the deleted service
"""
session = db_session()
project = session.query(Project).filter(Project.id == project_id).first()
if project is None:
raise NotFound("Could not delete service: project with id {} not found".format(service_id))
service = session.query(Service). \
filter(Service.id == service_id). \
filter(Service.project == project). \
first()
if service is None:
raise NotFound("Could not delete service: service with id {} not found".format(service_id))
session.delete(service)
try:
os.remove(get_file_path("nsd", service))
except:
session.rollback()
logger.exception("Could not delete service:")
raise
session.commit()
return service.as_dict()
def create_catalogue(workspace_id: int, catalogue_data):
"""
Creates a catalgoue in the given workspace. A catalogue is defined by its name and url. These are given as
json data
:param workspace_id: Workspace ID of the target workspace, where the catalogue should get created.
:return: Catalogue descriptor
"""
catalogue_name = shlex.quote(catalogue_data['name'])
catalogue_url = shlex.quote(catalogue_data['url'])
session = db_session()
workspace = session.query(Workspace).filter(Workspace.id == workspace_id).first()
if workspace is None:
raise NotFound("workspace with id {} could not be found".format(workspace_id))
existing_catalogues = session.query(Catalogue). \
filter(Catalogue.workspace == workspace). \
filter(Catalogue.name == catalogue_data['name']). \
all()
if len(existing_catalogues) > 0:
raise NameConflict("catalogue with name {} already exists".format(catalogue_data['name']))
catalogue = Catalogue(name=catalogue_name, url=catalogue_url, workspace=workspace)
session.add(catalogue)
session.commit()
update_workspace_descriptor(catalogue.workspace)
return catalogue.as_dict()
def create_platform(workspace_id: int, platform_data) -> dict:
"""
Create a new platform entry
:param workspace_id: The workspace ID
:param platform_data: The platform info
:return: The newly created platform descriptor
"""
platform_name = shlex.quote(platform_data['name'])
platform_url = shlex.quote(platform_data['url'])
session = db_session()
workspace = session.query(Workspace).filter(Workspace.id == workspace_id).first()
if workspace is None:
raise NotFound("workspace with id {} could not be found".format(workspace_id))
existing_platforms = session.query(Platform). \
filter(Platform.workspace == workspace). \
filter(Platform.name == platform_data['name']). \
all()
if len(existing_platforms) > 0:
raise NameConflict("Platform with name {} already exists".format(platform_data['name']))
platform = Platform(name=platform_name, url=platform_url, workspace=workspace)
session.add(platform)
update_workspace_descriptor(platform.workspace)
session.commit()
return platform.as_dict()
def update_catalogue(workspace_id, catalogue_id, catalogue_data):
"""
Updates a specific catalogue by its id. The catalogue
applies the given name and url, that are in the json parameter.
:param workspace_id: The Workspace ID
:param catalogue_id: The Catalogue ID
:return: The updated Catalogue descriptor
"""
catalogue_name = shlex.quote(catalogue_data['name'])
catalogue_url = shlex.quote(catalogue_data['url'])
session = db_session()
workspace = session.query(Workspace).filter(Workspace.id == workspace_id).first()
if workspace is None:
raise NotFound("workspace with id {} could not be found".format(workspace_id))
catalogue = session.query(Catalogue). \
filter(Catalogue.workspace == workspace). \
filter(Catalogue.id == catalogue_id). \
first()
if catalogue is None:
raise NotFound("catalogue with id {} could not be found".format(catalogue_id))
if catalogue_name != catalogue.name:
existing_catalogues = session.query(catalogue). \
filter(catalogue.workspace == workspace). \
filter(catalogue.name == catalogue_data['name']). \
all()
if len(existing_catalogues) > 0:
raise NameConflict("catalogue with name {} already exists".format(catalogue_data['name']))
catalogue.name = catalogue_name
catalogue.url = catalogue_url
session.commit()
update_workspace_descriptor(catalogue.workspace)
return catalogue.as_dict()
def query_private_nsfs(ws_id, vendor, name, version, is_vnf):
"""
Finds a function in the private catalogue
:param ws_id: The workspace ID
:param is_vnf: if descriptor is a VNF
:param vendor: the descriptors vendor
:param name: the descriptors name
:param version: the descriptors version
:return: The requested descriptor if found, None if nothing found
"""
session = db_session()
if is_vnf:
descriptor = session.query(PrivateFunction).filter(
PrivateFunction.name == name and PrivateFunction.vendor == vendor
and PrivateFunction.version == version
and PrivateFunction.workspace.id == ws_id
and PrivateFunction.workspace.owner == get_user(
session['user_data'])).first()
else:
descriptor = session.query(PrivateService).filter(
PrivateService.name == name and PrivateService.vendor == vendor
and PrivateService.version == version
and PrivateFunction.workspace.id == ws_id
and PrivateFunction.workspace.owner == get_user(
session['user_data'])).first()
return descriptor
def delete(workspace_id: int, platform_id: int) -> dict:
"""
Deletes the platform from the workspace
:param workspace_id:
:param platform_id:
:return: the deleted platform description
"""
session = db_session()
workspace = session.query(Workspace).filter(
Workspace.id == workspace_id).first()
if workspace is None:
raise NotFound(
"workspace with id {} could not be found".format(workspace_id))
platform = session.query(Platform). \
filter(Platform.workspace == workspace). \
filter(Platform.id == platform_id). \
first()
if platform is None:
raise NotFound(
"Platform with id {} could not be found".format(platform_id))
session.delete(platform)
update_workspace_descriptor(platform.workspace)
session.commit()
return platform.as_dict()
def delete(ws_id: int, project_id: int, remote_repo_name: str, organization_name: str = None):
"""
Deletes given project on remote repository
:param project_id:
:param ws_id: Workspace of the project
:param remote_repo_name: Remote repository name
:param organization_name: Optional parameter to specify the organization / login
:return:
"""
if organization_name is None:
owner = session['user_data']['login']
else:
owner = organization_name
sql_session = db_session()
project = get_project(ws_id, project_id, sql_session)
url_decode = parse.urlparse(project.repo_url)
if _repo_name_from_url(url_decode) == remote_repo_name:
result = _do_delete(owner, remote_repo_name)
if result.status_code == 204:
project.repo_url = None
sql_session.commit()
return create_info_dict("Successfully deleted")
else:
sql_session.rollback()
return create_info_dict(result.text, exitcode=1)
raise InvalidArgument("The given repo name does not correspond to the remote repository name")
def query_private_nsfs(ws_id, vendor, name, version, is_vnf):
"""
Finds a function in the private catalogue
:param ws_id: The workspace ID
:param is_vnf: if descriptor is a VNF
:param vendor: the descriptors vendor
:param name: the descriptors name
:param version: the descriptors version
:return: The requested descriptor if found, None if nothing found
"""
session = db_session()
if is_vnf:
descriptor = session.query(PrivateFunction).filter(PrivateFunction.name == name and
PrivateFunction.vendor == vendor and
PrivateFunction.version == version and
PrivateFunction.workspace.id == ws_id and
PrivateFunction.workspace.owner == get_user(
session['user_data'])).first()
else:
descriptor = session.query(PrivateService).filter(
PrivateService.name == name and
PrivateService.vendor == vendor and
PrivateService.version == version and
PrivateFunction.workspace.id == ws_id and
PrivateFunction.workspace.owner == get_user(session['user_data'])).first()
return descriptor
def delete_function(ws_id: int, project_id: int, function_id: int) -> dict:
"""
Deletes the function
:param ws_id:
:param project_id:
:param function_id:
:return: the deleted function
"""
session = db_session()
function = session.query(Function). \
join(Project). \
join(Workspace). \
filter(Workspace.id == ws_id). \
filter(Project.id == project_id). \
filter(Function.id == function_id).first()
if function is not None:
session.delete(function)
else:
session.rollback()
raise NotFound("Function with id " + function_id + " does not exist")
try:
os.remove(get_file_path("vnf", function))
except:
session.rollback()
logger.exception("Could not delete function:")
raise
session.commit()
return function.as_dict()
def save_image_file(ws_id, project_id, function_id, file):
"""
Saves the vnf image file into the vnfs folder
:param ws_id: The workspace ID
:param project_id: The project ID
:param function_id: The function ID
:param file: The image file
:return: A success message
"""
if file.filename == '':
raise InvalidArgument("No file attached!")
if file:
filename = secure_filename(file.filename)
session = db_session()
function = session.query(Function). \
join(Project). \
join(Workspace). \
filter(Workspace.id == ws_id). \
filter(Project.id == project_id). \
filter(Function.id == function_id).first()
if function is not None:
file_path = get_file_path("vnf", function)
file_path = file_path.replace(get_file_name(function), filename)
file.save(file_path)
return "File {} successfully uploaded!".format(filename)
else:
raise NotFound("Function with id " + function_id +
" does not exist")
def update_project(project_data, project_id):
"""
Update the Project
:param project_data: The project Data
:param project_id: The project ID to update
:return: The updated project descriptor
"""
session = db_session()
project = session.query(Project).filter(Project.id == project_id).first()
if project is None:
raise NotFound("Project with id {} could not be found".format(project_id))
# Update name
if 'name' in project_data and project_data['name'] != project.name:
if os.path.exists(get_project_path(project.workspace.path, project.rel_path)):
new_name = shlex.quote(project_data['name'])
old_path = get_project_path(project.workspace.path, project.rel_path)
new_path = rreplace(old_path, project.name, new_name, 1)
if os.path.exists(new_path):
raise NameConflict("Invalid name parameter, workspace '{}' already exists".format(new_name))
# Do not allow move directories outside of the workspaces_dir
if not new_path.startswith(WORKSPACES_DIR):
raise Exception("Invalid path parameter, you are not allowed to break out of {}".format(WORKSPACES_DIR))
else:
# Move the directory
shutil.move(old_path, new_path)
project.name = new_name
project.rel_path = new_name
set_data(project, project_data)
sync_project_descriptor(project)
db_session.commit()
return project.as_dict()
def create_service_on_platform(ws_id, platform_id, service_data):
"""
Deploys the service on the referenced Platform
:param ws_id:
:param platform_id:
:param service_data:
:return: A message if the function was deployed successfully
"""
service_id = int(service_data['id'])
session = db_session()
try:
workspace = session.query(Workspace).filter(Workspace.id == ws_id).first()
project = session.query(Project). \
join(Service). \
filter(Project.services.any(Service.id == service_id)). \
filter(Project.workspace == workspace). \
first() # type: Project
if not len(project.services) == 1:
raise InvalidArgument(
"Project must have exactly one service "
"to push to platform. Number of services: {}".format(
len(project.services)))
platform = session.query(Platform).filter(Platform.id == platform_id). \
filter(Platform.workspace == workspace).first()
package_path = pack_project(project)
service_uuid = push_to_platform(package_path, platform)
logger.info("Pushed to platform: " + str(service_uuid))
# deploy to private catalogue
service = project.services[0].as_dict()
publish_private_nsfs(ws_id, service["descriptor"], is_vnf=False)
publish_referenced_functions(ws_id, project.id, service["descriptor"])
return {'message': 'Deployed successfully: {}'.format(str(service_uuid))}
finally:
session.commit()
def save_image_file(ws_id, project_id, function_id, file):
"""
Saves the vnf image file into the vnfs folder
:param ws_id: The workspace ID
:param project_id: The project ID
:param function_id: The function ID
:param file: The image file
:return: A success message
"""
if file.filename == '':
raise InvalidArgument("No file attached!")
if file:
filename = secure_filename(file.filename)
session = db_session()
function = session.query(Function). \
join(Project). \
join(Workspace). \
filter(Workspace.id == ws_id). \
filter(Project.id == project_id). \
filter(Function.id == function_id).first()
if function is not None:
file_path = get_file_path("vnf", function)
file_path = file_path.replace(get_file_name(function), filename)
file.save(file_path)
return "File {} successfully uploaded!".format(filename)
else:
raise NotFound("Function with id " + function_id + " does not exist")
def test_pack_project(self):
session = db_session()
project = session.query(Project).filter(Project.id == self.pjid).first()
package_path = publishutil.pack_project(project)
self.assertTrue(os.path.exists(package_path))
self.assertTrue(os.path.isfile(package_path))
# create another service in project
request_dict = get_sample_ns("servicename", "vendorname", "0.1")
response = self.app.post(
'/' + WORKSPACES + '/' + self.wsid + '/' + PROJECTS + '/' + str(self.pjid) + '/' + SERVICES + '/',
data=json.dumps(request_dict),
content_type='application/json')
# should fail because the project name is invalid
project = session.query(Project).filter(Project.id == self.pjid).first()
try:
publishutil.pack_project(project)
except Exception as err:
self.assertTrue(isinstance(err, PackException))
# should fail as only one service can be packaged
project = session.query(Project).filter(Project.id == self.pjid2).first()
try:
publishutil.pack_project(project)
except Exception as err:
self.assertTrue(isinstance(err, PackException))
session.commit()
def delete(ws_id: int, project_id: int, remote_repo_name: str, organization_name: str = None):
"""
Deletes given project on remote repository
:param project_id:
:param ws_id: Workspace of the project
:param remote_repo_name: Remote repository name
:param organization_name: Optional parameter to specify the organization / login
:return: a dictionary containing the result of the operation
"""
if organization_name is None:
owner = session['user_data']['login']
else:
owner = organization_name
sql_session = db_session()
project = get_project(ws_id, project_id, sql_session)
url_decode = parse.urlparse(project.repo_url)
if _repo_name_from_url(url_decode) == remote_repo_name:
result = _do_delete(owner, remote_repo_name)
if result.status_code == 204:
project.repo_url = None
sql_session.commit()
return create_info_dict("Successfully deleted")
else:
sql_session.rollback()
return create_info_dict(result.text, exitcode=1)
raise InvalidArgument("The given repo name does not correspond to the remote repository name")
def update_platform(workspace_id: int, platform_id: int, platform_data) -> dict:
"""
Update the platform entry
:param workspace_id:
:param platform_id:
:return: The updated platform definition
"""
platform_name = shlex.quote(platform_data['name'])
platform_url = shlex.quote(platform_data['url'])
session = db_session()
workspace = session.query(Workspace).filter(Workspace.id == workspace_id).first()
if workspace is None:
raise NotFound("workspace with id {} could not be found".format(workspace_id))
platform = session.query(Platform). \
filter(Platform.workspace == workspace). \
filter(Platform.id == platform_id). \
first()
if platform is None:
raise NotFound("Platform with id {} could not be found".format(platform_id))
if platform_name != platform.name:
existing_platforms = session.query(Platform). \
filter(Platform.workspace == workspace). \
filter(Platform.name == platform_data['name']). \
all()
if len(existing_platforms) > 0:
raise NameConflict("Platform with name {} already exists".format(platform_data['name']))
platform.name = platform_name
platform.url = platform_url
update_workspace_descriptor(platform.workspace)
session.commit()
return platform.as_dict()
def create_workspace(login: str, workspace_data: dict) -> dict:
"""
Creates a workspace (on disk and in the database) from the given workspace data
:param workspace_data: The workspace configuration data
:return: The created workspace
"""
wsName = shlex.quote(workspace_data["name"])
session = db_session()
# test if ws Name exists in database
user = get_user(login)
existingWorkspaces = list(session.query(Workspace)
.filter(Workspace.owner == user)
.filter(Workspace.name == wsName))
if len(existingWorkspaces) > 0:
raise NameConflict("Workspace with name " + wsName + " already exists")
wsPath = path.join(WORKSPACES_DIR, user.name, wsName)
# prepare db insert
try:
ws = Workspace(name=wsName, path=wsPath, owner=user)
session.add(ws)
if 'platforms' in workspace_data:
for platform in workspace_data['platforms']:
ptf = Platform(platform['name'], platform['url'], True, ws)
if 'token' in platform:
ptf.token_path = create_token_file(platform['token'])
session.add(ptf)
test_url(platform['name'], platform['url'] + "/api/v2/packages")
if 'catalogues' in workspace_data:
for catalogue in workspace_data['catalogues']:
session.add(Catalogue(catalogue['name'], catalogue['url'], True, ws))
test_url(catalogue['name'], catalogue['url'])
except Exception as e:
logger.exception(e)
session.rollback()
raise
# create workspace on disk
proc = Popen(['son-workspace', '--init', '--workspace', wsPath], stdout=PIPE, stderr=PIPE)
out, err = proc.communicate()
exitcode = proc.returncode
if out.decode().find('existing') >= 0:
workspace_exists = True
else:
workspace_exists = False
if exitcode == 0 and not workspace_exists:
update_workspace_descriptor(ws)
session.commit()
return ws.as_dict()
else:
session.rollback()
if workspace_exists:
raise NameConflict(out.decode())
raise Exception(err, out)
def get_service(service_id):
"""Returns a service with the given service id"""
session = db_session()
service = session.query(Service).filter(Service.id == service_id).first()
# Check if the function could be retrieved
if not service:
raise NotFound("Service with id '{}' does not exist".format(service_id))
return service
def get_project(project_id):
""" Retrieves the project which matches the given project id. Otherwise it raises NotFound Exception """
session = db_session()
current_project = session.query(Project).filter(Project.id == project_id).first()
if current_project is not None:
return current_project
else:
raise NotFound("Project {} does not exist".format(project_id))
def get_function(function_id):
""" Returns a function with the given function id"""
session = db_session()
function = session.query(Function).filter(Function.id == function_id).first()
# Check if the function could be retrieved
if not function:
raise NotFound("Function with id '{}' does not exist".format(function_id))
return function
def get_project(ws_id, pj_id):
session = db_session()
project = session.query(Project).join(Workspace).filter(Workspace.id == ws_id).filter(Project.id == pj_id).first()
session.commit()
if project:
return project.as_dict()
else:
raise NotFound("No project with id {} could be found".format(pj_id))
def get_service(service_id):
"""Returns a service with the given service id"""
session = db_session()
service = session.query(Service).filter(Service.id == service_id).first()
# Check if the function could be retrieved
if not service:
raise NotFound("Service with id '{}' does not exist".format(service_id))
return service
def get_function(function_id):
""" Returns a function with the given function id"""
session = db_session()
function = session.query(Function).filter(Function.id == function_id).first()
# Check if the function could be retrieved
if not function:
raise NotFound("Function with id '{}' does not exist".format(function_id))
return function
def create_service(ws_id: int, project_id: int, service_data: dict) -> dict:
"""
Creates a service in the given project
:param ws_id: The Workspace of the project
:param project_id: The Project of the Service
:param service_data: the service descriptor
:return: The created service descriptor
"""
session = db_session()
project = session.query(Project).filter_by(id=project_id).first()
if project:
# Retrieve post parameters
try:
service_name = shlex.quote(service_data['descriptor']["name"])
vendor_name = shlex.quote(service_data['descriptor']["vendor"])
version = shlex.quote(service_data['descriptor']["version"])
except KeyError as ke:
raise InvalidArgument("Missing key {} in service data".format(
str(ke)))
existing_services = list(
session.query(Service).join(Project).join(Workspace).filter(
Workspace.id == ws_id).filter(
Service.project == project).filter(
Service.name == service_name).filter(
Service.vendor == vendor_name).filter(
Service.version == version))
if len(existing_services) > 0:
raise NameConflict(
"A service with this name/vendor/version already exists")
# validate service descriptor
workspace = session.query(Workspace).filter(
Workspace.id == ws_id).first()
validate_service_descriptor(workspace.ns_schema_index,
service_data["descriptor"])
# Create db object
service = Service(name=service_name,
vendor=vendor_name,
version=version,
project=project,
descriptor=json.dumps(service_data["descriptor"]),
meta=json.dumps(service_data["meta"]))
session.add(service)
try:
write_ns_vnf_to_disk("nsd", service)
except:
logger.exception("Could not create service:")
session.rollback()
raise
session.commit()
return service.as_dict()
else:
session.rollback()
raise NotFound("Project with id '{}‘ not found".format(project_id))
def update_function(ws_id: int, prj_id: int, func_id: int, func_data: dict) -> dict:
"""
Update the function descriptor
:param ws_id:
:param prj_id:
:param func_id:
:param func_data:
:return: The updated function descriptor
"""
session = db_session()
ws = session.query(Workspace).filter(Workspace.id == ws_id).first()
validate_vnf(ws.vnf_schema_index, func_data)
# test if ws Name exists in database
function = session.query(Function). \
join(Project). \
join(Workspace). \
filter(Workspace.id == ws_id). \
filter(Project.id == prj_id). \
filter(Function.id == func_id).first()
if function is None:
session.rollback()
raise NotFound("Function with id {} does not exist".format(func_id))
function.descriptor = json.dumps(func_data)
old_file_name = get_file_path("vnf", function)
old_folder_path = old_file_name.replace(get_file_name(function), "")
try:
function.name = shlex.quote(func_data["name"])
function.vendor = shlex.quote(func_data["vendor"])
function.version = shlex.quote(func_data["version"])
except KeyError as ke:
session.rollback()
raise InvalidArgument("Missing key {} in function data".format(str(ke)))
try:
new_file_name = get_file_path("vnf", function)
new_folder_path = new_file_name.replace(get_file_name(function), "")
if old_folder_path != new_folder_path:
# move old files to new location
os.makedirs(new_folder_path)
for file in os.listdir(old_folder_path):
if not old_file_name == os.path.join(old_folder_path, file): # don't move descriptor yet
shutil.move(os.path.join(old_folder_path, file), os.path.join(new_folder_path, file))
if not new_file_name == old_file_name:
shutil.move(old_file_name, new_file_name)
if old_folder_path != new_folder_path:
# cleanup old folder
shutil.rmtree(old_folder_path)
write_ns_vnf_to_disk("vnf", function)
except:
session.rollback()
logger.exception("Could not update descriptor file:")
raise
session.commit()
return function.as_dict()
def create_workspace(login: str, workspace_data: dict) -> dict:
"""
Creates a workspace (on disk and in the database) from the given workspace data
:param workspace_data: The workspace configuration data
:return: The created workspace
"""
wsName = shlex.quote(workspace_data["name"])
session = db_session()
# test if ws Name exists in database
user = get_user(login)
existingWorkspaces = list(
session.query(Workspace).filter(Workspace.owner == user).filter(
Workspace.name == wsName))
if len(existingWorkspaces) > 0:
raise NameConflict("Workspace with name " + wsName + " already exists")
wsPath = path.join(WORKSPACES_DIR, user.name, wsName)
# prepare db insert
try:
ws = Workspace(name=wsName, path=wsPath, owner=user)
session.add(ws)
if 'platforms' in workspace_data:
for platform in workspace_data['platforms']:
session.add(Platform(platform['name'], platform['url'], ws))
test_url(platform['name'], platform['url'] + "/packages")
if 'catalogues' in workspace_data:
for catalogue in workspace_data['catalogues']:
session.add(Catalogue(catalogue['name'], catalogue['url'], ws))
test_url(catalogue['name'], catalogue['url'])
except:
logger.exception()
session.rollback()
raise
# create workspace on disk
proc = Popen(['son-workspace', '--init', '--workspace', wsPath],
stdout=PIPE,
stderr=PIPE)
out, err = proc.communicate()
exitcode = proc.returncode
if out.decode().find('existing') >= 0:
workspace_exists = True
else:
workspace_exists = False
if exitcode == 0 and not workspace_exists:
synchronize_workspace_descriptor(ws, session)
session.commit()
return ws.as_dict()
else:
session.rollback()
if workspace_exists:
raise NameConflict(out.decode())
raise Exception(err, out)
def get_project(project_id):
""" Retrieves the project which matches the given project id. Otherwise it raises NotFound Exception """
session = db_session()
current_project = session.query(Project).filter(
Project.id == project_id).first()
if current_project is not None:
return current_project
else:
raise NotFound("Project {} does not exist".format(project_id))
def create_service(ws_id: int, project_id: int, service_data: dict) -> dict:
"""
Creates a service in the given project
:param ws_id: The Workspace of the project
:param project_id: The Project of the Service
:param service_data: the service descriptor
:return: The created service descriptor
"""
session = db_session()
project = session.query(Project).filter_by(id=project_id).first()
if project:
# Retrieve post parameters
try:
service_name = shlex.quote(service_data['descriptor']["name"])
vendor_name = shlex.quote(service_data['descriptor']["vendor"])
version = shlex.quote(service_data['descriptor']["version"])
except KeyError as ke:
raise InvalidArgument("Missing key {} in service data".format(str(ke)))
existing_services = list(session.query(Service)
.join(Project)
.join(Workspace)
.filter(Workspace.id == ws_id)
.filter(Service.project == project)
.filter(Service.name == service_name)
.filter(Service.vendor == vendor_name)
.filter(Service.version == version))
if len(existing_services) > 0:
raise NameConflict("A service with this name/vendor/version already exists")
# validate service descriptor
workspace = session.query(Workspace).filter(Workspace.id == ws_id).first()
validate_service_descriptor(workspace.ns_schema_index, service_data["descriptor"])
# Create db object
service = Service(name=service_name,
vendor=vendor_name,
version=version,
project=project,
descriptor=json.dumps(service_data["descriptor"]),
meta=json.dumps(service_data["meta"]))
session.add(service)
try:
write_ns_vnf_to_disk("nsd", service)
except:
logger.exception("Could not create service:")
session.rollback()
raise
session.commit()
return service.as_dict()
else:
session.rollback()
raise NotFound("Project with id '{}‘ not found".format(project_id))
def get_platforms(workspace_id: int) -> list:
"""
Get a list of platforms for this workspace
:param workspace_id:
:return:
"""
session = db_session()
platforms = session.query(Platform).filter(Platform.workspace_id == workspace_id).all()
session.commit()
return list(map(lambda x: x.as_dict(), platforms))
def get_projects(ws_id: int) -> list:
"""
Get a list of projects in this workspace
:param ws_id:
:return:
"""
session = db_session()
projects = session.query(Project).join(Workspace).filter(Workspace.id == ws_id).all()
session.commit()
return list(map(lambda x: x.as_dict(), projects))
def get_catalogues(workspace_id):
"""
Retrieves all catalogues of a specific workspace
:param workspace_id:
:return:
"""
session = db_session()
catalogues = session.query(Catalogue).filter(Catalogue.workspace_id == workspace_id).all()
session.commit()
return list(map(lambda x: x.as_dict(), catalogues))
def publish_referenced_functions(ws_id, proj_id, descriptor):
vnfs = descriptor["network_functions"]
session = db_session()
for vnf in vnfs:
function = session.query(Function).join(Project). \
filter(Project.id == proj_id). \
filter(Function.name == vnf['vnf_name']). \
filter(Function.vendor == vnf["vnf_vendor"]). \
filter(Function.version == vnf["vnf_version"]).first()
publish_private_nsfs(ws_id, function.as_dict()["descriptor"], True)
def publish_referenced_functions(ws_id, proj_id, descriptor):
vnfs = descriptor["network_functions"]
session = db_session()
for vnf in vnfs:
function = session.query(Function).join(Project). \
filter(Project.id == proj_id). \
filter(Function.name == vnf['vnf_name']). \
filter(Function.vendor == vnf["vnf_vendor"]). \
filter(Function.version == vnf["vnf_version"]).first()
publish_private_nsfs(ws_id, function.as_dict()["descriptor"], True)
def get_workspace(ws_id: int) -> Workspace:
"""
Returns the workspace model of the given workspace
:param ws_id:
:return:
"""
workspace = db_session().query(Workspace).filter(Workspace.id == ws_id).first()
if not workspace:
raise NotFound("Could not find workspace with id {}".format(ws_id))
return workspace
def create_commit_and_push(ws_id: int, project_id: int, remote_repo_name: str):
"""
Creates a remote GitHub repository named remote_repo_name and pushes given git project into it.
:param ws_id: Workspace ID
:param project_id: Project ID to create and push it
:param remote_repo_name: Remote repository name
:return:
"""
database_session = db_session()
try:
project = get_project(ws_id, project_id, database_session)
# curl -H "Authorization: token [TOKEN]" -X POST https://api.github.com/user/repos --data '{"name":"repo_name"}'
repo_data = {'name': remote_repo_name}
request = requests.post(Github.API_URL + Github.API_CREATE_REPO_REL,
json=repo_data,
headers=create_oauth_header())
# Handle exceptions
if request.status_code != 201:
# Repository already exists
if request.status_code == 422:
raise NameConflict(
"Repository with name {} already exist on GitHub".format(
remote_repo_name))
raise Exception("Unhandled status_code: {}\n{}".format(
request.status_code, request.text))
# Get git url and commit to db
data = json.loads(request.text)
git_url = data['svn_url']
project.repo_url = git_url
database_session.commit()
except Exception:
database_session.rollback()
raise
# Try to push project
try:
# Give github some time to see created repo
# (dirty hack)
time.sleep(0.5)
return commit_and_push(ws_id, project_id, "Initial commit")
except Exception:
# Delete newly created repository if commit and push failed.
result = requests.delete(build_github_delete(
session['user_data']['login'], remote_repo_name),
headers=create_oauth_header())
# Reraise
raise
def get_workspaces(login: str) -> list:
"""
Get all workspaces for the current user
:return: A list wof workspace dictionaries
"""
session = db_session()
user = get_user(login)
workspaces = session.query(Workspace). \
filter(Workspace.owner == user).all()
session.commit()
return list(map(lambda x: x.as_dict(), workspaces))
def get_platforms(workspace_id: int) -> list:
"""
Get a list of platforms for this workspace
:param workspace_id:
:return:
"""
session = db_session()
platforms = session.query(Platform).filter(
Platform.workspace_id == workspace_id).all()
session.commit()
return list(map(lambda x: x.as_dict(), platforms))
def get_workspace(ws_id: int) -> Workspace:
"""
Returns the workspace model of the given workspace
:param ws_id: The workspace ID
:return: The corresponding workspace model
"""
workspace = db_session().query(Workspace).filter(Workspace.id == ws_id).first()
if not workspace:
raise NotFound("Could not find workspace with id {}".format(ws_id))
return workspace
def get_workspaces(login: str) -> list:
"""
Get all workspaces for the current user
:return: A list wof workspace dictionaries
"""
session = db_session()
user = get_user(login)
workspaces = session.query(Workspace). \
filter(Workspace.owner == user).all()
session.commit()
return list(map(lambda x: x.as_dict(), workspaces))
def get_functions(ws_id: int, project_id: int) -> list:
"""
Get a list of all functions
:param ws_id: The workspace ID
:param project_id: The project id
:return:
"""
session = db_session()
functions = session.query(Function).join(Project).join(Workspace). \
filter(Workspace.id == ws_id). \
filter(Function.project_id == project_id).all()
return list(map(lambda x: x.as_dict(), functions))
def get_catalogue(catalogue_id):
"""
Retrieves a catalogue by its id
:param catalogue_id:
:return:
"""
session = db_session()
catalogue = session.query(Catalogue).filter(Catalogue.id == catalogue_id).first()
session.commit()
if catalogue is None:
raise NotFound("catalogue with id {} could not be found".format(catalogue_id))
return catalogue.as_dict()
def get_user(login: str):
"""
Gets the user from the Database if it exists or
creates a new user in the Database using the
login data from the session. If the database does
not yet have the full user Data it is queried
from Github using the access Token
:return: The database user model
"""
session = db_session()
user_name = shlex.quote(login)
user = session.query(User).filter(User.name == user_name).first()
# for now: if user does not exist we will create a user
# (that has no access to anything he has not created)
if user is None:
user = User(name=user_name)
session.add(user)
if user.email is None:
headers = {
"Accept": "application/json",
"Authorization": "token " + flask.session['access_token']
}
if 'github-orgs' in get_config():
# request user orgs
result = requests.get(
flask.session['user_data']['organizations_url'],
headers=headers)
orgs = json.loads(result.text)
valid_org_found = False
for org in orgs:
if org['login'] in get_config()['github-orgs']:
valid_org_found = True
break
if not valid_org_found:
raise UnauthorizedException(
"No valid github org found for this user: "******"Please ask the admin of this server to add "
"you to his organization or add your "
"orgaization to the list of valid organizations")
result = requests.get('https://api.github.com/user/emails',
headers=headers)
user_emails = json.loads(result.text)
for email in user_emails:
if email['primary']:
user.email = shlex.quote(email['email'])
break
session.commit()
return user
