def load_create_project():
""" Load Page to create a project
When launch the program, http://localhost:9002/create-project -> show page create-project.html
There are 2 steps to do when load this page
Step 1: Init all global variables when load this page, make sure that:
- List of components in workflow is empty
- List of links in workflow is empty
- Number of components is 0
Step 2: Set up project folder
- If /{user.home}/Documents/waves_project_space folder doesn't exist, create the folder
- Create a tmp folder for temporal saving the uploaded files,
the temporal files will be moved the project folder once user clicked submit project button
- Copy the docker-standalone folder into tmp folder,
when user submit the project, it will also be moved to project folder
to be used for building the Docker image and containers
"""
# Initial global variables
global workflow, clock, cmptCounter
workflow.cmpt_list = []
workflow.link_list = []
cmptCounter = 0
# Init Project Space folder
handler = FileHandler()
handler.setup_project_space_folder()
return render_template('create-project.html', clock=clock)
def delete_project():
""" Delete project """
pname = request.form['name']
handler = FileHandler()
handler.delete_project(pname)
return "{}"
def load_project_space():
""" Load Project Space
Read all project names from project space local folder /{user.home}/Documents/waves_project_space
The project names are the list of folder names
"""
handler = FileHandler()
pnames = handler.get_list_project()
projects = []
for pname in pnames:
project = handler.get_project_as_dict(pname)
projects.append(project)
return render_template('project-space.html', projects=projects)
def upload_single_file():
""" Upload single files
This method is used for static feed
The file will be saved at tmp folder at /{user.home}/Documents/waves_project_space/tmp/Data/StaticFeed
"""
handler = FileHandler()
files = request.files.getlist("location")
if len(files) == 1:
file = files[0]
handler.save_static_feed_file(file)
return '{}'
def load_project_details(name):
""" Load the details of project
:param name: name of the project
:return:
"""
handler = FileHandler()
# Load project information as json for processingin UI
project = handler.get_project_as_dict(name)
workflow_ui = handler.get_workflow_ui(name).rstrip()
trig = handler.get_project_as_trig(name)
return render_template('project-details.html',
project=project,
workflow_ui=workflow_ui,
trig=trig)
def load_project_component_settings(name):
""" Get the project all component's settings
:param name: name of the project
:return: Json format of the all the components settings
"""
handler = FileHandler()
project = handler.get_project_as_dict(name)
cmpt_id = request.form['id']
cmpt_list = project['workflow']['cmpt_list']
settings = {}
for cmpt in cmpt_list:
if cmpt['id'] == cmpt_id:
settings = cmpt['settings']
return json.dumps(settings)
def modify_project(name):
""" Modify the project
:param name: name of the project to be modified
:return:
"""
handler = FileHandler()
project = handler.get_project(name)
global projectInfo, clusterInfo, workflow, metrics, clock
# Modify the project by re-create workflow
projectInfo = project.projectInfo
clusterInfo = project.clusterInfo
workflow = Workflow()
metrics = project.metrics
clock = project.clock
return render_template('modify-project.html', project=project, clock=clock)
def upload_multi_files():
""" Upload Multi File
The name of input may be location or inputFolder, need to retrieve them independently.
For raw source, the name is location
For rdf source, the name is inputFolder
"""
handler = FileHandler()
if request.files.getlist("inputFolder") == []:
# If not inputFolder, uploaded file -> raw source
files = request.files.getlist("location")
for file in files:
handler.save_raw_source_file(file)
else:
# Files are for rdf source
files = request.files.getlist("inputFolder")
for file in files:
handler.save_rdf_source_file(file)
return '{}'
def save_component_settings():
""" Save component settings when user click on 'Save Settings' button.
Retrieve from key-value form data and set the component settings
The component's settings is in fact a python dictionary.
For some special keys, we will make some modifications
- cmpt_id : do not need to save it in settings, so pass it
- location : Since we deploy the program at container, the location should be
the location at the container.
Check if the location is URL or filename, if it's not a url location:
If raw source, make the location as "/usr/local/srcFolder/"
If others, "/usr/local/srcFiles/" + filename
- outputFile : Add prefix of "/usr/local/"
- inputFolder : Add prefix of '/usr/local/srcFolder'
Set component settings.
"""
global workflow
cmpt_id = request.form['cmpt_id']
cmpt_type = request.form['cmpt_type']
print(request.form)
cmpt = workflow.get_cmpt_by_id(cmpt_id)
settings = {}
settings['id'] = cmpt.settings['id']
for key, value in request.form.items():
# Do not save the component id
if key != 'cmpt_id':
settings[key] = value
# RawSource data location in docker
if cmpt_type == 'RawSource' and key == 'location':
# Check if it's url
# If it's not a url, means that it's a file source
if validators.url(value) != True:
# Save the location as the folder path in docker container
settings[key] = "/opt/data/csv"
# RdfSource data location in docker
if cmpt_type == 'Source' and key == 'inputFolder':
# Check if it's url
# If it's not a url, means that it's a file source
if validators.url(value) != True:
# Save the location as the folder path in docker container
settings[key] = "/opt/data/rdfSource"
# Static Feed data location in docker
if cmpt_type == 'RdfFeed' and key == 'location':
fhandler = FileHandler()
settings[
key] = "/opt/data/rdf/" + fhandler.get_static_feed_filename()
# Sink Location
if cmpt_type == 'Sink' and key == 'outputFile':
settings[key] = value.split('\\')[-1]
workflow.save_component_settings(cmpt_id, settings)
return '{}'
def create_project():
"""When user clicked on the button, retrieve the project information and build the project
Project information contains 5 parts:
- ProjectInfo : Information is obtained from the submitted form data
- ClusterInfo : Use default cluster IP and host information
- Workflow :
Workflow information is updated while adding, removing, save settings
of the components etc.
The information is already saved in workflow global variable,
no need to retrieve from the form data
- Metrics : Information is obtained from the submitted form data
- Clock : Information already saved in global variable clock
when user click on the button of saving clock settings
Step 1: Retrieve project information
- Get global project information from form data
- Use default cluster information
- Use already updated workflow components and links information
- Get metrics information form form data
- Use already updated clock information
Step 2: Save workflow UI
- The workflow_ui is the HTML tags '<div>...</div>' which saves the UI level's
workflow structure
- When view the details of project (the workflow structure, cmpt settings etc.)
The front end will load the workflow_ui from back end and replace with
the empty workflow space.
Step 3: Parse the project information into TriG
Step 4: Set up project folder
- New a folder with the name of project name at /{user.root}/Documents/waves_project_space/
- Generate TriG file and put it into docker configuration folder
- Put all docker folder, srcFiles, srcFolder into the project folder
- Save the workflow_ui HTML tags into a file
Notes:
- Saving the HTML tags is a compromised solution. I have to admit that re-draw the
workflow according to their saved locations and settings is the best solution.
However, the jsPlumb API always has some bugs when re-drawing the workflow i.e.:
- The connection points are not correct.
Source point starts from the top-left of screen but not component
- Can not dragging a link from re-drawwed component
- etc.
- To remedy this problem, I have to save the HTML tags. When repaint the workflow, just
replace it.
- When user want to modify the project, it has to re-create the project and can not modify
directly on the existing workflow.
"""
global projectInfo, clusterInfo, workflow, metrics, clock
print(request.form)
# Receive form data
projectInfo.name = request.form['name']
projectInfo.description = request.form['description']
projectInfo.license = request.form['license']
projectInfo.version = request.form['version']
projectInfo.createdAt = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
metrics.frequency = request.form['frequency']
metrics.reporters = request.form['reporters']
workflow_ui = request.form['workflow_ui']
# Create a new project
project = Project(projectInfo, clusterInfo, workflow, metrics, clock)
# Parse project -> TriG
resp = {}
resp['trig'] = project.parse_trig()
# Create the project folder
handler = FileHandler()
handler.setup_project_folder(project)
handler.save_workflow_ui(workflow_ui, projectInfo.name)
return json.dumps(resp)