def load_create_project(): """ Load Page to create a project When launch the program, http://localhost:9002/create-project -> show page create-project.html There are 2 steps to do when load this page Step 1: Init all global variables when load this page, make sure that: - List of components in workflow is empty - List of links in workflow is empty - Number of components is 0 Step 2: Set up project folder - If /{user.home}/Documents/waves_project_space folder doesn't exist, create the folder - Create a tmp folder for temporal saving the uploaded files, the temporal files will be moved the project folder once user clicked submit project button - Copy the docker-standalone folder into tmp folder, when user submit the project, it will also be moved to project folder to be used for building the Docker image and containers """ # Initial global variables global workflow, clock, cmptCounter workflow.cmpt_list = [] workflow.link_list = [] cmptCounter = 0 # Init Project Space folder handler = FileHandler() handler.setup_project_space_folder() return render_template('create-project.html', clock=clock)
def delete_project(): """ Delete project """ pname = request.form['name'] handler = FileHandler() handler.delete_project(pname) return "{}"
def load_project_space(): """ Load Project Space Read all project names from project space local folder /{user.home}/Documents/waves_project_space The project names are the list of folder names """ handler = FileHandler() pnames = handler.get_list_project() projects = [] for pname in pnames: project = handler.get_project_as_dict(pname) projects.append(project) return render_template('project-space.html', projects=projects)
def upload_single_file(): """ Upload single files This method is used for static feed The file will be saved at tmp folder at /{user.home}/Documents/waves_project_space/tmp/Data/StaticFeed """ handler = FileHandler() files = request.files.getlist("location") if len(files) == 1: file = files[0] handler.save_static_feed_file(file) return '{}'
def load_project_details(name): """ Load the details of project :param name: name of the project :return: """ handler = FileHandler() # Load project information as json for processingin UI project = handler.get_project_as_dict(name) workflow_ui = handler.get_workflow_ui(name).rstrip() trig = handler.get_project_as_trig(name) return render_template('project-details.html', project=project, workflow_ui=workflow_ui, trig=trig)
def load_project_component_settings(name): """ Get the project all component's settings :param name: name of the project :return: Json format of the all the components settings """ handler = FileHandler() project = handler.get_project_as_dict(name) cmpt_id = request.form['id'] cmpt_list = project['workflow']['cmpt_list'] settings = {} for cmpt in cmpt_list: if cmpt['id'] == cmpt_id: settings = cmpt['settings'] return json.dumps(settings)
def modify_project(name): """ Modify the project :param name: name of the project to be modified :return: """ handler = FileHandler() project = handler.get_project(name) global projectInfo, clusterInfo, workflow, metrics, clock # Modify the project by re-create workflow projectInfo = project.projectInfo clusterInfo = project.clusterInfo workflow = Workflow() metrics = project.metrics clock = project.clock return render_template('modify-project.html', project=project, clock=clock)
def upload_multi_files(): """ Upload Multi File The name of input may be location or inputFolder, need to retrieve them independently. For raw source, the name is location For rdf source, the name is inputFolder """ handler = FileHandler() if request.files.getlist("inputFolder") == []: # If not inputFolder, uploaded file -> raw source files = request.files.getlist("location") for file in files: handler.save_raw_source_file(file) else: # Files are for rdf source files = request.files.getlist("inputFolder") for file in files: handler.save_rdf_source_file(file) return '{}'
def save_component_settings(): """ Save component settings when user click on 'Save Settings' button. Retrieve from key-value form data and set the component settings The component's settings is in fact a python dictionary. For some special keys, we will make some modifications - cmpt_id : do not need to save it in settings, so pass it - location : Since we deploy the program at container, the location should be the location at the container. Check if the location is URL or filename, if it's not a url location: If raw source, make the location as "/usr/local/srcFolder/" If others, "/usr/local/srcFiles/" + filename - outputFile : Add prefix of "/usr/local/" - inputFolder : Add prefix of '/usr/local/srcFolder' Set component settings. """ global workflow cmpt_id = request.form['cmpt_id'] cmpt_type = request.form['cmpt_type'] print(request.form) cmpt = workflow.get_cmpt_by_id(cmpt_id) settings = {} settings['id'] = cmpt.settings['id'] for key, value in request.form.items(): # Do not save the component id if key != 'cmpt_id': settings[key] = value # RawSource data location in docker if cmpt_type == 'RawSource' and key == 'location': # Check if it's url # If it's not a url, means that it's a file source if validators.url(value) != True: # Save the location as the folder path in docker container settings[key] = "/opt/data/csv" # RdfSource data location in docker if cmpt_type == 'Source' and key == 'inputFolder': # Check if it's url # If it's not a url, means that it's a file source if validators.url(value) != True: # Save the location as the folder path in docker container settings[key] = "/opt/data/rdfSource" # Static Feed data location in docker if cmpt_type == 'RdfFeed' and key == 'location': fhandler = FileHandler() settings[ key] = "/opt/data/rdf/" + fhandler.get_static_feed_filename() # Sink Location if cmpt_type == 'Sink' and key == 'outputFile': settings[key] = value.split('\\')[-1] workflow.save_component_settings(cmpt_id, settings) return '{}'
def create_project(): """When user clicked on the button, retrieve the project information and build the project Project information contains 5 parts: - ProjectInfo : Information is obtained from the submitted form data - ClusterInfo : Use default cluster IP and host information - Workflow : Workflow information is updated while adding, removing, save settings of the components etc. The information is already saved in workflow global variable, no need to retrieve from the form data - Metrics : Information is obtained from the submitted form data - Clock : Information already saved in global variable clock when user click on the button of saving clock settings Step 1: Retrieve project information - Get global project information from form data - Use default cluster information - Use already updated workflow components and links information - Get metrics information form form data - Use already updated clock information Step 2: Save workflow UI - The workflow_ui is the HTML tags '<div>...</div>' which saves the UI level's workflow structure - When view the details of project (the workflow structure, cmpt settings etc.) The front end will load the workflow_ui from back end and replace with the empty workflow space. Step 3: Parse the project information into TriG Step 4: Set up project folder - New a folder with the name of project name at /{user.root}/Documents/waves_project_space/ - Generate TriG file and put it into docker configuration folder - Put all docker folder, srcFiles, srcFolder into the project folder - Save the workflow_ui HTML tags into a file Notes: - Saving the HTML tags is a compromised solution. I have to admit that re-draw the workflow according to their saved locations and settings is the best solution. However, the jsPlumb API always has some bugs when re-drawing the workflow i.e.: - The connection points are not correct. Source point starts from the top-left of screen but not component - Can not dragging a link from re-drawwed component - etc. - To remedy this problem, I have to save the HTML tags. When repaint the workflow, just replace it. - When user want to modify the project, it has to re-create the project and can not modify directly on the existing workflow. """ global projectInfo, clusterInfo, workflow, metrics, clock print(request.form) # Receive form data projectInfo.name = request.form['name'] projectInfo.description = request.form['description'] projectInfo.license = request.form['license'] projectInfo.version = request.form['version'] projectInfo.createdAt = datetime.now().strftime('%Y-%m-%d %H:%M:%S') metrics.frequency = request.form['frequency'] metrics.reporters = request.form['reporters'] workflow_ui = request.form['workflow_ui'] # Create a new project project = Project(projectInfo, clusterInfo, workflow, metrics, clock) # Parse project -> TriG resp = {} resp['trig'] = project.parse_trig() # Create the project folder handler = FileHandler() handler.setup_project_folder(project) handler.save_workflow_ui(workflow_ui, projectInfo.name) return json.dumps(resp)