def generate_project(code): """ generate project automatically without project path. :param code: project code. :return: project description. """ project_path = tempfile.mkdtemp() # generate project config file. project_config().dump_to_file(project_path + '/project.yaml') # generate project structure. os.makedirs(project_path + '/python_codes') os.makedirs(project_path + '/jar_dependencies') os.makedirs(project_path + '/resources') open(project_path + '/python_codes/__init__.py', 'w') fd, temp_file = tempfile.mkstemp(suffix='.py', dir=project_path + '/python_codes') with open(temp_file, 'w') as f: f.write(code) # generate project description. project_desc: ProjectDesc = get_project_description_from( project_path) project_desc.project_name = project_config().get_project_name() project_desc.project_config['entry_module_path'] = ( file_path_to_absolute_module(temp_file)).split('.')[-1] return project_desc
def test_project_upload_download_local(self): project_path = get_file_dir(__file__) project_desc = get_project_description_from(project_path + "/../") # blob_server.type = local blob_manager = BlobManagerFactory.get_blob_manager( project_desc.project_config) uploaded_path = blob_manager.upload_blob('1', project_path) self.assertEqual(uploaded_path, project_path) downloaded_path = blob_manager.download_blob('1', uploaded_path) self.assertEqual(project_path, downloaded_path)
def test_load_project_config(self): project_path = get_file_dir(__file__) set_project_config_file(project_path + "/project.yaml") project_desc = get_project_description_from(project_path) self.assertEqual(project_desc.project_config.get_master_uri(), "localhost:50051") self.assertIsNone( project_desc.project_config.get('ai_flow config', None)) self.assertEqual(project_desc.project_config['ai_flow_home'], '/opt/ai_flow') self.assertEqual( project_desc.project_config['ai_flow_job_master.host'], 'localhost') self.assertEqual( project_desc.project_config['ai_flow_job_master.port'], 8081) self.assertEqual(project_desc.project_config['ai_flow_conf'], 'taskmanager.slot=2')
def generate_project_desc(project_path: Text = None) -> ProjectDesc: """ Run project under the current project path. :param project_path: The path of the project path. :return: project description. """ if project_path is None: # generate project structure automatically without `project_path` parameter. def generate_project(code): """ generate project automatically without project path. :param code: project code. :return: project description. """ project_path = tempfile.mkdtemp() # generate project config file. project_config().dump_to_file(project_path + '/project.yaml') # generate project structure. os.makedirs(project_path + '/python_codes') os.makedirs(project_path + '/jar_dependencies') os.makedirs(project_path + '/resources') open(project_path + '/python_codes/__init__.py', 'w') fd, temp_file = tempfile.mkstemp(suffix='.py', dir=project_path + '/python_codes') with open(temp_file, 'w') as f: f.write(code) # generate project description. project_desc: ProjectDesc = get_project_description_from( project_path) project_desc.project_name = project_config().get_project_name() project_desc.project_config['entry_module_path'] = ( file_path_to_absolute_module(temp_file)).split('.')[-1] return project_desc return generate_project( inspect.getsource(sys._getframe().f_back.f_back)) else: # parse project description project_desc: ProjectDesc = get_project_description_from(project_path) project_desc.project_name = project_config().get_project_name() return project_desc
def submitWorkflow(self, request, context): try: rq: ScheduleWorkflowRequest = request workflow: Workflow = json_utils.loads(rq.workflow_json) workflow.workflow_name = rq.workflow_name config = {} config.update(workflow.project_desc.project_config['blob']) # config['local_repository'] = self._scheduler_config.repository() blob_manager = BlobManagerFactory.get_blob_manager(config) project_path: Text = blob_manager\ .download_blob(workflow_id=workflow.workflow_id, remote_path=workflow.project_desc.project_config.get('uploaded_project_path'), local_path=self._scheduler_config.repository()) project_desc: ProjectDesc = get_project_description_from( project_path) project_name = project_desc.project_name # update workflow workflow.project_desc = project_desc for n, j in workflow.jobs.items(): j.job_config.project_desc = project_desc j.job_config.project_path = project_path workflow_info = self._scheduler.submit_workflow( workflow, project_desc) if workflow_info is None: return WorkflowInfoResponse(result=ResultProto( status=StatusProto.ERROR, error_message='{}, {} do not exist!'.format( project_name, workflow.workflow_name))) return WorkflowInfoResponse( result=ResultProto(status=StatusProto.OK), workflow=workflow_to_proto(workflow_info)) except Exception as err: return WorkflowInfoResponse( result=ResultProto(status=StatusProto.ERROR, error_message=traceback.format_exc()))
def set_project_path(path): global _default_project_desc set_project_config_file(os.path.join(path, 'project.yaml')) _default_project_desc = get_project_description_from(path) _default_project_desc.project_config = _default_project_config