def generate_project(code):
"""
generate project automatically without project path.
:param code: project code.
:return: project description.
"""
project_path = tempfile.mkdtemp()
# generate project config file.
project_config().dump_to_file(project_path + '/project.yaml')
# generate project structure.
os.makedirs(project_path + '/python_codes')
os.makedirs(project_path + '/jar_dependencies')
os.makedirs(project_path + '/resources')
open(project_path + '/python_codes/__init__.py', 'w')
fd, temp_file = tempfile.mkstemp(suffix='.py',
dir=project_path +
'/python_codes')
with open(temp_file, 'w') as f:
f.write(code)
# generate project description.
project_desc: ProjectDesc = get_project_description_from(
project_path)
project_desc.project_name = project_config().get_project_name()
project_desc.project_config['entry_module_path'] = (
file_path_to_absolute_module(temp_file)).split('.')[-1]
return project_desc
def test_project_upload_download_local(self):
project_path = get_file_dir(__file__)
project_desc = get_project_description_from(project_path + "/../")
# blob_server.type = local
blob_manager = BlobManagerFactory.get_blob_manager(
project_desc.project_config)
uploaded_path = blob_manager.upload_blob('1', project_path)
self.assertEqual(uploaded_path, project_path)
downloaded_path = blob_manager.download_blob('1', uploaded_path)
self.assertEqual(project_path, downloaded_path)
def test_load_project_config(self):
project_path = get_file_dir(__file__)
set_project_config_file(project_path + "/project.yaml")
project_desc = get_project_description_from(project_path)
self.assertEqual(project_desc.project_config.get_master_uri(),
"localhost:50051")
self.assertIsNone(
project_desc.project_config.get('ai_flow config', None))
self.assertEqual(project_desc.project_config['ai_flow_home'],
'/opt/ai_flow')
self.assertEqual(
project_desc.project_config['ai_flow_job_master.host'],
'localhost')
self.assertEqual(
project_desc.project_config['ai_flow_job_master.port'], 8081)
self.assertEqual(project_desc.project_config['ai_flow_conf'],
'taskmanager.slot=2')
def generate_project_desc(project_path: Text = None) -> ProjectDesc:
"""
Run project under the current project path.
:param project_path: The path of the project path.
:return: project description.
"""
if project_path is None:
# generate project structure automatically without `project_path` parameter.
def generate_project(code):
"""
generate project automatically without project path.
:param code: project code.
:return: project description.
"""
project_path = tempfile.mkdtemp()
# generate project config file.
project_config().dump_to_file(project_path + '/project.yaml')
# generate project structure.
os.makedirs(project_path + '/python_codes')
os.makedirs(project_path + '/jar_dependencies')
os.makedirs(project_path + '/resources')
open(project_path + '/python_codes/__init__.py', 'w')
fd, temp_file = tempfile.mkstemp(suffix='.py',
dir=project_path +
'/python_codes')
with open(temp_file, 'w') as f:
f.write(code)
# generate project description.
project_desc: ProjectDesc = get_project_description_from(
project_path)
project_desc.project_name = project_config().get_project_name()
project_desc.project_config['entry_module_path'] = (
file_path_to_absolute_module(temp_file)).split('.')[-1]
return project_desc
return generate_project(
inspect.getsource(sys._getframe().f_back.f_back))
else:
# parse project description
project_desc: ProjectDesc = get_project_description_from(project_path)
project_desc.project_name = project_config().get_project_name()
return project_desc
def submitWorkflow(self, request, context):
try:
rq: ScheduleWorkflowRequest = request
workflow: Workflow = json_utils.loads(rq.workflow_json)
workflow.workflow_name = rq.workflow_name
config = {}
config.update(workflow.project_desc.project_config['blob'])
# config['local_repository'] = self._scheduler_config.repository()
blob_manager = BlobManagerFactory.get_blob_manager(config)
project_path: Text = blob_manager\
.download_blob(workflow_id=workflow.workflow_id,
remote_path=workflow.project_desc.project_config.get('uploaded_project_path'),
local_path=self._scheduler_config.repository())
project_desc: ProjectDesc = get_project_description_from(
project_path)
project_name = project_desc.project_name
# update workflow
workflow.project_desc = project_desc
for n, j in workflow.jobs.items():
j.job_config.project_desc = project_desc
j.job_config.project_path = project_path
workflow_info = self._scheduler.submit_workflow(
workflow, project_desc)
if workflow_info is None:
return WorkflowInfoResponse(result=ResultProto(
status=StatusProto.ERROR,
error_message='{}, {} do not exist!'.format(
project_name, workflow.workflow_name)))
return WorkflowInfoResponse(
result=ResultProto(status=StatusProto.OK),
workflow=workflow_to_proto(workflow_info))
except Exception as err:
return WorkflowInfoResponse(
result=ResultProto(status=StatusProto.ERROR,
error_message=traceback.format_exc()))
def set_project_path(path):
global _default_project_desc
set_project_config_file(os.path.join(path, 'project.yaml'))
_default_project_desc = get_project_description_from(path)
_default_project_desc.project_config = _default_project_config
