def upload_blob(self, workflow_id: Text, prj_pkg_path: Text) -> Text:
with tempfile.TemporaryDirectory() as temp_dir:
zip_file_name = 'workflow_{}_project.zip'.format(workflow_id)
temp_dir_path = Path(temp_dir)
zip_file_path = temp_dir_path / zip_file_name
make_dir_zipfile(prj_pkg_path, zip_file_path)
object_key = 'ai-flow-k8s/' + zip_file_name
self.bucket.put_object_from_file(key=object_key,
filename=str(zip_file_path))
return object_key
def upload_blob(self, workflow_id: Text, prj_pkg_path: Text) -> Text:
if self._remote_repo is not None:
with tempfile.TemporaryDirectory() as temp_dir:
zip_file_name = 'workflow_{}_project.zip'.format(workflow_id)
upload_file_path = Path('{}/{}'.format(self._remote_repo,
zip_file_name))
if os.path.exists(upload_file_path):
os.remove(upload_file_path)
temp_dir_path = Path(temp_dir)
zip_file_path = temp_dir_path / zip_file_name
make_dir_zipfile(prj_pkg_path, zip_file_path)
os.rename(zip_file_path, upload_file_path)
return str(upload_file_path)
else:
return prj_pkg_path
def upload_project(self, workflow_snapshot_id: Text,
project_path: Text) -> Text:
"""
upload a given project to blob server for remote execution.
:param workflow_snapshot_id: It is the unique identifier for each workflow generation.
:param project_path: The path of this project.
:return The uri of the uploaded project file in blob server.
"""
with tempfile.TemporaryDirectory() as temp_dir:
zip_file_name = 'workflow_{}_project.zip'.format(
workflow_snapshot_id)
temp_dir_path = Path(temp_dir)
zip_file_path = temp_dir_path / zip_file_name
make_dir_zipfile(project_path, zip_file_path)
object_key = self.repo_name + '/' + zip_file_name
self.bucket.put_object_from_file(key=object_key,
filename=str(zip_file_path))
return object_key
def save_model(self, local_path, remote_path=None) -> Text:
"""
save a local export model to remote storage(Aliyun OSS)
:param local_path: the local path of the model, the oss storage only supports binary files, thus we have to
make the local path file a zip.
:param remote_path: the object_key of the uploaded file in oss with the pattern like abc/efg/123.jpg
:return: the object_key of uploaded file.
"""
with tempfile.TemporaryDirectory() as temp_dir:
local_path = Path(local_path)
zip_file_name = os.path.splitext(local_path.name)[0] + '.zip'
temp_dir_path = Path(temp_dir)
zip_file_path = temp_dir_path / zip_file_name
if local_path.is_dir():
make_dir_zipfile(str(local_path), zip_file_path)
else:
make_file_zipfile(str(local_path), zip_file_path)
if remote_path is None:
object_key = 'ai-flow-model-manager/' + zip_file_name
else:
object_key = remote_path
self.bucket.put_object_from_file(key=object_key,
filename=str(zip_file_path))
return object_key
def upload_project(self, workflow_snapshot_id: Text,
project_path: Text) -> Text:
"""
upload a given project to blob server for remote execution.
:param workflow_snapshot_id: It is the unique identifier for each workflow generation.
:param project_path: The path of this project.
:return The uri of the uploaded project file in blob server.
"""
if self._remote_repo is not None:
with tempfile.TemporaryDirectory() as temp_dir:
zip_file_name = 'workflow_{}_project.zip'.format(
workflow_snapshot_id)
upload_file_path = Path('{}/{}'.format(self._remote_repo,
zip_file_name))
if os.path.exists(upload_file_path):
os.remove(upload_file_path)
temp_dir_path = Path(temp_dir)
zip_file_path = temp_dir_path / zip_file_name
make_dir_zipfile(project_path, zip_file_path)
shutil.move(zip_file_path, upload_file_path)
return str(upload_file_path)
else:
return project_path
def submit_job(self,
job: Job,
job_runtime_env: JobRuntimeEnv = None) -> JobHandle:
handle = FlinkJobHandle(
job=job, job_execution=job_runtime_env.job_execution_info)
flink_job: FlinkJob = job
job_config: FlinkJobConfig = FlinkJobConfig.from_job_config(
flink_job.job_config)
env = os.environ.copy()
env.update(job_config.properties.get('env', {}))
if not flink_job.is_java:
run_graph_file = os.path.join(job_runtime_env.generated_dir,
flink_job.run_graph_file)
flink_env_file = os.path.join(job_runtime_env.generated_dir,
flink_job.flink_env_file)
# Add PYTHONPATH
copy_path = sys.path.copy()
copy_path.insert(0, job_runtime_env.workflow_dir)
copy_path.insert(0, job_runtime_env.python_dep_dir)
env['PYTHONPATH'] = ':'.join(copy_path)
current_path = os.path.dirname(__file__)
script_path = os.path.join(current_path, 'flink_run_main.py')
python3_location = sys.executable
if job_config.run_mode == 'local':
bash_command = [
python3_location, script_path, run_graph_file,
job_runtime_env.working_dir, flink_env_file
]
elif job_config.run_mode == 'cluster':
bash_command = ['flink', 'run']
if job_config.flink_run_args is not None:
bash_command.extend(job_config.flink_run_args)
bash_command.append('-pyfs')
files = [job_runtime_env.workflow_dir]
if os.path.exists(job_runtime_env.python_dep_dir):
files.append(job_runtime_env.python_dep_dir)
bash_command.append(','.join(files))
if os.path.exists(job_runtime_env.resource_dir):
zip_file_util.make_dir_zipfile(
job_runtime_env.resource_dir,
os.path.join(job_runtime_env.working_dir,
'resources.zip'))
bash_command.extend([
'-pyarch',
os.path.join(job_runtime_env.working_dir,
'resources.zip#resources')
])
bash_command.extend([
'-py', script_path, run_graph_file,
job_runtime_env.working_dir, flink_env_file
])
else:
raise Exception(
'Flink supports run_mode local or cluster, do not support {}.'
.format(job_config.run_mode))
else:
# flink java job
bash_command = ['flink', 'run']
if job_config.flink_run_args is not None:
bash_command.extend(job_config.flink_run_args)
if os.path.exists(job_runtime_env.resource_dir):
zip_file_util.make_dir_zipfile(
job_runtime_env.resource_dir,
os.path.join(job_runtime_env.working_dir, 'resources.zip'))
processor: FlinkJavaProcessor = serialization_utils.\
read_object_from_serialized_file(os.path.join(job_runtime_env.generated_dir, flink_job.processor_file))
if processor.entry_class is not None:
bash_command.extend(['-c', processor.entry_class])
bash_command.append(
os.path.join(job_runtime_env.jar_dep_dir,
processor.main_jar_file))
bash_command.extend(processor.args)
self.log.info(' '.join(bash_command))
stdout_log = log_path_utils.stdout_log_path(job_runtime_env.log_dir,
job.job_name)
stderr_log = log_path_utils.stderr_log_path(job_runtime_env.log_dir,
job.job_name)
if not os.path.exists(job_runtime_env.log_dir):
os.makedirs(job_runtime_env.log_dir)
sub_process = self.submit_process(
bash_command=bash_command,
env=env,
working_dir=job_runtime_env.working_dir,
stdout_log=stdout_log,
stderr_log=stderr_log)
handle.sub_process = sub_process
handle.stdout_log = stdout_log
handle.stderr_log = stderr_log
if flink_job.is_java:
# write job_id to file.
num = 0
while True:
if os.path.exists(stdout_log):
break
else:
time.sleep(1)
num += 1
if 0 == num % 20:
self.log.info("Waiting for stdout log file created...")
while True:
with open(stdout_log, 'r') as f:
lines = f.readlines()
if len(lines) >= 1:
line = lines[0]
if line.startswith("Job has been submitted with JobID"):
job_id = line.split(' ')[6][:-1]
with open(
os.path.join(job_runtime_env.working_dir,
'job_id'), 'w') as fp:
fp.write(job_id)
self.log.info('Flink job id {}'.format(job_id))
break
else:
time.sleep(1)
return handle