def _copy_predictive_object_files(source_path, target_path, is_dir, src_credentials, tgt_credentials):
'''
Copy either file or folder from source location to target location
'''
# Cleanup existing file path if exists
if _file_util.is_local_path(target_path) and _os.path.exists(target_path):
_shutil.rmtree(target_path)
if _file_util.is_s3_path(source_path) and _file_util.is_s3_path(target_path):
# compare credentials
_check_aws_credentials(src_credentials, tgt_credentials, source_path)
# intra s3 copy model
_file_util.intra_s3_copy_model(source_path, target_path, is_dir, tgt_credentials)
elif _file_util.is_local_path(source_path):
_file_util.copy_from_local(source_path, target_path, is_dir = is_dir)
else:
tmp_dir = _tempfile.mkdtemp(prefix = 'copy_predictive_object')
try:
# download to local first
local_path = _os.path.join(tmp_dir, 'temp_po_file')
if _file_util.is_s3_path(source_path):
_file_util.download_from_s3(
source_path,
local_path,
is_dir=is_dir,
aws_credentials=src_credentials,
silent=False)
elif _file_util.is_hdfs_path(source_path):
_file_util.download_from_hdfs(source_path, local_path, is_dir = False)
else:
raise RuntimeError('Unsupported file system type: %s' % source_path)
# upload from local to remote
if _file_util.is_s3_path(target_path):
_file_util.upload_to_s3(local_path, target_path, is_dir=is_dir,
aws_credentials=tgt_credentials, silent=False)
elif _file_util.is_hdfs_path(target_path):
_file_util.hdfs_mkdir(target_path)
_file_util.upload_to_hdfs(local_path, target_path, force=True, silent=False)
else:
_file_util.upload_to_local(local_path, target_path, is_dir=is_dir, silent=False)
finally:
_shutil.rmtree(tmp_dir)
def subprocess_exe(exe, args, setup=None, teardown=None, out_log_prefix=None, environment_variables=None):
"""
Wrapper function to execute an external program.
This function is exception safe, and always catches
the error.
Parameters
----------
exe : str
The command to run
args : list[str]
Arguments to passed to the command
setup : function
Setup function to run before executing the command
teardown : function
Teardown function to run after executing the command
out_log_prefix: str
The path prefix to the saved log file.
If set, the logs will be save to the following locations:
<prefix>.stdout
<prefix>.stderr
and the return value will contain paths to the log files.
The path can be local or hdfs or s3.
Return
------
out : dict
A dictionary containing the following keys:
success : bool
True if the command succeeded
return_code : int
The return code of the command
stderr : str
Path to the stderr log of the process
stdout : str
Path to the stdout log of the process
python_exception : Exception
Python exception
"""
import logging
import os
ret = {'success': True,
'return_code': None,
'stdout': None,
'stderr': None,
'python_exception': None}
# Creates local log files
try:
local_log_stdout = tempfile.NamedTemporaryFile(delete=False)
local_log_stderr = tempfile.NamedTemporaryFile(delete=False)
except Exception as e:
ret['success'] = False
ret['python_exception'] = e
# Run setup
try:
if setup is not None:
setup()
except Exception as e:
ret['success'] = False
ret['python_exception'] = e
# Executes the command
if ret['success']:
try:
if environment_variables is not None:
environment_variables = os.environ.copy().update(environment_variables)
proc = subprocess.Popen([exe] + args,
stdout=local_log_stdout,
stderr=local_log_stderr,
env=environment_variables)
proc.communicate()
ret['success'] = proc.returncode == 0
ret['return_code'] = proc.returncode
except Exception as e:
ret['success'] = False
ret['python_exception'] = e
finally:
try:
local_log_stdout.close()
local_log_stderr.close()
if out_log_prefix is not None:
# persistent logfiles. When local log closed,
# they will be loaded to the corresponding hdfs or s3 path
file_log_stdout = out_log_prefix + '.stdout'
file_log_stderr = out_log_prefix + '.stderr'
# copy to target log path
file_util.copy_from_local(local_log_stdout.name, file_log_stdout)
file_util.copy_from_local(local_log_stderr.name, file_log_stderr)
ret['stdout'] = file_log_stdout
ret['stderr'] = file_log_stderr
else:
ret['stdout'] = open(local_log_stdout.name).read()
ret['stderr'] = open(local_log_stderr.name).read()
os.remove(local_log_stdout.name)
os.remove(local_log_stderr.name)
except Exception as e:
ret['_save_log_exception'] = e
logging.warn(str(e))
# Teardown
if teardown is not None:
try:
teardown()
except Exception as e:
ret['_tear_down_exception'] = e
logging.warn(str(e))
return ret