def _copy_asset(self, asset):
"""
Copy app assets.
Args:
self: class instance
asset: what to copy
Returns:
On success: True.
On failure: False.
"""
if not self._copy_prefix:
Log.a().warning(
'copy prefix must be specified when copying app assets'
)
return False
if not asset.get('dst'):
Log.a().warning('asset dst required for app %s', self._app['name'])
return False
if not asset.get('src'):
Log.a().warning('asset src required for app %s', self._app['name'])
return False
# create asset destination
asset_path = Path(self._path / asset['dst'])
asset_path.mkdir(exist_ok=True)
if 'zip' in asset:
# create a tar.gz of src
cmd = 'tar -czf "{}" --directory="{}" .'.format(
str(Path(asset_path / '{}.tar.gz'.format(asset['zip']))),
str(Path(self._copy_prefix) / asset['src'])
)
Log.some().info('zipping: %s', cmd)
cmd_result = ShellWrapper.invoke(cmd)
if cmd_result is False:
Log.a().warning('cannot zip asset src: %s', cmd)
return False
Log.some().info('tar stdout: %s', cmd_result)
else:
# move without creating tar.gz
cmd = 'cp -R "{}" "{}"'.format(
str(Path(self._copy_prefix) / asset['src']),
str(asset_path)
)
Log.some().info('copying: %s', cmd)
cmd_result = ShellWrapper.invoke(cmd)
if cmd_result is False:
Log.a().warning('cannot copy asset src: %s', cmd)
return False
Log.some().info('copy stdout: %s', cmd_result)
return True
def check_running_jobs(self):
"""
Check the status/progress of all map-reduce items and update _map status.
Args:
self: class instance.
Returns:
True.
"""
# check if procs are running, finished, or failed
for map_item in self._map:
try:
if ShellWrapper.is_running(
map_item['run'][map_item['attempt']]['proc']
):
map_item['status'] = 'RUNNING'
else:
if map_item['run'][map_item['attempt']]['proc'].returncode:
map_item['status'] = 'FAILED'
else:
map_item['status'] = 'FINISHED'
map_item['run'][map_item['attempt']]['status']\
= map_item['status']
except (OSError, AttributeError) as err:
Log.a().warning(
'process polling failed for map item "%s" [%s]',
map_item['filename'], str(err)
)
map_item['status'] = 'FAILED'
self._update_status_db(self._status, '')
return True
def check_running_jobs(self):
"""
Check the status/progress of all map-reduce items and update _map status.
Args:
self: class instance.
Returns:
True.
"""
# check if procs are running, finished, or failed
for map_item in self._map:
if map_item['status'] in ['RUNNING', 'UNKNOWN']:
try:
if not ShellWrapper.is_running(
map_item['run'][map_item['attempt']]['proc']):
returncode = map_item['run'][
map_item['attempt']]['proc'].returncode
if returncode:
map_item['status'] = 'FAILED'
else:
map_item['status'] = 'FINISHED'
Log.a().debug('[step.%s]: exit status: %s -> %s',
self._step['name'],
map_item['template']['output'],
returncode)
# decrease num running procs
if self._num_running > 0:
self._num_running -= 1
except (OSError, AttributeError) as err:
Log.a().warning(
'process polling failed for map item "%s" [%s]',
map_item['filename'], str(err))
map_item['status'] = 'UNKNOWN'
map_item['run'][map_item['attempt']]['status']\
= map_item['status']
self._update_status_db(self._status, '')
return True
def step_impl(context, shell):
context.shell[shell] = {}
context.shell[shell]['shell'] = ShellWrapper()
def _run_map(self, map_item):
"""
Run a job for each map item and store the proc and PID.
Args:
self: class instance.
map_item: map item object (item of self._map).
Returns:
On success: True.
On failure: False.
"""
# load default app inputs, overwrite with template inputs
inputs = {}
for input_key in self._app['inputs']:
if input_key in map_item['template']:
inputs[input_key] = map_item['template'][input_key]
else:
if self._app['inputs'][input_key]['default']:
inputs[input_key] = self._app['inputs'][input_key]['default']
# load default app parameters, overwrite with template parameters
parameters = {}
for param_key in self._app['parameters']:
if param_key in map_item['template']:
parameters[param_key] = map_item['template'][param_key]
else:
if self._app['parameters'][param_key]['default'] not in [None, '']:
parameters[param_key] \
= self._app['parameters'][param_key]['default']
# construct shell command
cmd = self._app['implementation']['local']['script']
for input_key in inputs:
if inputs[input_key]:
cmd += ' --{}="{}"'.format(
input_key,
URIParser.parse(inputs[input_key])['chopped_path']
)
for param_key in parameters:
if param_key == 'output':
cmd += ' --output="{}/{}"'.format(
self._parsed_data_uris[self._source_context]\
['chopped_path'],
parameters['output']
)
else:
cmd += ' --{}="{}"'.format(
param_key, parameters[param_key]
)
# add exeuction method
cmd += ' --exec_method="{}"'.format(self._step['execution']['method'])
# specify execution init commands if 'init' param given
if 'init' in self._step['execution']['parameters']:
cmd += ' --exec_init="{}"'.format(self._step['execution']['parameters']['init'])
# add stdout and stderr
log_path = '{}/_log/gf-{}-{}-{}'.format(
self._parsed_data_uris[self._source_context]['chopped_path'],
map_item['attempt'],
slugify(self._step['name'], regex_pattern=r'[^-a-z0-9_]+'),
slugify(map_item['template']['output'], regex_pattern=r'[^-a-z0-9_]+')
)
cmd += ' > "{}.out" 2> "{}.err"'.format(log_path, log_path)
Log.a().debug('command: %s', cmd)
# launch process
proc = ShellWrapper.spawn(cmd)
if proc is False:
msg = 'shell process error: {}'.format(cmd)
Log.an().error(msg)
return self._fatal(msg)
# record job info
map_item['run'][map_item['attempt']]['proc'] = proc
map_item['run'][map_item['attempt']]['pid'] = proc.pid
# set status of process
map_item['status'] = 'RUNNING'
map_item['run'][map_item['attempt']]['status'] = 'RUNNING'
return True
def _build_asset(self, asset):
"""
Build app assets.
Args:
self: class instance
asset: what to build
Returns:
On success: True.
On failure: False.
"""
# make sure the build path exists
build_path = self._path / 'build'
build_path.mkdir(exist_ok=True)
build_repo_path = None
if not asset.get('folder'):
Log.a().warning(
'repo folder must be set when specifying a build asset'
)
return False
# clone build repo
build_repo_path = build_path / asset['folder']
if asset.get('repo'):
# if repo is set, clone and build it
try:
if asset.get('tag'):
Repo.clone_from(
asset['repo'], str(build_repo_path),
branch=asset['tag'], config='http.sslVerify=false'
)
else:
Repo.clone_from(
asset['repo'], str(build_repo_path),
config='http.sslVerify=false'
)
except GitError as err:
Log.an().error(
'cannot clone git repo for build: %s [%s]',
asset['repo'], str(err)
)
return False
# if repo is not set, packaged build scripts are included with the
# workflow in the build_repo_path
# build
cmd = 'make -C "{}"'.format(str(build_repo_path))
Log.some().info('build command: %s', cmd)
cmd_result = ShellWrapper.invoke(cmd)
if cmd_result is False:
Log.a().warning('cannot build app: %s', cmd)
return False
Log.some().info('make stdout: %s', cmd_result)
# move built assets
# make sure asset folder exists
if not asset.get('dst'):
Log.a().warning('asset dst required for app %s', self._app['name'])
return False
if not asset.get('src'):
Log.a().warning('asset src required for app %s', self._app['name'])
return False
# create asset destination
asset_path = self._path / asset['dst']
asset_path.mkdir(exist_ok=True)
# set src path
src_path = self._path / asset['src']
if 'zip' in asset:
# create a tar.gz of src
cmd = 'tar -czf "{}" --directory="{}" .'.format(
str(asset_path / '{}.tar.gz'.format(asset['zip'])),
str(src_path)
)
Log.some().info('zipping: %s', cmd)
cmd_result = ShellWrapper.invoke(cmd)
if cmd_result is False:
Log.a().warning('cannot zip asset src: %s', cmd)
return False
Log.some().info('tar stdout: %s', cmd_result)
else:
# move without creating tar.gz
cmd = 'mv "{}" "{}"'.format(str(src_path), str(asset_path))
Log.some().info('moving: %s', cmd)
cmd_result = ShellWrapper.invoke(cmd)
if cmd_result is False:
Log.a().warning('cannot move asset src: %s', cmd)
return False
Log.some().info('mv stdout: %s', cmd_result)
return True