def smart_path(path, mapping, is_coordinator=False):
# Try to prepend home_dir and FS scheme if needed.
# If path starts by a parameter try to get its value from the list of parameters submitted by the user or the coordinator.
# This dynamic checking enable the use of <prepares> statements in a workflow scheduled manually of by a coordinator.
# The logic is a bit complicated but Oozie is not consistent with data paths, prepare, coordinator paths and Fs action.
if not path.startswith('$') and not path.startswith(
'/') and not urlparse.urlsplit(path).scheme:
path = '/user/%(username)s/%(path)s' % {
'username':
'******' if is_coordinator else '${wf:user()}',
'path': path
}
if path.startswith('$'):
variables = find_variables(path)
for var in variables:
prefix = '${%s}' % var
if path.startswith(prefix):
if var in mapping:
if not urlparse.urlsplit(
mapping[var]
).scheme and not mapping[var].startswith('$'):
path = '%(nameNode)s%(path)s' % {
'nameNode': '${nameNode}',
'path': path
}
else:
if not urlparse.urlsplit(path).scheme:
path = '%(nameNode)s%(path)s' % {
'nameNode': '${nameNode}',
'path': path
}
return path
def smart_path(path, mapping=None, is_coordinator=False):
# Try to prepend home_dir and FS scheme if needed.
# If path starts by a parameter try to get its value from the list of parameters submitted by the user or the coordinator.
# This dynamic checking enable the use of <prepares> statements in a workflow scheduled manually of by a coordinator.
# The logic is a bit complicated but Oozie is not consistent with data paths, prepare, coordinator paths and Fs action.
if mapping is None:
mapping = {}
if not path.startswith('$') and not path.startswith('/') and not urlparse.urlsplit(path).scheme:
path = '/user/%(username)s/%(path)s' % {
'username': '******' if is_coordinator else '${wf:user()}',
'path': path
}
if path.startswith('$'):
variables = find_variables(path)
for var in variables:
prefix = '${%s}' % var
if path.startswith(prefix):
if var in mapping:
if not urlparse.urlsplit(mapping[var]).scheme and not mapping[var].startswith('$'):
path = '%(nameNode)s%(path)s' % {'nameNode': '${nameNode}', 'path': path}
else:
if not urlparse.urlsplit(path).scheme:
path = '%(nameNode)s%(path)s' % {'nameNode': '${nameNode}', 'path': path}
return path
def _get_external_parameters(self, xml, properties=None):
from oozie.models import DATASET_FREQUENCY
parameters = dict([(var, '') for var in find_variables(xml, include_named=False) if not self._is_coordinator() or var not in DATASET_FREQUENCY])
if properties:
parameters.update(dict([line.strip().split('=')
for line in properties.split('\n') if not line.startswith('#') and len(line.strip().split('=')) == 2]))
return parameters
def _get_external_parameters(self, xml, properties=None):
from oozie.models import DATASET_FREQUENCY
parameters = dict([(var, '') for var in find_variables(xml, include_named=False) if not self._is_coordinator() or var not in DATASET_FREQUENCY])
if properties:
parameters.update(dict([line.strip().split('=')
for line in properties.split('\n') if not line.startswith('#') and len(line.strip().split('=')) == 2]))
return parameters
def make_parameterization_form(query_str):
"""
Creates a django form on the fly with arguments from the
query.
"""
variables = find_variables(query_str)
if len(variables) > 0:
class Form(forms.Form):
for name in sorted(variables):
locals()[name] = forms.CharField(widget=forms.TextInput(attrs={'required': True}))
return Form
else:
return None
def make_parameterization_form(query_str):
"""
Creates a django form on the fly with arguments from the
query.
"""
variables = find_variables(query_str)
if len(variables) > 0:
class Form(forms.Form):
for name in sorted(variables):
locals()[name] = forms.CharField(required=True)
return Form
else:
return None
def contains_symlink(path, mapping):
vars = find_variables(path)
return any([var in mapping and '#' in mapping[var]
for var in vars]) or '#' in path
def contains_symlink(path, mapping): vars = find_variables(path) return any([var in mapping and '#' in mapping[var] for var in vars]) or '#' in path