Esempio n. 1
0
def smart_path(path, mapping, is_coordinator=False):
    # Try to prepend home_dir and FS scheme if needed.
    # If path starts by a parameter try to get its value from the list of parameters submitted by the user or the coordinator.
    # This dynamic checking enable the use of <prepares> statements in a workflow scheduled manually of by a coordinator.
    # The logic is a bit complicated but Oozie is not consistent with data paths, prepare, coordinator paths and Fs action.
    if not path.startswith('$') and not path.startswith(
            '/') and not urlparse.urlsplit(path).scheme:
        path = '/user/%(username)s/%(path)s' % {
            'username':
            '******' if is_coordinator else '${wf:user()}',
            'path': path
        }

    if path.startswith('$'):
        variables = find_variables(path)
        for var in variables:
            prefix = '${%s}' % var
            if path.startswith(prefix):
                if var in mapping:
                    if not urlparse.urlsplit(
                            mapping[var]
                    ).scheme and not mapping[var].startswith('$'):
                        path = '%(nameNode)s%(path)s' % {
                            'nameNode': '${nameNode}',
                            'path': path
                        }
    else:
        if not urlparse.urlsplit(path).scheme:
            path = '%(nameNode)s%(path)s' % {
                'nameNode': '${nameNode}',
                'path': path
            }

    return path
Esempio n. 2
0
File: utils.py Progetto: 10sr/hue
def smart_path(path, mapping=None, is_coordinator=False):
  # Try to prepend home_dir and FS scheme if needed.
  # If path starts by a parameter try to get its value from the list of parameters submitted by the user or the coordinator.
  # This dynamic checking enable the use of <prepares> statements in a workflow scheduled manually of by a coordinator.
  # The logic is a bit complicated but Oozie is not consistent with data paths, prepare, coordinator paths and Fs action.
  if mapping is None:
    mapping = {}

  if not path.startswith('$') and not path.startswith('/') and not urlparse.urlsplit(path).scheme:
    path = '/user/%(username)s/%(path)s' % {
        'username': '******' if is_coordinator else '${wf:user()}',
        'path': path
    }

  if path.startswith('$'):
    variables = find_variables(path)
    for var in variables:
      prefix = '${%s}' % var
      if path.startswith(prefix):
        if var in mapping:
          if not urlparse.urlsplit(mapping[var]).scheme and not mapping[var].startswith('$'):
            path = '%(nameNode)s%(path)s' % {'nameNode': '${nameNode}', 'path': path}
  else:
    if not urlparse.urlsplit(path).scheme:
      path = '%(nameNode)s%(path)s' % {'nameNode': '${nameNode}', 'path': path}

  return path
Esempio n. 3
0
  def _get_external_parameters(self, xml, properties=None):
    from oozie.models import DATASET_FREQUENCY
    parameters = dict([(var, '') for var in find_variables(xml, include_named=False) if not self._is_coordinator() or var not in DATASET_FREQUENCY])

    if properties:
      parameters.update(dict([line.strip().split('=')
                              for line in properties.split('\n') if not line.startswith('#') and len(line.strip().split('=')) == 2]))
    return parameters
Esempio n. 4
0
  def _get_external_parameters(self, xml, properties=None):
    from oozie.models import DATASET_FREQUENCY
    parameters = dict([(var, '') for var in find_variables(xml, include_named=False) if not self._is_coordinator() or var not in DATASET_FREQUENCY])

    if properties:
      parameters.update(dict([line.strip().split('=')
                              for line in properties.split('\n') if not line.startswith('#') and len(line.strip().split('=')) == 2]))
    return parameters
Esempio n. 5
0
def make_parameterization_form(query_str):
  """
  Creates a django form on the fly with arguments from the
  query.
  """
  variables = find_variables(query_str)
  if len(variables) > 0:
    class Form(forms.Form):
      for name in sorted(variables):
        locals()[name] = forms.CharField(widget=forms.TextInput(attrs={'required': True}))
    return Form
  else:
    return None
Esempio n. 6
0
def make_parameterization_form(query_str):
  """
  Creates a django form on the fly with arguments from the
  query.
  """
  variables = find_variables(query_str)
  if len(variables) > 0:
    class Form(forms.Form):
      for name in sorted(variables):
        locals()[name] = forms.CharField(required=True)
    return Form
  else:
    return None
Esempio n. 7
0
def contains_symlink(path, mapping):
    vars = find_variables(path)
    return any([var in mapping and '#' in mapping[var]
                for var in vars]) or '#' in path
Esempio n. 8
0
File: utils.py Progetto: ronwxy/hue
def contains_symlink(path, mapping):
  vars = find_variables(path)
  return any([var in mapping and '#' in mapping[var] for var in vars]) or '#' in path