Exemplo n.º 1
0
def job_design_migration_for_jar(jd):
    """Migrate one jar type design"""
    data = json.loads(jd.data)
    action = OozieJavaAction(action_type=OozieJavaAction.ACTION_TYPE,
                             jar_path=data['jarfile'],
                             main_class="please.specify.in.the.job.design",
                             args=data['arguments'])
    action.save()

    design = OozieDesign(owner=jd.owner,
                         name=jd.name + ' (incomplete)',
                         description=jd.description,
                         root_action=action)
    design.save()
Exemplo n.º 2
0
def job_design_migration_for_jar(jd):
  """Migrate one jar type design"""
  data = json.loads(jd.data)
  action = OozieJavaAction(action_type=OozieJavaAction.ACTION_TYPE,
                           jar_path=data['jarfile'],
                           main_class="please.specify.in.the.job.design",
                           args=data['arguments'])
  action.save()

  design = OozieDesign(owner=jd.owner,
                       name=jd.name + ' (incomplete)',
                       description=jd.description,
                       root_action=action)
  design.save()
Exemplo n.º 3
0
def job_design_migration_for_streaming(jd):
    """Migrate one streaming type design"""
    data = json.loads(jd.data)

    files = json.dumps(data['cache_files'])
    archives = json.dumps(data['cache_archives'])
    properties = data['hadoop_properties']

    def add_property(key, value):
        if value:
            properties[key] = value

    add_property('mapred.input.dir', ','.join(data['input']))
    add_property('mapred.output.dir', data['output'])
    add_property('mapred.combiner.class', data['combiner_class'])
    add_property('mapred.mapper.class', data['mapper_class'])
    add_property('mapred.reducer.class', data['reducer_class'])
    add_property('mapred.partitioner.class', data['partitioner_class'])
    add_property('mapred.input.format.class', data['inputformat_class'])
    add_property('mapred.output.format.class', data['outputformat_class'])
    add_property('mapred.reduce.tasks', data['num_reduce_tasks'])

    property_list = []
    for k, v in properties.iteritems():
        property_list.append(dict(name=k, value=v))

    action = OozieStreamingAction(action_type=OozieStreamingAction.ACTION_TYPE,
                                  mapper=data['mapper_cmd'],
                                  reducer=data['reducer_cmd'],
                                  files=files,
                                  archives=archives,
                                  job_properties=json.dumps(property_list))
    action.save()

    design = OozieDesign(owner=jd.owner,
                         name=jd.name,
                         description=jd.description,
                         root_action=action)
    design.save()
Exemplo n.º 4
0
def job_design_migration_for_streaming(jd):
  """Migrate one streaming type design"""
  data = json.loads(jd.data)

  files = json.dumps(data['cache_files'])
  archives = json.dumps(data['cache_archives'])
  properties = data['hadoop_properties']

  def add_property(key, value):
    if value:
      properties[key] = value

  add_property('mapred.input.dir', ','.join(data['input']))
  add_property('mapred.output.dir', data['output'])
  add_property('mapred.combiner.class', data['combiner_class'])
  add_property('mapred.mapper.class', data['mapper_class'])
  add_property('mapred.reducer.class', data['reducer_class'])
  add_property('mapred.partitioner.class', data['partitioner_class'])
  add_property('mapred.input.format.class', data['inputformat_class'])
  add_property('mapred.output.format.class', data['outputformat_class'])
  add_property('mapred.reduce.tasks', data['num_reduce_tasks'])

  property_list = [ ]
  for k, v in properties.iteritems():
    property_list.append(dict(name=k, value=v))

  action = OozieStreamingAction(action_type=OozieStreamingAction.ACTION_TYPE,
                                mapper=data['mapper_cmd'],
                                reducer=data['reducer_cmd'],
                                files=files,
                                archives=archives,
                                job_properties=json.dumps(property_list))
  action.save()

  design = OozieDesign(owner=jd.owner,
                       name=jd.name,
                       description=jd.description,
                       root_action=action)
  design.save()