def job_design_migration_for_jar(jd):
"""Migrate one jar type design"""
data = json.loads(jd.data)
action = OozieJavaAction(action_type=OozieJavaAction.ACTION_TYPE,
jar_path=data['jarfile'],
main_class="please.specify.in.the.job.design",
args=data['arguments'])
action.save()
design = OozieDesign(owner=jd.owner,
name=jd.name + ' (incomplete)',
description=jd.description,
root_action=action)
design.save()
def job_design_migration_for_jar(jd):
"""Migrate one jar type design"""
data = json.loads(jd.data)
action = OozieJavaAction(action_type=OozieJavaAction.ACTION_TYPE,
jar_path=data['jarfile'],
main_class="please.specify.in.the.job.design",
args=data['arguments'])
action.save()
design = OozieDesign(owner=jd.owner,
name=jd.name + ' (incomplete)',
description=jd.description,
root_action=action)
design.save()
def job_design_migration_for_streaming(jd):
"""Migrate one streaming type design"""
data = json.loads(jd.data)
files = json.dumps(data['cache_files'])
archives = json.dumps(data['cache_archives'])
properties = data['hadoop_properties']
def add_property(key, value):
if value:
properties[key] = value
add_property('mapred.input.dir', ','.join(data['input']))
add_property('mapred.output.dir', data['output'])
add_property('mapred.combiner.class', data['combiner_class'])
add_property('mapred.mapper.class', data['mapper_class'])
add_property('mapred.reducer.class', data['reducer_class'])
add_property('mapred.partitioner.class', data['partitioner_class'])
add_property('mapred.input.format.class', data['inputformat_class'])
add_property('mapred.output.format.class', data['outputformat_class'])
add_property('mapred.reduce.tasks', data['num_reduce_tasks'])
property_list = []
for k, v in properties.iteritems():
property_list.append(dict(name=k, value=v))
action = OozieStreamingAction(action_type=OozieStreamingAction.ACTION_TYPE,
mapper=data['mapper_cmd'],
reducer=data['reducer_cmd'],
files=files,
archives=archives,
job_properties=json.dumps(property_list))
action.save()
design = OozieDesign(owner=jd.owner,
name=jd.name,
description=jd.description,
root_action=action)
design.save()
def job_design_migration_for_streaming(jd):
"""Migrate one streaming type design"""
data = json.loads(jd.data)
files = json.dumps(data['cache_files'])
archives = json.dumps(data['cache_archives'])
properties = data['hadoop_properties']
def add_property(key, value):
if value:
properties[key] = value
add_property('mapred.input.dir', ','.join(data['input']))
add_property('mapred.output.dir', data['output'])
add_property('mapred.combiner.class', data['combiner_class'])
add_property('mapred.mapper.class', data['mapper_class'])
add_property('mapred.reducer.class', data['reducer_class'])
add_property('mapred.partitioner.class', data['partitioner_class'])
add_property('mapred.input.format.class', data['inputformat_class'])
add_property('mapred.output.format.class', data['outputformat_class'])
add_property('mapred.reduce.tasks', data['num_reduce_tasks'])
property_list = [ ]
for k, v in properties.iteritems():
property_list.append(dict(name=k, value=v))
action = OozieStreamingAction(action_type=OozieStreamingAction.ACTION_TYPE,
mapper=data['mapper_cmd'],
reducer=data['reducer_cmd'],
files=files,
archives=archives,
job_properties=json.dumps(property_list))
action.save()
design = OozieDesign(owner=jd.owner,
name=jd.name,
description=jd.description,
root_action=action)
design.save()