def hadoop_jar(stdout, stderr, environ, *args):
if len(args) < 1:
print('RunJar jarFile [mainClass] args...', file=stderr)
return -1
jar_path = args[0]
if not os.path.exists(jar_path):
print('Exception in thread "main" java.io.IOException: Error opening'
' job jar: %s' % jar_path,
file=stderr)
return -1
# use this to simulate log4j
def mock_log4j(message, level='INFO', logger='mapreduce.JOB', now=None):
now = now or datetime.datetime.now()
line = '%s %s %s: %s' % (now.strftime('%Y/%m/%d %H:%M:%S'), level,
logger, message)
print(line, file=stderr)
# simulate counters
counters = next_mock_hadoop_counters()
if counters:
num_counters = sum(len(g) for g in counters.values())
mock_log4j('Counters: %d' % num_counters)
# subsequent lines are actually part of same log record
for group, group_counters in sorted(counters.items()):
print(('\t%s' % group), file=stderr)
for counter, amount in sorted(group_counters.items()):
print(('\t\t%s=%d' % (counter, amount)), file=stderr)
# simulate output for streaming steps
if _HADOOP_STREAMING_JAR_RE.match(os.path.basename(jar_path)):
streaming_args = args[1:]
output_idx = list(streaming_args).index('-output')
assert output_idx != -1
output_dir = streaming_args[output_idx + 1]
real_output_dir = hdfs_uri_to_real_path(output_dir, environ)
mock_output_dir = get_mock_hadoop_output()
if mock_output_dir is None:
mock_log4j('Job failed!')
return -1
if os.path.isdir(real_output_dir):
os.rmdir(real_output_dir)
shutil.move(mock_output_dir, real_output_dir)
now = datetime.datetime.now()
mock_log4j(now.strftime('Running job: job_%Y%m%d%H%M_0001'))
mock_log4j('Job succeeded!')
return 0
def hadoop_jar(stdout, stderr, environ, *args):
if len(args) < 1:
print('RunJar jarFile [mainClass] args...', file=stderr)
return -1
jar_path = args[0]
if not os.path.exists(jar_path):
print('Exception in thread "main" java.io.IOException: Error opening'
' job jar: %s' % jar_path, file=stderr)
return -1
# use this to simulate log4j
def mock_log4j(message, level='INFO', logger='mapreduce.JOB', now=None):
now = now or datetime.datetime.now()
line = '%s %s %s: %s' % (now.strftime('%Y/%m/%d %H:%M:%S'),
level, logger, message)
print(line, file=stderr)
# simulate counters
counters = next_mock_hadoop_counters()
if counters:
num_counters = sum(len(g) for g in counters.values())
mock_log4j('Counters: %d' % num_counters)
# subsequent lines are actually part of same log record
for group, group_counters in sorted(counters.items()):
print(('\t%s' % group), file=stderr)
for counter, amount in sorted(group_counters.items()):
print(('\t\t%s=%d' % (counter, amount)), file=stderr)
# simulate output for streaming steps
if _HADOOP_STREAMING_JAR_RE.match(os.path.basename(jar_path)):
streaming_args = args[1:]
output_idx = list(streaming_args).index('-output')
assert output_idx != -1
output_dir = streaming_args[output_idx + 1]
real_output_dir = hdfs_uri_to_real_path(output_dir, environ)
mock_output_dir = get_mock_hadoop_output()
if mock_output_dir is None:
mock_log4j('Job failed!')
return -1
if os.path.isdir(real_output_dir):
os.rmdir(real_output_dir)
shutil.move(mock_output_dir, real_output_dir)
now = datetime.datetime.now()
mock_log4j(now.strftime('Running job: job_%Y%m%d%H%M_0001'))
mock_log4j('Job succeeded!')
return 0