def hadoop_jar(stdout, stderr, environ, *args): if len(args) < 1: print('RunJar jarFile [mainClass] args...', file=stderr) return -1 jar_path = args[0] if not os.path.exists(jar_path): print('Exception in thread "main" java.io.IOException: Error opening' ' job jar: %s' % jar_path, file=stderr) return -1 # use this to simulate log4j def mock_log4j(message, level='INFO', logger='mapreduce.JOB', now=None): now = now or datetime.datetime.now() line = '%s %s %s: %s' % (now.strftime('%Y/%m/%d %H:%M:%S'), level, logger, message) print(line, file=stderr) # simulate counters counters = next_mock_hadoop_counters() if counters: num_counters = sum(len(g) for g in counters.values()) mock_log4j('Counters: %d' % num_counters) # subsequent lines are actually part of same log record for group, group_counters in sorted(counters.items()): print(('\t%s' % group), file=stderr) for counter, amount in sorted(group_counters.items()): print(('\t\t%s=%d' % (counter, amount)), file=stderr) # simulate output for streaming steps if _HADOOP_STREAMING_JAR_RE.match(os.path.basename(jar_path)): streaming_args = args[1:] output_idx = list(streaming_args).index('-output') assert output_idx != -1 output_dir = streaming_args[output_idx + 1] real_output_dir = hdfs_uri_to_real_path(output_dir, environ) mock_output_dir = get_mock_hadoop_output() if mock_output_dir is None: mock_log4j('Job failed!') return -1 if os.path.isdir(real_output_dir): os.rmdir(real_output_dir) shutil.move(mock_output_dir, real_output_dir) now = datetime.datetime.now() mock_log4j(now.strftime('Running job: job_%Y%m%d%H%M_0001')) mock_log4j('Job succeeded!') return 0
def hadoop_jar(stdout, stderr, environ, *args): if len(args) < 1: print('RunJar jarFile [mainClass] args...', file=stderr) return -1 jar_path = args[0] if not os.path.exists(jar_path): print('Exception in thread "main" java.io.IOException: Error opening' ' job jar: %s' % jar_path, file=stderr) return -1 # use this to simulate log4j def mock_log4j(message, level='INFO', logger='mapreduce.JOB', now=None): now = now or datetime.datetime.now() line = '%s %s %s: %s' % (now.strftime('%Y/%m/%d %H:%M:%S'), level, logger, message) print(line, file=stderr) # simulate counters counters = next_mock_hadoop_counters() if counters: num_counters = sum(len(g) for g in counters.values()) mock_log4j('Counters: %d' % num_counters) # subsequent lines are actually part of same log record for group, group_counters in sorted(counters.items()): print(('\t%s' % group), file=stderr) for counter, amount in sorted(group_counters.items()): print(('\t\t%s=%d' % (counter, amount)), file=stderr) # simulate output for streaming steps if _HADOOP_STREAMING_JAR_RE.match(os.path.basename(jar_path)): streaming_args = args[1:] output_idx = list(streaming_args).index('-output') assert output_idx != -1 output_dir = streaming_args[output_idx + 1] real_output_dir = hdfs_uri_to_real_path(output_dir, environ) mock_output_dir = get_mock_hadoop_output() if mock_output_dir is None: mock_log4j('Job failed!') return -1 if os.path.isdir(real_output_dir): os.rmdir(real_output_dir) shutil.move(mock_output_dir, real_output_dir) now = datetime.datetime.now() mock_log4j(now.strftime('Running job: job_%Y%m%d%H%M_0001')) mock_log4j('Job succeeded!') return 0