def test_pydoop_jar_path(self):
jar_path = pydoop.jar_path()
if jar_path is not None:
self.assertTrue(os.path.exists(jar_path))
directory, filename = os.path.split(jar_path)
self.assertEqual(filename, pydoop.jar_name())
self.assertEqual('pydoop', os.path.basename(directory))
def test_pydoop_jar_path(self):
jar_path = pydoop.jar_path()
if jar_path is not None:
self.assertTrue(os.path.exists(jar_path))
directory, filename = os.path.split(jar_path)
self.assertEqual(filename, pydoop.jar_name())
self.assertEqual('pydoop', os.path.basename(directory))
def __init__(self, hadoop_vinfo):
self.hadoop_vinfo = hadoop_vinfo
self.jar_name = pydoop.jar_name(self.hadoop_vinfo)
self.classpath = pydoop.hadoop_classpath()
self.java_files = []
self.dependencies = []
self.properties = []
if hadoop_vinfo.main >= (2, 0, 0) and \
(not hadoop_vinfo.is_cloudera() or hadoop_vinfo.is_yarn()):
# This version of Hadoop has the v2 pipes API
# FIXME: kinda hardwired to avro for now
self.properties.append(
(os.path.join("it/crs4/pydoop/mapreduce/pipes",
PROP_BN), PROP_FN))
self.java_files.extend(
glob.glob('src/v2/it/crs4/pydoop/pipes/*.java'))
self.java_files.extend(
glob.glob('src/v2/it/crs4/pydoop/mapreduce/pipes/*.java'))
# for things such as avro-mapreduce
self.dependencies.extend(glob.glob('lib/*.jar'))
else:
# Else we should be dealing with v1 pipes
self.java_files.extend(
glob.glob('src/v1/org/apache/hadoop/mapred/pipes/*.java'))
if hadoop_vinfo.has_mrv2():
# If the installation has MRv2 we need to use v2 I/O classes
self.java_files.extend(
glob.glob('src/v2/it/crs4/pydoop/mapreduce/lib/output/*.java'))
self.java_files.extend(
["src/v2/it/crs4/pydoop/NoSeparatorTextOutputFormat.java"])
else:
self.java_files.extend(
["src/v1/it/crs4/pydoop/NoSeparatorTextOutputFormat.java"])
def __init__(self, hadoop_vinfo):
self.hadoop_vinfo = hadoop_vinfo
self.jar_name = pydoop.jar_name(self.hadoop_vinfo)
self.classpath = pydoop.hadoop_classpath()
self.java_files = []
self.dependencies = []
self.properties = []
if (hadoop_vinfo.main >= (2, 0, 0)
and (not hadoop_vinfo.is_cloudera()
or hadoop_vinfo.is_yarn())):
# FIXME: kinda hardwired to avro for now
self.properties.append((os.path.join(
"it/crs4/pydoop/mapreduce/pipes", PROP_BN),
PROP_FN))
self.java_files.extend([
"src/v2/it/crs4/pydoop/NoSeparatorTextOutputFormat.java"
])
self.java_files.extend(glob.glob(
'src/v2/it/crs4/pydoop/pipes/*.java'
))
self.java_files.extend(glob.glob(
'src/v2/it/crs4/pydoop/mapreduce/pipes/*.java'
))
# for now we have only hadoop2 deps (avro-mapred)
self.dependencies.extend(glob.glob('lib/*.jar'))
else:
self.java_files.extend([
"src/v1/it/crs4/pydoop/NoSeparatorTextOutputFormat.java"
])
self.java_files.extend(glob.glob(
'src/v1/org/apache/hadoop/mapred/pipes/*.java'
))
def __init__(self, hadoop_vinfo):
self.hadoop_vinfo = hadoop_vinfo
self.jar_name = pydoop.jar_name(self.hadoop_vinfo)
self.classpath = pydoop.hadoop_classpath()
self.java_files = []
self.dependencies = []
self.properties = []
if (hadoop_vinfo.main >= (2, 0, 0) and
(not hadoop_vinfo.is_cloudera() or hadoop_vinfo.is_yarn())):
# FIXME: kinda hardwired to avro for now
self.properties.append(
(os.path.join("it/crs4/pydoop/mapreduce/pipes",
PROP_BN), PROP_FN))
self.java_files.extend(
["src/v2/it/crs4/pydoop/NoSeparatorTextOutputFormat.java"])
self.java_files.extend(
glob.glob('src/v2/it/crs4/pydoop/pipes/*.java'))
self.java_files.extend(
glob.glob('src/v2/it/crs4/pydoop/mapreduce/pipes/*.java'))
# for now we have only hadoop2 deps (avro-mapred)
self.dependencies.extend(glob.glob('lib/*.jar'))
else:
self.java_files.extend(
["src/v1/it/crs4/pydoop/NoSeparatorTextOutputFormat.java"])
self.java_files.extend(
glob.glob('src/v1/org/apache/hadoop/mapred/pipes/*.java'))
def __init__(self, hadoop_vinfo, pipes_src_dir):
self.hadoop_vinfo = hadoop_vinfo
self.jar_name = pydoop.jar_name(self.hadoop_vinfo)
self.classpath = pydoop.hadoop_classpath()
self.java_files = ["src/it/crs4/pydoop/NoSeparatorTextOutputFormat.java"]
if self.hadoop_vinfo.has_security():
if hadoop_vinfo.cdh >= (4, 0, 0) and not hadoop_vinfo.ext:
return # TODO: add support for mrv2
# add our fix for https://issues.apache.org/jira/browse/MAPREDUCE-4000
self.java_files.extend(glob.glob("%s/*" % pipes_src_dir))
def __init__(self, hadoop_vinfo, pipes_src_dir):
self.hadoop_vinfo = hadoop_vinfo
self.jar_name = pydoop.jar_name(self.hadoop_vinfo)
self.classpath = pydoop.hadoop_classpath()
self.java_files = ["src/it/crs4/pydoop/NoSeparatorTextOutputFormat.java"]
if self.hadoop_vinfo.has_security():
if hadoop_vinfo.cdh >= (4, 0, 0) and not hadoop_vinfo.ext:
return # TODO: add support for mrv2
# add our fix for https://issues.apache.org/jira/browse/MAPREDUCE-4000
self.java_files.extend(glob.glob("%s/*" % pipes_src_dir))
def __init__(self):
self.jar_name = pydoop.jar_name()
self.classpath = pydoop.hadoop_classpath()
self.java_files = glob.glob(
"src/it/crs4/pydoop/mapreduce/pipes/*.java") + [
"src/it/crs4/pydoop/NoSeparatorTextOutputFormat.java"
]
self.dependencies = glob.glob('lib/*.jar')
self.properties = [(os.path.join("it/crs4/pydoop/mapreduce/pipes",
PROP_BN), PROP_FN)]
def __init__(self):
self.jar_name = pydoop.jar_name()
self.classpath = pydoop.hadoop_classpath()
self.java_files = glob.glob(
"src/it/crs4/pydoop/mapreduce/pipes/*.java"
) + ["src/it/crs4/pydoop/NoSeparatorTextOutputFormat.java"]
self.dependencies = glob.glob('lib/*.jar')
self.properties = [(
os.path.join("it/crs4/pydoop/mapreduce/pipes", PROP_BN),
PROP_FN
)]
def __init__(self, hadoop_vinfo):
self.hadoop_vinfo = hadoop_vinfo
self.jar_name = pydoop.jar_name(self.hadoop_vinfo)
self.classpath = pydoop.hadoop_classpath()
self.java_files = []
#if hadoop_vinfo.main >= (2, 2, 0):
if hadoop_vinfo.main >= (2, 0, 0) and hadoop_vinfo.is_yarn():
self.java_files.extend(
["src/v2/it/crs4/pydoop/NoSeparatorTextOutputFormat.java"])
self.java_files.extend(
glob.glob('src/v2/it/crs4/pydoop/pipes/*.java'))
self.java_files.extend(
glob.glob('src/v2/it/crs4/pydoop/mapreduce/pipes/*.java'))
else:
self.java_files.extend(
["src/v1/it/crs4/pydoop/NoSeparatorTextOutputFormat.java"])
self.java_files.extend(
glob.glob('src/v1/org/apache/hadoop/mapred/pipes/*.java'))
def __init__(self, hadoop_vinfo):
self.hadoop_vinfo = hadoop_vinfo
self.jar_name = pydoop.jar_name(self.hadoop_vinfo)
self.classpath = pydoop.hadoop_classpath()
self.java_files = []
self.dependencies = []
self.properties = []
if hadoop_vinfo.main >= (2, 0, 0) and \
(not hadoop_vinfo.is_cloudera() or hadoop_vinfo.is_yarn()):
# This version of Hadoop has the v2 pipes API
# FIXME: kinda hardwired to avro for now
self.properties.append((os.path.join(
"it/crs4/pydoop/mapreduce/pipes", PROP_BN),
PROP_FN))
self.java_files.extend(glob.glob(
'src/v2/it/crs4/pydoop/pipes/*.java'
))
self.java_files.extend(glob.glob(
'src/v2/it/crs4/pydoop/mapreduce/pipes/*.java'
))
# for things such as avro-mapreduce
self.dependencies.extend(glob.glob('lib/*.jar'))
else:
# Else we should be dealing with v1 pipes
self.java_files.extend(glob.glob(
'src/v1/org/apache/hadoop/mapred/pipes/*.java'
))
if hadoop_vinfo.has_mrv2():
# If the installation has MRv2 we need to use v2 I/O classes
self.java_files.extend(glob.glob(
'src/v2/it/crs4/pydoop/mapreduce/lib/output/*.java'
))
self.java_files.extend([
"src/v2/it/crs4/pydoop/NoSeparatorTextOutputFormat.java"
])
else:
self.java_files.extend([
"src/v1/it/crs4/pydoop/NoSeparatorTextOutputFormat.java"
])
