def test_treasury_directory_out(self):
logging.info("Testing bsoninput from directory path")
PARAMETERS = DEFAULT_PARAMETERS.copy()
PARAMETERS[
"mongo.job.input.format"] = "com.mongodb.hadoop.BSONFileInputFormat"
PARAMETERS[
"mongo.job.output.format"] = "com.mongodb.hadoop.BSONFileOutputFormat"
PARAMETERS["mapred.output.dir"] = os.path.join(
"file://" + self.temp_outdir, "mongo_hadoop_out")
PARAMETERS["mapred.max.split.size"] = '200000'
PARAMETERS[
"bson.pathfilter.class"] = 'com.mongodb.hadoop.BSONPathFilter'
logging.info(PARAMETERS)
runbsonjob(os.path.join("file://" + self.temp_outdir, "mongo_hadoop"),
PARAMETERS, self.server_hostname)
paths = os.listdir(os.path.join(self.temp_outdir, 'mongo_hadoop_out'))
for p in paths:
fullpath = os.path.join(self.temp_outdir, "mongo_hadoop_out", p)
if not fullpath.endswith('.bson'):
continue
mongo_manager.mongo_restore(self.server_hostname, "mongo_hadoop",
"yield_historical.out", fullpath)
out_col = self.server.connection(
)['mongo_hadoop']['yield_historical.out']
self.assertTrue(compare_results(out_col))
def test_treasury(self):
self.temp_outdir = tempfile.mkdtemp(prefix='hadooptest_', dir=TEMPDIR)
logging.info("Testing bson output")
PARAMETERS = DEFAULT_PARAMETERS.copy()
PARAMETERS["mongo.job.output.format"] = "com.mongodb.hadoop.BSONFileOutputFormat"
PARAMETERS["mapred.output.file"] = os.path.join("file://" + self.temp_outdir, "mongo_hadoop","results.bson")
runjob(self.server_hostname, PARAMETERS)
mongo_manager.mongo_restore(self.server_hostname, "mongo_hadoop", "yield_historical.out", os.path.join(self.temp_outdir, "mongo_hadoop","results.bson"))
out_col = self.server.connection()['mongo_hadoop']['yield_historical.out']
self.assertTrue(compare_results(out_col))
def test_treasury(self):
self.temp_outdir = tempfile.mkdtemp(prefix='hadooptest_', dir=TEMPDIR)
logging.info("Testing bson output")
PARAMETERS = DEFAULT_PARAMETERS.copy()
PARAMETERS[
"mongo.job.output.format"] = "com.mongodb.hadoop.BSONFileOutputFormat"
PARAMETERS["mapred.output.file"] = os.path.join(
"file://" + self.temp_outdir, "mongo_hadoop", "results.bson")
runjob(self.server_hostname, PARAMETERS)
mongo_manager.mongo_restore(
self.server_hostname, "mongo_hadoop", "yield_historical.out",
os.path.join(self.temp_outdir, "mongo_hadoop", "results.bson"))
out_col = self.server.connection(
)['mongo_hadoop']['yield_historical.out']
self.assertTrue(compare_results(out_col))
def test_treasury_directory_out(self):
logging.info("Testing bsoninput from directory path")
PARAMETERS = DEFAULT_PARAMETERS.copy()
PARAMETERS["mongo.job.input.format"] = "com.mongodb.hadoop.BSONFileInputFormat"
PARAMETERS["mongo.job.output.format"] = "com.mongodb.hadoop.BSONFileOutputFormat"
PARAMETERS["mapred.output.dir"] = os.path.join("file://" + self.temp_outdir, "mongo_hadoop_out")
PARAMETERS["mapred.max.split.size"] = '200000'
PARAMETERS["bson.pathfilter.class"] = 'com.mongodb.hadoop.BSONPathFilter'
logging.info(PARAMETERS)
runbsonjob(os.path.join("file://" + self.temp_outdir, "mongo_hadoop"), PARAMETERS, self.server_hostname)
paths = os.listdir(os.path.join(self.temp_outdir,'mongo_hadoop_out'))
for p in paths:
fullpath = os.path.join(self.temp_outdir, "mongo_hadoop_out", p)
if not fullpath.endswith('.bson'):
continue
mongo_manager.mongo_restore(self.server_hostname, "mongo_hadoop", "yield_historical.out",fullpath)
out_col = self.server.connection()['mongo_hadoop']['yield_historical.out']
self.assertTrue(compare_results(out_col))