def run_distributed(self,data):
mod = asp_module.ASPModule(cache_dir = "/root/spark/examples/target/scala-2.9.1.final/classes/", use_scala=True)
scala_estimate= ast_tools.ConvertPyAST_ScalaAST().visit(self.estimate_ast)
scala_reduce = ast_tools.ConvertPyAST_ScalaAST().visit(self.reduce_ast)
scala_average = ast_tools.ConvertPyAST_ScalaAST().visit(self.average_ast)
scala_gen = SourceGenerator(self.TYPE_DECS)
rendered_scala_input_funcs = scala_gen.to_source(scala_estimate)+'\n' + scala_gen.to_source(scala_reduce) \
+'\n'+ scala_gen.to_source(scala_average)
rendered_scala = self.prepend_scala_blb_core_funcs(rendered_scala_input_funcs)
#should generate function signature for run here.. using codegen ?
rendered_scala_object = avro_backend.generate_scala_object("run","",rendered_scala)
#NOTE: must append outer to function name above to get the classname
# because of how scala_object created by avro_backend
mod.add_function("run_outer", rendered_scala_object, backend = "scala")
#NOTE: must add dependencies in make_dependency_jar so that slave nodes will receive proper files
time_stamp = str(int(round(time.time() * 1000)))
os.system('/root/BLB/distributed/make_dependency_jar ' + '/root/BLB/distributed/dependencies/' + time_stamp)
os.environ['DEPEND_LOC'] = '/root/BLB/distributed/dependencies/' + time_stamp +'/depend.jar'
num_spark_tasks = self.get_num_spark_tasks()
print 'Running Spark with', num_spark_tasks, 'tasks'
return mod.run_outer(data, num_spark_tasks, self.dim, self.num_subsamples, self.num_bootstraps, self.subsample_len_exp)
def run_distributed(self, data):
mod = asp_module.ASPModule(cache_dir="/root/spark/examples/target/scala-2.9.1.final/classes/", use_scala=True)
scala_estimate = ast_tools.ConvertPyAST_ScalaAST().visit(self.estimate_ast)
scala_reduce = ast_tools.ConvertPyAST_ScalaAST().visit(self.reduce_ast)
scala_average = ast_tools.ConvertPyAST_ScalaAST().visit(self.average_ast)
TYPE_DECS = (
["compute_estimate", ["BootstrapData"], "double"],
["reduce_bootstraps", [("list", "double")], "double"],
["average", [("array", "double")], "double"],
)
scala_gen = SourceGenerator(TYPE_DECS)
rendered_scala = (
scala_gen.to_source(scala_estimate)
+ "\n"
+ scala_gen.to_source(scala_reduce)
+ "\n"
+ scala_gen.to_source(scala_average)
)
rendered_scala = combine(rendered_scala)
rendered = avro_backend.generate_scala_object("run", "", rendered_scala)
# NOTE: must append outer to function name above to get the classname
# because of how scala_object created by avro_backend
mod.add_function("run_outer", rendered, backend="scala")
# NOTE: must add dependencies in make_dependency_jar so that slave nodes will receive proper files
time_stamp = str(int(round(time.time() * 1000)))
os.system("/root/BLB/distr_support/make_dependency_jar " + "/root/BLB/distr_support/dependencies/" + time_stamp)
os.environ["DEPEND_LOC"] = "/root/BLB/distr_support/dependencies/" + time_stamp + "/depend.jar"
email_filename = data[0]
model_filename = data[1]
return mod.run_outer(
email_filename, model_filename, self.dim, self.num_subsamples, self.num_bootstraps, self.subsample_len_exp
)
def run_distributed(self, data):
mod = asp_module.ASPModule(
cache_dir="/root/spark/examples/target/scala-2.9.1.final/classes/",
use_scala=True)
scala_estimate = ast_tools.ConvertPyAST_ScalaAST().visit(
self.estimate_ast)
scala_reduce = ast_tools.ConvertPyAST_ScalaAST().visit(self.reduce_ast)
scala_average = ast_tools.ConvertPyAST_ScalaAST().visit(
self.average_ast)
scala_gen = SourceGenerator(self.TYPE_DECS)
rendered_scala_input_funcs = scala_gen.to_source(scala_estimate)+'\n' + scala_gen.to_source(scala_reduce) \
+'\n'+ scala_gen.to_source(scala_average)
rendered_scala = self.prepend_scala_blb_core_funcs(
rendered_scala_input_funcs)
#should generate function signature for run here.. using codegen ?
rendered_scala_object = avro_backend.generate_scala_object(
"run", "", rendered_scala)
#NOTE: must append outer to function name above to get the classname
# because of how scala_object created by avro_backend
mod.add_function("run_outer", rendered_scala_object, backend="scala")
#NOTE: must add dependencies in make_dependency_jar so that slave nodes will receive proper files
time_stamp = str(int(round(time.time() * 1000)))
os.system('/root/BLB/distributed/make_dependency_jar ' +
'/root/BLB/distributed/dependencies/' + time_stamp)
os.environ[
'DEPEND_LOC'] = '/root/BLB/distributed/dependencies/' + time_stamp + '/depend.jar'
num_spark_tasks = self.get_num_spark_tasks()
print 'Running Spark with', num_spark_tasks, 'tasks'
return mod.run_outer(data, num_spark_tasks, self.dim,
self.num_subsamples, self.num_bootstraps,
self.subsample_len_exp)
