from caffe import layers as L
from caffe import params as P
"""
This function calls CaffeOnSpark to train the model. It is similar in
structure to the LeNext example, e.g., see
https://github.com/yahoo/CaffeOnSpark/wiki/GetStarted_python
In fact, the Python interface for CaffeOnSpark currently (July 2016)
allows for very little deviation from this format.
"""
if __name__ == '__main__':
sparkConf = SparkConf().setAppName("BeijingTomorrow").setMaster("local")
sc=SparkContext(conf=sparkConf)
registerContext(sc)
sqlContext = SQLContext(sc)
registerSQLContext(sqlContext)
cos=CaffeOnSpark(sc,sqlContext)
cfg=Config(sc)
this_file = os.path.abspath(inspect.getfile(inspect.currentframe()))
project_dir = os.path.dirname(os.path.dirname(os.path.dirname(this_file)))
visualProtoFile= os.path.join(project_dir,"resources/caffe_prototxt/beijing_pollution_solver_visual.prototxt")
visualModelFile= os.path.join(project_dir,"resources/caffe_models/beijing_pollution_model_visual.model")
aerosolProtoFile= os.path.join(project_dir,"resources/caffe_prototxt/beijing_pollution_solver_aerosol.prototxt")
aerosolModelFile= os.path.join(project_dir,"resources/caffe_models/beijing_pollution_model_aerosol.model")
cfg.protoFile = visualProtoFile
cfg.modelPath = 'file:' + visualModelFile
cfg.devices = 1
cfg.isFeature=True
def __init__(self,sc):
registerContext(sc)
wrapClass("com.yahoo.ml.caffe.tools.Conversions$")
self.__dict__['conversions']=toPython(getScalaSingleton("com.yahoo.ml.caffe.tools.Conversions"))
self.__dict__['sqlContext']=SQLContext(sc)
from caffe import SGDSolver
from caffe import layers as L
from caffe import params as P
"""
This function calls CaffeOnSpark to train the model. It is similar in
structure to the LeNext example, e.g., see
https://github.com/yahoo/CaffeOnSpark/wiki/GetStarted_python
In fact, the Python interface for CaffeOnSpark currently (July 2016)
allows for very little deviation from this format.
"""
if __name__ == '__main__':
sparkConf = SparkConf().setAppName("BeijingTomorrow").setMaster("local")
sc = SparkContext(conf=sparkConf)
registerContext(sc)
sqlContext = SQLContext(sc)
registerSQLContext(sqlContext)
cos = CaffeOnSpark(sc, sqlContext)
cfg = Config(sc)
this_file = os.path.abspath(inspect.getfile(inspect.currentframe()))
project_dir = os.path.dirname(os.path.dirname(os.path.dirname(this_file)))
visualProtoFile = os.path.join(
project_dir,
"resources/caffe_prototxt/beijing_pollution_solver_visual.prototxt")
visualModelFile = os.path.join(
project_dir,
"resources/caffe_models/beijing_pollution_model_visual.model")
aerosolProtoFile = os.path.join(
project_dir,
"resources/caffe_prototxt/beijing_pollution_solver_aerosol.prototxt")
def __init__(self,sc):
registerContext(sc)
self.vocab=wrapClass("com.yahoo.ml.caffe.tools.Vocab")
self.sqlContext=SQLContext(sc)
self.vocabObject=self.vocab(self.sqlContext)
def __init__(self, sc):
registerContext(sc)
self.vocab = wrapClass("com.yahoo.ml.caffe.tools.Vocab")
self.sqlContext = SQLContext(sc)
self.vocabObject = self.vocab(self.sqlContext)
def __init__(self, sc):
registerContext(sc)
wrapClass("com.yahoo.ml.caffe.tools.Conversions$")
self.__dict__['conversions'] = toPython(
getScalaSingleton("com.yahoo.ml.caffe.tools.Conversions"))
self.__dict__['sqlContext'] = SQLContext(sc)