def _extract_model(self, name, model, session, verbose=0):
"""
Extracts source code and any objects required to deploy the model.
Parameters
----------
name: string
name of your model
model: YhatModel
an instance of a Yhat model
session: globals()
your Python's session variables (i.e. "globals()")
verbose: int
log level
"""
code = ""
print "extracting model"
# detect subclasses pickle errors by attempting to pickle all of the
# objects in the global() session
# http://stackoverflow.com/a/1948057/2325264
# for k, v in session.items():
# try:
# terragon.dump(v, devnull)
# except terragon.pickle.PicklingError as e:
# try:
# base = type(v).__module__
# leaf = type(v).__class__.__name__
# parts = base.split(".")
# for i, _ in enumerate(parts):
# importpath = ".".join(parts[:i+1])
# globals()[importpath] = __import__(importpath)
# subclasses = []
# for attr in dir(v):
# if attr.startswith("__"):
# continue
# if types.TypeType == type(getattr(v, attr)):
# subclasses.append(attr)
# if not subclasses:
# continue
# for c in subclasses:
# truepath = ".".join([base, leaf, c])
# code += "\n" + "\n".join([
# "import " + base,
# "setattr(sys.modules['%s'], '%s', %s)" % (base, ".".join([base, leaf]), truepath),
# ])
# except Exception as e:
# print e
# except Exception as e:
# print e
if 1 == 2 and _get_source(YhatModel.execute) == _get_source(model.execute):
msg = """'execute' method was not implemented.
If you believe that you did implement the 'execute' method,
check to make sure there isn't an indentation error."""
raise Exception(msg)
bundle = save_function(model, session, verbose=verbose)
bundle["largefile"] = True
bundle["username"] = self.username
bundle["language"] = "python"
bundle["modelname"] = name
bundle["className"] = model.__name__
bundle["code"] = code + "\n" + bundle.get("code", "")
user_reqs = getattr(model, "REQUIREMENTS", "")
if isinstance(user_reqs, basestring):
user_reqs = [r for r in user_reqs.splitlines() if r]
if user_reqs:
print "model specified requirements"
for r in user_reqs:
if "==" not in r:
r = r + " (warning: unversioned)"
print " [+]", r
if DETECT_REQUIREMENTS:
# Requirements auto-detection.
bundle["reqs"] = "\n".join(
str(r) for r in merge(session, getattr(model, "REQUIREMENTS", ""))
)
else:
# The old way: REQUIREMENTS line.
reqs = getattr(model, "REQUIREMENTS", "")
if isinstance(reqs, list):
reqs = '\n'.join(reqs)
bundle["reqs"] = reqs
# make sure we freeze Yhat so we're sure we're using the right version
# this makes it a lot easier to upgrade the client
import yhat
bundle["reqs"] += '\n' + "yhat==" + yhat.__version__
bundle["reqs"] = bundle["reqs"].strip().replace('"', '').replace("'", "")
reqs = [r for r in bundle["reqs"].splitlines() if r]
detected_reqs = [r for r in reqs if r not in user_reqs]
if detected_reqs:
print "requirements automatically detected"
for r in detected_reqs:
print " [+]", r
# print modules information
modules = bundle.get("modules", [])
if modules:
print "model source files"
for module in modules:
name = module["name"]
parent_dir = module.get("parent_dir", "")
if parent_dir != "":
name = os.path.join(parent_dir, name)
print " [+]", name
objects = bundle.get("objects", {})
if objects:
print "model variables"
for name, pkl in objects.iteritems():
try:
obj = terragon.loads_from_base64(pkl)
t = type(obj)
del obj
except Exception as e:
print "ERROR pickling object:", name
raise e
size = 3. * float(len(pkl)) / 4.
print " [+]", name, t, sizeof_fmt(size)
return bundle
print "Future imports detected:\n%s" % __bundle['future'],
exec(__bundle["future"] + "\n" + __bundle["code"])
else:
exec(__bundle['code'])
print "Loading user objects:",
time.sleep(.01)
# Load each object into the `__tmp` variable then reassign it
for __name,__pk in __bundle['objects'].iteritems():
print "Loading object [%s]" % __name,
__tmp = None
try:
__tmp = pickle.loads(__pk)
except:
__tmp = terragon.loads_from_base64(__pk)
if __tmp is None:
print "Could not load object %s" % __name,
sys.exit(2)
exec('%s = __tmp' % (__name,))
# Create an instance of the user's model as `__YhatModel`
__model_class = __bundle["className"]
exec("__YhatModel = %s()" % (__model_class,))
# These things can be big. Important to delete the bundle
del __bundle
# the kernel communicates with the parent app through stdout. this function
# open's that channel, send a message, then closes it again
def __yhat_return(__msg):
def _extract_model(self,
name,
model,
session,
autodetect,
is_tensorflow=False,
verbose=0):
"""
Extracts source code and any objects required to deploy the model.
Parameters
----------
name: string
name of your model
model: YhatModel
an instance of a Yhat model
session: globals()
your Python's session variables (i.e. "globals()")
verbose: int
log level
"""
code = ""
print("extracting model")
levels = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG}
if verbose > 2:
verbose = 2
logging.basicConfig(format='[%(levelname)s]: %(message)s',
level=levels[verbose])
bundle = save_function(model, session)
bundle["largefile"] = True
bundle["username"] = self.username
bundle["language"] = "python"
bundle["modelname"] = name
bundle["className"] = model.__name__
bundle["code"] = code + "\n" + bundle.get("code", "")
# REQUIREMENTS
if DETECT_REQUIREMENTS and autodetect and getImplicitRequirements:
requirements = getImplicitRequirements(model, session)
elif getExplicitRequirements:
requirements = getExplicitRequirements(model, session)
else:
requirements = "\n".join(getattr(model, "REQUIREMENTS", []))
bundle["reqs"] = requirements
submodules = detect_explicit_submodules(model)
# MODULES
modules = bundle.get("modules", [])
existing_filenames = [
os.path.join(m['parent_dir'], m['name']) for m in modules
]
for submodule in submodules:
new_filename = os.path.join(submodule['parent_dir'],
submodule['name'])
if new_filename not in existing_filenames:
modules.append(submodule)
modules = sorted(
modules, key=lambda x: os.path.join(x['parent_dir'], x['name']))
if modules:
print("model source files")
for module in modules:
name = module["name"]
parent_dir = module.get("parent_dir", "")
if parent_dir != "":
name = os.path.join(parent_dir, name)
print(" [+]", name)
# OBJECTS
objects = bundle.get("objects", {})
if objects:
print("model variables")
for name, pkl in objects.items():
if name == '__tensorflow_session':
continue
try:
try:
obj = terragon.loads_from_base64(pkl)
except:
try:
obj = terragon.loads_spark_from_base64(
session['sc'], pkl)
except:
obj = terragon.load_pom_from_base64(pkl)
t = type(obj)
del obj
except Exception as e:
print("ERROR pickling object:", name)
raise e
size = 3. * float(len(pkl)) / 4.
print(" [+]", name, t, sizeof_fmt(size))
if is_tensorflow == True:
bundle['objects'][
'__tensorflow_session'] = terragon.sparkle.save_tensorflow_graph(
session['sess'])
return bundle
print "Future imports detected:\n%s" % __bundle['future'],
exec(__bundle["future"] + "\n" + __bundle["code"])
else:
exec(__bundle['code'])
print "Loading user objects:",
time.sleep(.01)
# Load each object into the `__tmp` variable then reassign it
for __name, __pk in __bundle['objects'].iteritems():
print "Loading object [%s]" % __name,
__tmp = None
try:
__tmp = pickle.loads(__pk)
except:
__tmp = terragon.loads_from_base64(__pk)
if __tmp is None:
print "Could not load object %s" % __name,
sys.exit(2)
exec('%s = __tmp' % (__name, ))
# Create an instance of the user's model as `__YhatModel`
__model_class = __bundle["className"]
exec("__YhatModel = %s()" % (__model_class, ))
# These things can be big. Important to delete the bundle
del __bundle
# the kernel communicates with the parent app through stdout. this function
# open's that channel, send a message, then closes it again
def _extract_model(self, name, model, session, autodetect, is_tensorflow=False, verbose=0):
"""
Extracts source code and any objects required to deploy the model.
Parameters
----------
name: string
name of your model
model: YhatModel
an instance of a Yhat model
session: globals()
your Python's session variables (i.e. "globals()")
verbose: int
log level
"""
code = ""
print("extracting model")
levels = {
0: logging.WARNING,
1: logging.INFO,
2: logging.DEBUG
}
if verbose > 2:
verbose = 2
logging.basicConfig(format='[%(levelname)s]: %(message)s', level=levels[verbose])
bundle = save_function(model, session)
bundle["largefile"] = True
bundle["username"] = self.username
bundle["language"] = "python"
bundle["modelname"] = name
bundle["className"] = model.__name__
bundle["code"] = code + "\n" + bundle.get("code", "")
# REQUIREMENTS
if DETECT_REQUIREMENTS and autodetect and getImplicitRequirements:
requirements = getImplicitRequirements(model, session)
elif getExplicitRequirements:
requirements = getExplicitRequirements(model, session)
else:
requirements = "\n".join(getattr(model, "REQUIREMENTS", []))
bundle["reqs"] = requirements
submodules = detect_explicit_submodules(model)
# MODULES
modules = bundle.get("modules", [])
existing_filenames = [os.path.join(m['parent_dir'], m['name']) for m in modules]
for submodule in submodules:
new_filename = os.path.join(submodule['parent_dir'], submodule['name'])
if new_filename not in existing_filenames:
modules.append(submodule)
modules = sorted(modules, key=lambda x: os.path.join(x['parent_dir'], x['name']))
if modules:
print("model source files")
for module in modules:
name = module["name"]
parent_dir = module.get("parent_dir", "")
if parent_dir != "":
name = os.path.join(parent_dir, name)
print(" [+]", name)
# OBJECTS
objects = bundle.get("objects", {})
if objects:
print("model variables")
for name, pkl in objects.items():
if name=='__tensorflow_session':
continue
try:
try:
obj = terragon.loads_from_base64(pkl)
except:
try:
obj = terragon.loads_spark_from_base64(session['sc'], pkl)
except:
obj = terragon.load_pom_from_base64(pkl)
t = type(obj)
del obj
except Exception as e:
print("ERROR pickling object:", name)
raise e
size = 3. * float(len(pkl)) / 4.
print(" [+]", name, t, sizeof_fmt(size))
if is_tensorflow==True:
bundle['objects']['__tensorflow_session'] = terragon.sparkle.save_tensorflow_graph(session['sess'])
return bundle
def _extract_model(self, name, model, session, autodetect, verbose=0):
"""
Extracts source code and any objects required to deploy the model.
Parameters
----------
name: string
name of your model
model: YhatModel
an instance of a Yhat model
session: globals()
your Python's session variables (i.e. "globals()")
verbose: int
log level
"""
code = ""
print "extracting model"
if 1 == 2 and _get_source(YhatModel.execute) == _get_source(
model.execute):
msg = """'execute' method was not implemented.
If you believe that you did implement the 'execute' method,
check to make sure there isn't an indentation error."""
raise Exception(msg)
bundle = save_function(model, session, verbose=verbose)
bundle["largefile"] = True
bundle["username"] = self.username
bundle["language"] = "python"
bundle["modelname"] = name
bundle["className"] = model.__name__
bundle["code"] = code + "\n" + bundle.get("code", "")
# REQUIREMENTS
if DETECT_REQUIREMENTS and autodetect:
requirements = getImplicitRequirements(model, session)
else:
requirements = getExplicitRequirmets(model, session)
bundle["reqs"] = requirements
# MODULES
modules = bundle.get("modules", [])
if modules:
print "model source files"
for module in modules:
name = module["name"]
parent_dir = module.get("parent_dir", "")
if parent_dir != "":
name = os.path.join(parent_dir, name)
print " [+]", name
# OBJETCS
objects = bundle.get("objects", {})
if objects:
print "model variables"
for name, pkl in objects.iteritems():
try:
obj = terragon.loads_from_base64(pkl)
t = type(obj)
del obj
except Exception as e:
print "ERROR pickling object:", name
raise e
size = 3. * float(len(pkl)) / 4.
print " [+]", name, t, sizeof_fmt(size)
return bundle