Beispiel #1
0
    def _extract_model(self, name, model, session, verbose=0):
        """
        Extracts source code and any objects required to deploy the model.

        Parameters
        ----------
        name: string
            name of your model
        model: YhatModel
            an instance of a Yhat model
        session: globals()
            your Python's session variables (i.e. "globals()")
        verbose: int
            log level
        """
        code = ""
        print "extracting model"
        # detect subclasses pickle errors by attempting to pickle all of the
        # objects in the global() session
        # http://stackoverflow.com/a/1948057/2325264
        # for k, v in session.items():
        #     try:
        #         terragon.dump(v, devnull)
        #     except terragon.pickle.PicklingError as e:
        #         try:
        #             base = type(v).__module__
        #             leaf = type(v).__class__.__name__
        #             parts = base.split(".")
        #             for i, _ in enumerate(parts):
        #                 importpath = ".".join(parts[:i+1])
        #                 globals()[importpath] = __import__(importpath)
        #             subclasses = []
        #             for attr in dir(v):
        #                 if attr.startswith("__"):
        #                     continue
        #                 if types.TypeType == type(getattr(v, attr)):
        #                     subclasses.append(attr)
        #             if not subclasses:
        #                 continue
        #             for c in subclasses:
        #                 truepath = ".".join([base, leaf, c])
        #                 code += "\n" + "\n".join([
        #                     "import " + base,
        #                     "setattr(sys.modules['%s'], '%s', %s)" % (base, ".".join([base, leaf]), truepath),
        #                 ])
        #         except Exception as e:
        #             print e
        #     except Exception as e:
        #         print e

        if 1 == 2 and _get_source(YhatModel.execute) == _get_source(model.execute):
            msg = """'execute' method was not implemented.
            If you believe that you did implement the 'execute' method,
            check to make sure there isn't an indentation error."""
            raise Exception(msg)

        bundle = save_function(model, session, verbose=verbose)
        bundle["largefile"] = True
        bundle["username"] = self.username
        bundle["language"] = "python"
        bundle["modelname"] = name
        bundle["className"] = model.__name__
        bundle["code"] = code + "\n" + bundle.get("code", "")


        user_reqs = getattr(model, "REQUIREMENTS", "")
        if isinstance(user_reqs, basestring):
            user_reqs = [r for r in user_reqs.splitlines() if r]
        if user_reqs:
            print "model specified requirements"
            for r in user_reqs:
                if "==" not in r:
                    r = r + " (warning: unversioned)"
                print " [+]", r

        if DETECT_REQUIREMENTS:
            # Requirements auto-detection.
            bundle["reqs"] = "\n".join(
                str(r) for r in merge(session, getattr(model, "REQUIREMENTS", ""))
            )

        else:
            # The old way: REQUIREMENTS line.
            reqs = getattr(model, "REQUIREMENTS", "")
            if isinstance(reqs, list):
                reqs = '\n'.join(reqs)
            bundle["reqs"] = reqs

            # make sure we freeze Yhat so we're sure we're using the right version
            # this makes it a lot easier to upgrade the client
            import yhat
            bundle["reqs"] += '\n' + "yhat==" + yhat.__version__
            bundle["reqs"] = bundle["reqs"].strip().replace('"', '').replace("'", "")

        reqs = [r for r in bundle["reqs"].splitlines() if r]

        detected_reqs = [r for r in reqs if r not in user_reqs]
        if detected_reqs:
            print "requirements automatically detected"
            for r in detected_reqs:
                print " [+]", r

        # print modules information
        modules = bundle.get("modules", [])
        if modules:
            print "model source files"
            for module in modules:
                name = module["name"]
                parent_dir = module.get("parent_dir", "")
                if parent_dir != "":
                    name = os.path.join(parent_dir, name)
                print " [+]", name

        objects = bundle.get("objects", {})
        if objects:
            print "model variables"
            for name, pkl in objects.iteritems():
                try:
                    obj = terragon.loads_from_base64(pkl)
                    t = type(obj)
                    del obj
                except Exception as e:
                    print "ERROR pickling object:", name
                    raise e
                size = 3. * float(len(pkl)) / 4.
                print " [+]", name, t, sizeof_fmt(size)

        return bundle
Beispiel #2
0
    print "Future imports detected:\n%s" % __bundle['future'],
    exec(__bundle["future"] + "\n" + __bundle["code"])
else:
    exec(__bundle['code'])

print "Loading user objects:",
time.sleep(.01)

# Load each object into the `__tmp` variable then reassign it
for __name,__pk in __bundle['objects'].iteritems():
    print "Loading object [%s]" % __name,
    __tmp = None
    try:
	__tmp = pickle.loads(__pk)
    except:
	__tmp = terragon.loads_from_base64(__pk)
    if __tmp is None:
	print "Could not load object %s" % __name,
	sys.exit(2)
    exec('%s = __tmp' % (__name,))

# Create an instance of the user's model as `__YhatModel`
__model_class = __bundle["className"]
exec("__YhatModel = %s()" % (__model_class,))

# These things can be big. Important to delete the bundle
del __bundle

# the kernel communicates with the parent app through stdout. this function
# open's that channel, send a message, then closes it again
def __yhat_return(__msg):
Beispiel #3
0
    def _extract_model(self,
                       name,
                       model,
                       session,
                       autodetect,
                       is_tensorflow=False,
                       verbose=0):
        """
        Extracts source code and any objects required to deploy the model.

        Parameters
        ----------
        name: string
            name of your model
        model: YhatModel
            an instance of a Yhat model
        session: globals()
            your Python's session variables (i.e. "globals()")
        verbose: int
            log level
        """
        code = ""
        print("extracting model")

        levels = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG}
        if verbose > 2:
            verbose = 2

        logging.basicConfig(format='[%(levelname)s]: %(message)s',
                            level=levels[verbose])
        bundle = save_function(model, session)
        bundle["largefile"] = True
        bundle["username"] = self.username
        bundle["language"] = "python"
        bundle["modelname"] = name
        bundle["className"] = model.__name__
        bundle["code"] = code + "\n" + bundle.get("code", "")

        # REQUIREMENTS
        if DETECT_REQUIREMENTS and autodetect and getImplicitRequirements:
            requirements = getImplicitRequirements(model, session)
        elif getExplicitRequirements:
            requirements = getExplicitRequirements(model, session)
        else:
            requirements = "\n".join(getattr(model, "REQUIREMENTS", []))

        bundle["reqs"] = requirements

        submodules = detect_explicit_submodules(model)
        # MODULES
        modules = bundle.get("modules", [])
        existing_filenames = [
            os.path.join(m['parent_dir'], m['name']) for m in modules
        ]
        for submodule in submodules:
            new_filename = os.path.join(submodule['parent_dir'],
                                        submodule['name'])
            if new_filename not in existing_filenames:
                modules.append(submodule)

        modules = sorted(
            modules, key=lambda x: os.path.join(x['parent_dir'], x['name']))
        if modules:
            print("model source files")
            for module in modules:
                name = module["name"]
                parent_dir = module.get("parent_dir", "")
                if parent_dir != "":
                    name = os.path.join(parent_dir, name)
                print(" [+]", name)

        # OBJECTS
        objects = bundle.get("objects", {})
        if objects:
            print("model variables")
            for name, pkl in objects.items():
                if name == '__tensorflow_session':
                    continue
                try:
                    try:
                        obj = terragon.loads_from_base64(pkl)
                    except:
                        try:
                            obj = terragon.loads_spark_from_base64(
                                session['sc'], pkl)
                        except:
                            obj = terragon.load_pom_from_base64(pkl)

                    t = type(obj)
                    del obj
                except Exception as e:
                    print("ERROR pickling object:", name)
                    raise e
                size = 3. * float(len(pkl)) / 4.
                print(" [+]", name, t, sizeof_fmt(size))

        if is_tensorflow == True:
            bundle['objects'][
                '__tensorflow_session'] = terragon.sparkle.save_tensorflow_graph(
                    session['sess'])

        return bundle
Beispiel #4
0
    print "Future imports detected:\n%s" % __bundle['future'],
    exec(__bundle["future"] + "\n" + __bundle["code"])
else:
    exec(__bundle['code'])

print "Loading user objects:",
time.sleep(.01)

# Load each object into the `__tmp` variable then reassign it
for __name, __pk in __bundle['objects'].iteritems():
    print "Loading object [%s]" % __name,
    __tmp = None
    try:
        __tmp = pickle.loads(__pk)
    except:
        __tmp = terragon.loads_from_base64(__pk)
    if __tmp is None:
        print "Could not load object %s" % __name,
        sys.exit(2)
    exec('%s = __tmp' % (__name, ))

# Create an instance of the user's model as `__YhatModel`
__model_class = __bundle["className"]
exec("__YhatModel = %s()" % (__model_class, ))

# These things can be big. Important to delete the bundle
del __bundle


# the kernel communicates with the parent app through stdout. this function
# open's that channel, send a message, then closes it again
Beispiel #5
0
    def _extract_model(self, name, model, session, autodetect, is_tensorflow=False, verbose=0):
        """
        Extracts source code and any objects required to deploy the model.

        Parameters
        ----------
        name: string
            name of your model
        model: YhatModel
            an instance of a Yhat model
        session: globals()
            your Python's session variables (i.e. "globals()")
        verbose: int
            log level
        """
        code = ""
        print("extracting model")

        levels = {
            0: logging.WARNING,
            1: logging.INFO,
            2: logging.DEBUG
        }
        if verbose > 2:
            verbose = 2

        logging.basicConfig(format='[%(levelname)s]: %(message)s', level=levels[verbose])
        bundle = save_function(model, session)
        bundle["largefile"] = True
        bundle["username"] = self.username
        bundle["language"] = "python"
        bundle["modelname"] = name
        bundle["className"] = model.__name__
        bundle["code"] = code + "\n" + bundle.get("code", "")

        # REQUIREMENTS
        if DETECT_REQUIREMENTS and autodetect and getImplicitRequirements:
            requirements = getImplicitRequirements(model, session)
        elif getExplicitRequirements:
            requirements = getExplicitRequirements(model, session)
        else:
            requirements = "\n".join(getattr(model, "REQUIREMENTS", []))

        bundle["reqs"] = requirements

        submodules = detect_explicit_submodules(model)
        # MODULES
        modules = bundle.get("modules", [])
        existing_filenames = [os.path.join(m['parent_dir'], m['name']) for m in modules]
        for submodule in submodules:
            new_filename = os.path.join(submodule['parent_dir'], submodule['name'])
            if new_filename not in existing_filenames:
                modules.append(submodule)

        modules = sorted(modules, key=lambda x: os.path.join(x['parent_dir'], x['name']))
        if modules:
            print("model source files")
            for module in modules:
                name = module["name"]
                parent_dir = module.get("parent_dir", "")
                if parent_dir != "":
                    name = os.path.join(parent_dir, name)
                print(" [+]", name)

        # OBJECTS
        objects = bundle.get("objects", {})
        if objects:
            print("model variables")
            for name, pkl in objects.items():
                if name=='__tensorflow_session':
                    continue
                try:
                    try:
                        obj = terragon.loads_from_base64(pkl)
                    except:
                        try:
                            obj = terragon.loads_spark_from_base64(session['sc'], pkl)
                        except:
                            obj = terragon.load_pom_from_base64(pkl)

                    t = type(obj)
                    del obj
                except Exception as e:
                    print("ERROR pickling object:", name)
                    raise e
                size = 3. * float(len(pkl)) / 4.
                print(" [+]", name, t, sizeof_fmt(size))

        if is_tensorflow==True:
            bundle['objects']['__tensorflow_session'] = terragon.sparkle.save_tensorflow_graph(session['sess'])

        return bundle
Beispiel #6
0
    def _extract_model(self, name, model, session, autodetect, verbose=0):
        """
        Extracts source code and any objects required to deploy the model.

        Parameters
        ----------
        name: string
            name of your model
        model: YhatModel
            an instance of a Yhat model
        session: globals()
            your Python's session variables (i.e. "globals()")
        verbose: int
            log level
        """
        code = ""
        print "extracting model"

        if 1 == 2 and _get_source(YhatModel.execute) == _get_source(
                model.execute):
            msg = """'execute' method was not implemented.
            If you believe that you did implement the 'execute' method,
            check to make sure there isn't an indentation error."""
            raise Exception(msg)

        bundle = save_function(model, session, verbose=verbose)
        bundle["largefile"] = True
        bundle["username"] = self.username
        bundle["language"] = "python"
        bundle["modelname"] = name
        bundle["className"] = model.__name__
        bundle["code"] = code + "\n" + bundle.get("code", "")

        # REQUIREMENTS
        if DETECT_REQUIREMENTS and autodetect:
            requirements = getImplicitRequirements(model, session)
        else:
            requirements = getExplicitRequirmets(model, session)
        bundle["reqs"] = requirements

        # MODULES
        modules = bundle.get("modules", [])
        if modules:
            print "model source files"
            for module in modules:
                name = module["name"]
                parent_dir = module.get("parent_dir", "")
                if parent_dir != "":
                    name = os.path.join(parent_dir, name)
                print " [+]", name

        # OBJETCS
        objects = bundle.get("objects", {})
        if objects:
            print "model variables"
            for name, pkl in objects.iteritems():
                try:
                    obj = terragon.loads_from_base64(pkl)
                    t = type(obj)
                    del obj
                except Exception as e:
                    print "ERROR pickling object:", name
                    raise e
                size = 3. * float(len(pkl)) / 4.
                print " [+]", name, t, sizeof_fmt(size)

        return bundle