Beispiel #1
0
def validate_result(result, filter_plugins = True, print_files = False):
    '''Validate cached.filelist for the given result'''
        
    path_to_report_dir = result.get_report_dir()
    path_to_file = os.path.join(path_to_report_dir, 'cached.filelist')
    print "validating %s" % path_to_file
    
    if result.isThumbnail:
        print("cannot validate: thumbnail")
        return None
    
#    for this_type in dmtypes.FILESET_TYPES:
#        dmfs = result.get_filestat(this_type)
#        if dmfs.action_state != 'L':
#            print "cannot validate: not local"
#            return None
    dmfs = result.get_filestat(dmtypes.SIG)

    # Get the cached filelist from cached.filelist file
    try:
        with open(path_to_file, 'rb') as fhandle:
            pickle = Unpickler(fhandle)
            cached_filelist = pickle.load()
    except IOError as ioerror:
        print "%s" % ioerror
        return None
    
    # Get a list of files on the filesystem currently
    dirs = [dmfs.result.get_report_dir(), dmfs.result.experiment.expDir]
    current_fs_filelist = get_walk_filelist(dirs)

    # Ignore plugin_out directories
    if filter_plugins:
        current_fs_filelist = [filename for filename in current_fs_filelist if not '/plugin_out' in filename]
        
    # Ignore the cached.filelist file
    current_fs_filelist = [filename for filename in current_fs_filelist if not 'cached.filelist' in filename]
    # Ignore the status.txt file
    current_fs_filelist = [filename for filename in current_fs_filelist if not 'status.txt' in filename]
    # Ignore the serialized_*.json file
    current_fs_filelist = [filename for filename in current_fs_filelist if not 'serialized_' in filename]
            
                
    # See if there are differences
    #leftovers = list(set(cached_filelist) - set(current_fs_filelist))
    #N.B. This difference here will tell us, "Of the files in the filesystem right now, how many are NOT in the cached.filelist file"
    #Even if the cached.filelist contains more files than are currently on the filesystem.
    #I am thinking this means we do not care if any action_state is not 'L'.  It doesn't matter because we are looking for deficient
    #cached.filelist.
    leftovers = list(set(current_fs_filelist) - set(cached_filelist))
    if print_files:
        for i, item in enumerate(leftovers):
            if not i: print "FILES MISSING FROM CACHED.FILELIST:"
            print item
    else:
        if len(leftovers) > 0:
            print "FILES MISSING FROM CACHED.FILELIST: %d" % len(leftovers)
    print "- %s\n" % ("Not valid" if len(leftovers) > 0 else "Valid")
                
    return None
Beispiel #2
0
 def test_config_and_collector_pickling(self):
     from cPickle import Pickler, Unpickler
     dir1 = self.tmpdir.ensure("somedir", dir=1)
     config = py.test.config._reparse([self.tmpdir])
     col = config.getfsnode(config.topdir)
     col1 = col.join(dir1.basename)
     assert col1.parent is col 
     io = py.std.cStringIO.StringIO()
     pickler = Pickler(io)
     pickler.dump(config)
     pickler.dump(col)
     pickler.dump(col1)
     pickler.dump(col)
     io.seek(0) 
     unpickler = Unpickler(io)
     newconfig = unpickler.load()
     topdir = self.tmpdir.ensure("newtopdir", dir=1)
     newconfig._initafterpickle(topdir)
     topdir.ensure("somedir", dir=1)
     newcol = unpickler.load()
     newcol2 = unpickler.load()
     newcol3 = unpickler.load()
     assert newcol2._config is newconfig 
     assert newcol2.parent == newcol 
     assert newcol._config is newconfig
     assert newconfig.topdir == topdir
     assert newcol3 is newcol
     assert newcol.fspath == topdir 
     assert newcol2.fspath.basename == dir1.basename
     assert newcol2.fspath.relto(topdir)
Beispiel #3
0
def unpickle(file_path=None):
    """Load pickled Python objects from a file.

    Almost like ``cPickle.load(open(file_path))``, but also loads object saved
    with older versions of Eelbrain, and allows using a system file dialog to
    select a file.

    Parameters
    ----------
    file_path : None | str
        Path to a pickled file. If None, a system file dialog will be used. If
        the user cancels the file dialog, a RuntimeError is raised.
    """
    if file_path is None:
        filetypes = [("Pickles (*.pickled)", '*.pickled'), ("All files", '*')]
        file_path = ui.ask_file("Select File to Unpickle", "Select a pickled "
                                "file to unpickle", filetypes)
    if file_path is False:
        raise RuntimeError("User canceled")

    if not os.path.exists(file_path):
        new_path = os.extsep.join((file_path, 'pickled'))
        if os.path.exists(new_path):
            file_path = new_path

    with open(file_path, 'r') as fid:
        unpickler = Unpickler(fid)
        unpickler.find_global = map_paths
        obj = unpickler.load()

    return obj
Beispiel #4
0
 def load_weights(self):
     ''' Loads the stored data from previous sessions, if possible.'''
     valid = False
     try: fp = open(self.filename, 'r')
     except IOError:
         self.log_debug(11, "Couldn't read stats file '%s'", self.filename)
     else:
         self.log_debug(11, "Loading stats file '%s'", self.filename)
         try:
             pickler = Unpickler(fp)
             self.input_headers = pickler.load()
             wi = pickler.load()
             self.output_headers = pickler.load()
             wo = pickler.load()
             #self.seasons = pickler.load()
             #self.powers = pickler.load()
             #self.locs = pickler.load()
             #self.provinces = pickler.load()
             #self.centers = pickler.load()
             #self.coastals = pickler.load()
             #self.coastlines = pickler.load()
             #self.borders = pickler.load()
         finally:
             fp.close()
         
         ni = len(self.input_headers)
         no = len(self.output_headers)
         nh = len(wo)
         self.log_debug(7, "%d inputs => %d hidden => %d outputs",
                 ni, nh, no)
         self.net = NN(ni, nh, no, wi, wo)
         valid = True
     return valid
def deepCopy(obj):
    stream = StringIO()
    p = Pickler(stream, 1)
    p.dump(obj)
    stream.seek(0)
    u = Unpickler(stream)
    return u.load()
Beispiel #6
0
    def setklassstate(self, object):
        try:
            oid=object._p_oid
            __traceback_info__=oid
            p, serial = self._storage.load(oid, self._version)
            file=StringIO(p)
            unpickler=Unpickler(file)
            unpickler.persistent_load=self._persistent_load

            copy = unpickler.load()

            klass, args = copy

            if klass is not ExtensionKlass:
                LOG('ZODB',ERROR,
                    "Unexpected klass when setting class state on %s"
                    % getattr(object,'__name__','(?)'))
                return

            copy=apply(klass,args)
            object.__dict__.clear()
            object.__dict__.update(copy.__dict__)

            object._p_oid=oid
            object._p_jar=self
            object._p_changed=0
            object._p_serial=serial
        except:
            LOG('ZODB',ERROR, 'setklassstate failed', error=sys.exc_info())
            raise
Beispiel #7
0
    def load_state(self, state):
        """Load an image_set_list's state from the string returned from save_state"""

        self.__image_sets = []
        self.__image_sets_by_key = {}

        # Make a safe unpickler
        p = Unpickler(StringIO(state))

        def find_global(module_name, class_name):
            logger.debug("Pickler wants %s:%s", module_name, class_name)
            if module_name not in ("numpy", "numpy.core.multiarray"):
                logger.critical(
                    "WARNING WARNING WARNING - your batch file has asked to load %s.%s."
                    " If this looks in any way suspicious please contact us at www.cellprofiler.org",
                    module_name,
                    class_name,
                )
                raise ValueError("Illegal attempt to unpickle class %s.%s", (module_name, class_name))
            __import__(module_name)
            mod = sys.modules[module_name]
            return getattr(mod, class_name)

        p.find_global = find_global

        count = p.load()
        all_keys = [p.load() for i in range(count)]
        self.__legacy_fields = p.load()
        #
        # Have to do in this order in order for the image set's
        # legacy_fields property to hook to the right legacy_fields
        #
        for i in range(count):
            self.get_image_set(all_keys[i])
Beispiel #8
0
 def network_setup(self):
     dict = {}
     self.net_mtime = self.network_connect()
     if self.net_mtime != None:
         if os.path.exists( self.user_data_file ):
             self.network_update()
             local_mtime = int(os.stat( self.user_data_file ).st_mtime)
             if local_mtime > self.net_mtime:
                 self.network_upload()
             elif local_mtime < self.net_mtime:
                 self.network_download()
         else:
             self.network_download()
     if os.path.exists( self.user_data_file ):
         ph = open( self.user_data_file )
         dict = Unpickler( ph ).load()[-1]
     if not os.path.exists( self.user_data_file ) and self.net_mtime == None:
         ph = open( self.user_data_file, 'w+' )
         Pickler( ph ).dump( dict )
         ph.close()
         os.utime( self.user_data_file, (0,0) )
     last_month = dateDelta( date.today() ).get_last_month()
     keys = dict.keys()
     keys.sort()
     for key in keys:
         if key[:7] < '%0.4d-%0.2d' % (last_month.year,last_month.month):
             dict.pop( key )
         else:
             break
     self.freetime.update( dict )
Beispiel #9
0
def unpickle(file_path=None):
    """Load pickled Python objects from a file.

    Almost like ``cPickle.load(open(file_path))``, but also loads object saved
    with older versions of Eelbrain, and allows using a system file dialog to
    select a file.

    Parameters
    ----------
    file_path : None | str
        Path to a pickled file. If None (default), a system file dialog will be
        shown. If the user cancels the file dialog, a RuntimeError is raised.
    """
    if file_path is None:
        filetypes = [("Pickles (*.pickled)", '*.pickled'), ("All files", '*')]
        file_path = ui.ask_file("Select File to Unpickle", "Select a pickled "
                                "file to unpickle", filetypes)
        if file_path is False:
            raise RuntimeError("User canceled")
        else:
            print repr(file_path)
    else:
        file_path = os.path.expanduser(file_path)
        if not os.path.exists(file_path):
            new_path = os.extsep.join((file_path, 'pickled'))
            if os.path.exists(new_path):
                file_path = new_path

    with open(file_path, 'r') as fid:
        unpickler = Unpickler(fid)
        unpickler.find_global = map_paths
        obj = unpickler.load()

    return obj
Beispiel #10
0
def zodb_unpickle(data):
    """Unpickle an object stored using the format expected by ZODB."""
    f = StringIO(data)
    u = Unpickler(f)
    u.persistent_load = persistent_load
    klass_info = u.load()
    if isinstance(klass_info, types.TupleType):
        if isinstance(klass_info[0], type):
            # Unclear:  what is the second part of klass_info?
            klass, xxx = klass_info
            assert not xxx
        else:
            if isinstance(klass_info[0], tuple):
                modname, klassname = klass_info[0]
            else:
                modname, klassname = klass_info
            if modname == "__main__":
                ns = globals()
            else:
                mod = import_helper(modname)
                ns = mod.__dict__
            try:
                klass = ns[klassname]
            except KeyError:
                print >> sys.stderr, "can't find %s in %r" % (klassname, ns)
        inst = klass()
    else:
        raise ValueError("expected class info: %s" % repr(klass_info))
    state = u.load()
    inst.__setstate__(state)
    return inst
 def __init__(self, inp_filename):
     self.filename = inp_filename
     with open(self.filename, 'rb') as inpfile:
         reader = Unpickler(inpfile)
         self.numgroups = reader.load()
         self.tot_msgs = reader.load()
         self.num_msgs = reader.load()
Beispiel #12
0
 def getNewState(cls, file):
     # Would like to do load(file) here... but it doesn't work with universal line endings, see Python bug 1724366
     from cStringIO import StringIO
     unpickler = Unpickler(StringIO(file.read()))
     # Magic to keep us backward compatible in the face of packages changing...
     unpickler.find_global = cls.findGlobal
     return unpickler.load()
 def setstate(self, object):
     """
     Unlike the 'stock' Connection class' setstate, this method
     doesn't raise ConflictErrors.  This is potentially dangerous
     for applications that need absolute consistency, but
     sessioning is not one of those.
     """
     oid=object._p_oid
     invalid = self._invalid
     if invalid(None):
         # only raise a conflict if there was
         # a mass invalidation, but not if we see this
         # object's oid as invalid
         raise ConflictError, `oid`
     p, serial = self._storage.load(oid, self._version)
     file=StringIO(p)
     unpickler=Unpickler(file)
     unpickler.persistent_load=self._persistent_load
     unpickler.load()
     state = unpickler.load()
     if hasattr(object, '__setstate__'):
         object.__setstate__(state)
     else:
         d=object.__dict__
         for k,v in state.items(): d[k]=v
     object._p_serial=serial
Beispiel #14
0
    def got_data(self, port_agent_packet):
        """
        Called by the instrument connection when data is available.

        Also add data to the chunker and when received call got_chunk
        to publish results.
        """

        data_length = port_agent_packet.get_data_length()
        data = port_agent_packet.get_data()
        timestamp = port_agent_packet.get_timestamp()

        log.debug("Got Data: %s" % data)
        log.debug("Add Port Agent Timestamp: %s" % timestamp)

        unpickler = Unpickler(StringIO(data))
        # Disable class unpickling, for security; record should be all
        # built-in types. Note this only works with cPickle.
        unpickler.find_global = None

        # pkt is an antelope.Pkt.Packet object converted to a dict. Refer to
        # the documentation for the Antelope Python bindings for compelete
        # details.

        pkt = unpickler.load()

        for particle in self._particle_factory(pkt, timestamp):
            self._publish_particle(particle)
def _cloneByPickle(self, obj):
    """Returns a deep copy of a ZODB object, loading ghosts as needed.
    """
    modifier = getToolByName(self, 'portal_modifier')
    callbacks = modifier.getOnCloneModifiers(obj)
    if callbacks is not None:
        pers_id, pers_load, inside_orefs, outside_orefs = callbacks[0:4]
    else:
        inside_orefs, outside_orefs = (), ()

    stream = StringIO()
    p = Pickler(stream, 1)
    if callbacks is not None:
        p.persistent_id = pers_id
    cmf_uid = getattr(obj, 'cmf_uid', None)
    if IUniqueIdAnnotation.providedBy(cmf_uid):
        setattr(obj, 'cmf_uid', cmf_uid())
    try:
        p.dump(aq_base(obj))
    except TypeError:
        # just try again, this then seems to work
        # WTF?
        p.dump(aq_base(obj))
    approxSize = stream.tell()
    stream.seek(0)
    u = Unpickler(stream)
    if callbacks is not None:
        u.persistent_load = pers_load
    return approxSize, u.load(), inside_orefs, outside_orefs
Beispiel #16
0
def cloneByPickle(obj, ignore_list=()):
    """Makes a copy of a ZODB object, loading ghosts as needed.

    Ignores specified objects along the way, replacing them with None
    in the copy.
    """
    ignore_dict = {}
    for o in ignore_list:
        ignore_dict[id(o)] = o

    def persistent_id(ob, ignore_dict=ignore_dict):
        if ignore_dict.has_key(id(ob)):
            return 'ignored'
        if getattr(ob, '_p_changed', 0) is None:
            ob._p_changed = 0
        return None

    def persistent_load(ref):
        assert ref == 'ignored'
        # Return a placeholder object that will be replaced by
        # removeNonVersionedData().
        placeholder = SimpleItem()
        placeholder.id = "ignored_subobject"
        return placeholder

    stream = StringIO()
    p = Pickler(stream, 1)
    p.persistent_id = persistent_id
    p.dump(obj)
    stream.seek(0)
    u = Unpickler(stream)
    u.persistent_load = persistent_load
    return u.load()
Beispiel #17
0
 def test_config_and_collector_pickling(self, testdir):
     from cPickle import Pickler, Unpickler
     tmpdir = testdir.tmpdir
     dir1 = tmpdir.ensure("somedir", dir=1)
     config = testdir.parseconfig()
     col = config.getfsnode(config.topdir)
     col1 = col.join(dir1.basename)
     assert col1.parent is col
     io = py.std.cStringIO.StringIO()
     pickler = Pickler(io)
     pickler.dump(col)
     pickler.dump(col1)
     pickler.dump(col)
     io.seek(0)
     unpickler = Unpickler(io)
     topdir = tmpdir.ensure("newtopdir", dir=1)
     topdir.ensure("somedir", dir=1)
     old = topdir.chdir()
     try:
         newcol = unpickler.load()
         newcol2 = unpickler.load()
         newcol3 = unpickler.load()
         assert newcol2.config is newcol.config
         assert newcol2.parent == newcol
         assert newcol2.config.topdir.realpath() == topdir.realpath()
         assert newcol.fspath.realpath() == topdir.realpath()
         assert newcol2.fspath.basename == dir1.basename
         assert newcol2.fspath.relto(newcol2.config.topdir)
     finally:
         old.chdir()
Beispiel #18
0
    def network_update(self):
        #
        # We have network, we have local file, we may not have
        # remote file.
        if self.userkey in self.sftp.listdir( self.category ):
            local_mtime = os.stat( self.user_data_file ).st_mtime
            if int(local_mtime) == 0:
                self.sftp.get( self.category + '/' + self.userkey, self.user_data_file + '.tmp' )
                ph = open( self.user_data_file + '.tmp' )
                net_data = Unpickler( ph ).load()[-1]
                ph.close()
                os.unlink( self.user_data_file + '.tmp' )

                ph = open( self.user_data_file )
                local_data = Unpickler( ph ).load()[-1]
                ph.close()

                local_data.update( net_data )
                self.freetime = freeTime( dict=local_data )

                ph = open( self.user_data_file, 'w+' )
                Pickler( ph ).dump( self.bundle_data() )
                ph.close()
        else:
            self.sftp.put( self.user_data_file, self.category + '/' + self.userkey )
            mtime = self.sftp.stat( self.category + '/' + self.userkey ).st_mtime
            os.utime( self.user_data_file, (mtime,mtime) )
Beispiel #19
0
 def replay(self, name):
     replay_file_path = os.path.join(PROJECTS, self.name, "replays", name)
     replay = None
     with open(replay_file_path, "r") as fp:
         upkl = Unpickler(fp)
         replay = upkl.load()
     return replay
Beispiel #20
0
def zodb_unpickle(data):
    """Unpickle an object stored using the format expected by ZODB."""
    f = StringIO(data)
    u = Unpickler(f)
    u.persistent_load = persistent_load
    klass_info = u.load()
    if isinstance(klass_info, tuple):
        if isinstance(klass_info[0], type):
            # Unclear:  what is the second part of klass_info?
            klass, xxx = klass_info
            assert not xxx
        else:
            if isinstance(klass_info[0], tuple):
                modname, klassname = klass_info[0]
            else:
                modname, klassname = klass_info
            if modname == "__main__":
                ns = globals()
            else:
                mod = import_helper(modname)
                ns = mod.__dict__
            try:
                klass = ns[klassname]
            except KeyError:
                print >> sys.stderr, "can't find %s in %r" % (klassname, ns)
        inst = klass()
    else:
        raise ValueError("expected class info: %s" % repr(klass_info))
    state = u.load()
    inst.__setstate__(state)
    return inst
def cloneByPickle(obj, ignore_list=()):
    """Makes a copy of a ZODB object, loading ghosts as needed.

    Ignores specified objects along the way, replacing them with None
    in the copy.
    """
    ignore_dict = {}
    for o in ignore_list:
        ignore_dict[id(o)] = o

    def persistent_id(ob, ignore_dict=ignore_dict):
        if ignore_dict.has_key(id(ob)):
            return 'ignored'
        if getattr(ob, '_p_changed', 0) is None:
            ob._p_changed = 0
        return None

    def persistent_load(ref):
        assert ref == 'ignored'
        # Return a placeholder object that will be replaced by
        # removeNonVersionedData().
        placeholder = SimpleItem()
        placeholder.id = "ignored_subobject"
        return placeholder

    stream = StringIO()
    p = Pickler(stream, 1)
    p.persistent_id = persistent_id
    p.dump(obj)
    stream.seek(0)
    u = Unpickler(stream)
    u.persistent_load = persistent_load
    return u.load()
Beispiel #22
0
 def loads(self, s):
     up = Unpickler(BytesIO(s))
     up.persistent_load = self._get_object
     try:
         return up.load()
     except KeyError, e:
         raise UnpicklingError, "Could not find Node class for %s" % e
Beispiel #23
0
    def deserialize(self, event, state):
        assert IFullDeserializationEvent.isImplementedBy(event)
        assert isinstance(event.obj, Persistent)

        # Set up to resolve cyclic references to the object.
        event.deserialized('self', event.obj)

        state = state.strip()
        if state:
            if state.startswith('#'):
                # Text-encoded pickles start with a pound sign.
                # (A pound sign is not a valid pickle opcode.)
                data = decode_from_text(state)
            else:
                data = state
            infile = StringIO(data)
            u = Unpickler(infile)
            u.persistent_load = event.resolve_internal
            s = u.load()
            if not hasattr(s, 'items'):
                # Turn the list back into a dictionary
                s_list = s
                s = {}
                for key, value in s_list:
                    s[key] = value
            event.obj.__dict__.update(s)
            try:
                unmanaged = u.load()
            except EOFError:
                # old pickle with no list of unmanaged objects
                pass
            else:
                event.upos.extend(unmanaged)
Beispiel #24
0
def _unpickle(pickled):
    """ Unpickles a string and catch all types of errors it can throw,
    to raise only NotReadableJobError in case of error.

    OpenERP stores the text fields as 'utf-8', so we specify the encoding.

    `loads()` may raises many types of exceptions (AttributeError,
    IndexError, TypeError, KeyError, ...). They are all catched and
    raised as `NotReadableJobError`).

    Pickle could be exploited by an attacker who would write a value in a job
    that would run arbitrary code when unpickled. This is why we set a custom
    ``find_global`` method on the ``Unpickler``, only jobs and a whitelist of
    classes/functions are allowed to be unpickled (plus the builtins types).
    """
    def restricted_find_global(mod_name, fn_name):
        __import__(mod_name)
        mod = sys.modules[mod_name]
        fn = getattr(mod, fn_name)
        if not (fn in JOB_REGISTRY or fn in _UNPICKLE_WHITELIST):
            raise UnpicklingError(
                '{}.{} is not allowed in jobs'.format(mod_name, fn_name)
            )
        return fn

    unpickler = Unpickler(StringIO(pickled))
    unpickler.find_global = restricted_find_global
    try:
        unpickled = unpickler.load()
    except (StandardError, UnpicklingError):
        raise NotReadableJobError('Could not unpickle.', pickled)
    return unpickled
Beispiel #25
0
    def __getitem__(self, oid, tt=type(())):
        obj = self._cache.get(oid, None)
        if obj is not None:
            return obj

        __traceback_info__ = (oid)
        self.before_load()
        p, serial = self._storage.load(oid, self._version)
        __traceback_info__ = (oid, p)
        file=StringIO(p)
        unpickler=Unpickler(file)
        # unpickler.persistent_load=self._persistent_load

        try:
            classification = unpickler.load()
        except:
            raise ("Could not load oid %s. Pickled data in traceback info may "
                   "contain clues." % (oid))
        osio = self._get_osio()
        obj = osio.new_instance(oid, classification)
        assert obj is not None

        obj._p_oid=oid
        obj._p_jar=self
        obj._p_changed=None
        self._set_serial(obj, serial)

        self._cache[oid] = obj
        
        if oid == osio.conf.oid_gen.root_oid:
            self._root_=obj # keep a ref
        return obj
 def loads(self, s):
     up = Unpickler(BytesIO(s))
     up.persistent_load = self._get_object
     try:
         return up.load()
     except KeyError, e:
         raise UnpicklingError("Could not find Node class for %s" % e)
Beispiel #27
0
    def deserialize(self, message, task_id=None):
        """Deserialize an object

        :param message: A serialized object (string).
        :param deferred: When true load deferreds. When false
            raise an error if the message contains deferreds.
        """
        fail = []
        if task_id is None:
            def persistent_load(task_id):
                raise UnpicklingError('message contained references to '
                    'external objects: %s' % task_id)
        else:
            args = self._queue.get_arguments(task_id)
            args = {k: loads(v) for k, v in args.items()}

            def persistent_load(arg_id):
                value = args[arg_id]
                if isinstance(value, TaskFailure):
                    fail.append(value)
                return value
        data = StringIO(message)
        pickle = Unpickler(data)
        pickle.persistent_load = persistent_load
        obj = pickle.load()
        if fail and not obj.on_error_pass:
            # TODO detect errors earlier, fail earlier, cancel enqueued tasks
            self.set_result(obj, fail[0])
            obj = None
        return obj
 def getNewState(self, file):
     # Would like to do load(file) here... but it doesn't work with universal line endings, see Python bug 1724366
     from cStringIO import StringIO
     unpickler = Unpickler(StringIO(file.read()))
     # Magic to keep us backward compatible in the face of packages changing...
     unpickler.find_global = self.findGlobal
     return unpickler.load()
Beispiel #29
0
 def test_config_and_collector_pickling(self, testdir):
     from cPickle import Pickler, Unpickler
     tmpdir = testdir.tmpdir
     dir1 = tmpdir.ensure("somedir", dir=1)
     config = testdir.parseconfig()
     col = config.getfsnode(config.topdir)
     col1 = col.join(dir1.basename)
     assert col1.parent is col 
     io = py.std.cStringIO.StringIO()
     pickler = Pickler(io)
     pickler.dump(col)
     pickler.dump(col1)
     pickler.dump(col)
     io.seek(0) 
     unpickler = Unpickler(io)
     topdir = tmpdir.ensure("newtopdir", dir=1)
     topdir.ensure("somedir", dir=1)
     old = topdir.chdir()
     try:
         newcol = unpickler.load()
         newcol2 = unpickler.load()
         newcol3 = unpickler.load()
         assert newcol2.config is newcol.config
         assert newcol2.parent == newcol 
         assert newcol2.config.topdir.realpath() == topdir.realpath()
         assert newcol.fspath.realpath() == topdir.realpath()
         assert newcol2.fspath.basename == dir1.basename
         assert newcol2.fspath.relto(newcol2.config.topdir)
     finally:
         old.chdir() 
Beispiel #30
0
 def validateSource(self):
     skip_to_pos = 0
     if os.path.exists(self.pickle):
         upfh = open(self.pickle, "rb")
         unpickler = Unpickler(upfh)
         old_opt,old_pos = unpickler.load()
         if self.opt == old_opt:
             skip_to_pos = old_pos
             for i in range(0,skip_to_pos,1):
                 sys.stdout.write(".")
     pos = -1
     files = self.files['humans'].keys()
     files.sort()
     cp = ConfigParser()
     cp.read(os.path.join(path("config"), "test.cnf"))
     validator_path = cp.get("validation", "validator")
     csl_schema_path = cp.get("validation", "schema")
     cslm_schema_path = cp.get("validation", "schema-m")
     for filename in files:
         pos += 1
         if pos < skip_to_pos: continue
         p = self.files['humans'][filename]
         test = CslTest(opt,p,filename,pos=pos)
         test.parse()
         test.validate(validator_path, csl_schema_path, cslm_schema_path)
     if os.path.exists( self.pickle ):
         os.unlink(self.pickle)
Beispiel #31
0
 def validateSource(self):
     skip_to_pos = 0
     if os.path.exists(self.pickle):
         upfh = open(self.pickle, "rb")
         unpickler = Unpickler(upfh)
         old_opt, old_pos = unpickler.load()
         if self.opt == old_opt:
             skip_to_pos = old_pos
             for i in range(0, skip_to_pos, 1):
                 sys.stdout.write(".")
     pos = -1
     files = self.files['humans'].keys()
     files.sort()
     cp = ConfigParser()
     cp.read(os.path.join(path("config"), "test.cnf"))
     validator_path = cp.get("validation", "validator")
     csl_schema_path = cp.get("validation", "schema")
     cslm_schema_path = cp.get("validation", "schema-m")
     for filename in files:
         pos += 1
         if pos < skip_to_pos: continue
         p = self.files['humans'][filename]
         test = CslTest(opt, p, filename, pos=pos)
         test.parse()
         test.validate(validator_path, csl_schema_path, cslm_schema_path)
     if os.path.exists(self.pickle):
         os.unlink(self.pickle)
def zodb_unpickle(data):
    """Unpickle an object stored using the format expected by ZODB."""
    f = StringIO(data)
    u = Unpickler(f)
    klass_info = u.load()
    if isinstance(klass_info, types.TupleType):
        if isinstance(klass_info[0], types.TupleType):
            modname, klassname = klass_info[0]
            args = klass_info[1]
        else:
            modname, klassname = klass_info
            args = None
        if modname == "__main__":
            ns = globals()
        else:
            mod = import_helper(modname)
            ns = mod.__dict__
        try:
            klass = ns[klassname]
        except KeyError:
            sys.stderr.write("can't find %s in %s" % (klassname,
                                                      repr(ns)))
        inst = klass()
    else:
        raise ValueError, "expected class info: %s" % repr(klass_info)
    state = u.load()
    inst.__setstate__(state)
    return inst
 def __init__(self, inp_filename):
   self.filename = inp_filename
   with open(self.filename, 'rb') as inpfile:
     reader = Unpickler(inpfile)
     self.numgroups = reader.load()
     self.tot_msgs = reader.load()
     self.num_msgs = reader.load()
Beispiel #34
0
    def deserialize(self, message, task_id=None):
        """Deserialize an object

        :param message: A serialized object (string).
        :param deferred: When true load deferreds. When false
            raise an error if the message contains deferreds.
        """
        fail = []
        if task_id is None:

            def persistent_load(task_id):
                raise UnpicklingError('message contained references to '
                                      'external objects: %s' % task_id)
        else:
            args = self._queue.get_arguments(task_id)
            args = {k: loads(v) for k, v in args.items()}

            def persistent_load(arg_id):
                value = args[arg_id]
                if isinstance(value, TaskFailure):
                    fail.append(value)
                return value

        data = StringIO(message)
        pickle = Unpickler(data)
        pickle.persistent_load = persistent_load
        obj = pickle.load()
        if fail and not obj.on_error_pass:
            # TODO detect errors earlier, fail earlier, cancel enqueued tasks
            self.set_result(obj, fail[0])
            obj = None
        return obj
Beispiel #35
0
    def __getitem__(self, oid, tt=type(())):
        obj = self._cache.get(oid, None)
        if obj is not None:
            return obj

        __traceback_info__ = (oid)
        self.before_load()
        p, serial = self._storage.load(oid, self._version)
        __traceback_info__ = (oid, p)
        file = StringIO(p)
        unpickler = Unpickler(file)
        # unpickler.persistent_load=self._persistent_load

        try:
            classification = unpickler.load()
        except:
            raise ("Could not load oid %s. Pickled data in traceback info may "
                   "contain clues." % (oid))
        osio = self._get_osio()
        obj = osio.new_instance(oid, classification)
        assert obj is not None

        obj._p_oid = oid
        obj._p_jar = self
        obj._p_changed = None
        self._set_serial(obj, serial)

        self._cache[oid] = obj

        if oid == osio.conf.oid_gen.root_oid:
            self._root_ = obj  # keep a ref
        return obj
    def got_data(self, port_agent_packet):
        """
        Called by the instrument connection when data is available.

        Also add data to the chunker and when received call got_chunk
        to publish results.
        """

        data_length = port_agent_packet.get_data_length()
        data = port_agent_packet.get_data()
        timestamp = port_agent_packet.get_timestamp()

        log.debug("Got Data: %s" % data)
        log.debug("Add Port Agent Timestamp: %s" % timestamp)

        unpickler = Unpickler(StringIO(data))
        # Disable class unpickling, for security; record should be all
        # built-in types. Note this only works with cPickle.
        unpickler.find_global = None

        # pkt is an antelope.Pkt.Packet object converted to a dict. Refer to
        # the documentation for the Antelope Python bindings for compelete
        # details.

        pkt = unpickler.load()

        for particle in self._particle_factory(pkt, timestamp):
            self._publish_particle(particle)
Beispiel #37
0
 def load_configuration(cls, file_name):
     """ Unpickle lattice configuration from file"""
     with open(file_name, 'r') as f:
         upkl = Unpickler(f)
         lattice = upkl.load()
     if lattice is None:
         print("failed to unpickle or something else happened")
     return lattice
def state(self, oid, serial, prfactory, p=''):
    p = p or self.loadSerial(oid, serial)
    file = StringIO(p)
    unpickler = Unpickler(file)
    unpickler.persistent_load = prfactory.persistent_load
    class_tuple = unpickler.load()
    state = unpickler.load()
    return state
Beispiel #39
0
 def oldstate(self, object, serial):
     oid=object._p_oid
     p = self._storage.loadSerial(oid, serial)
     file=StringIO(p)
     unpickler=Unpickler(file)
     unpickler.persistent_load=self._persistent_load
     unpickler.load()
     return  unpickler.load()
Beispiel #40
0
def pickle_load(file_name):
    f = open(file_name, "rb")
    obj = None
    if f != None:
        u = Unpickler(f)
        obj = u.load()
        f.close()
    return obj
Beispiel #41
0
def deepcopy(obj):
    """Makes a deep copy of the object using the pickle mechanism.
    """
    stream = StringIO()
    p = Pickler(stream, 1)
    p.dump(aq_base(obj))
    stream.seek(0)
    u = Unpickler(stream)
    return u.load()
def deepcopy(obj):
    """Makes a deep copy of the object using the pickle mechanism.
    """
    stream = StringIO()
    p = Pickler(stream, 1)
    p.dump(aq_base(obj))
    stream.seek(0)
    u = Unpickler(stream)
    return u.load()
Beispiel #43
0
    def testPickleUnpickle(self):
        s = StringIO()
        p = Pickler(s)
        p.dump(Allow)
        s.seek(0)
        u = Unpickler(s)
        newAllow = u.load()

        self.failUnless(newAllow is Allow)
Beispiel #44
0
def get_action_param_file(pfilename):
    '''
    Argument is name of file to unpickle.  Return value is dictionary value.
    '''
    from cPickle import Unpickler
    fileh = open(pfilename, 'rb')
    pickle = Unpickler(fileh)
    list_of_dict_files = pickle.load()
    fileh.close()
    return list_of_dict_files
Beispiel #45
0
def get_action_param_var(pfilename):
    '''
    Argument is name of file to unpickle.  Return value is dictionary value.
    '''
    from cPickle import Unpickler
    with open(pfilename,'rb') as fileh:
        pickle = Unpickler(fileh)
        list_of_dict_files = pickle.load()
    return list_of_dict_files
    '''
Beispiel #46
0
def server_decode(msg):
    """Decodes msg and returns its parts"""
    unpickler = Unpickler(StringIO(msg))
    unpickler.find_global = server_find_global

    try:
        return unpickler.load()  # msgid, flags, name, args
    except:
        log("can't decode message: %s" % short_repr(msg), level=logging.ERROR)
        raise
Beispiel #47
0
class IndexFile(object):
    '''Open an index file for reading.

    filename - the file containing the index.

    Use the get and get_all method to return objects in the index.
    '''
    def __init__(self, filename):
        if not os.path.exists(filename):
            raise IndexFileMissingError(filename)
        self.filename = filename
        self.handle = open(self.filename)
        self.unpickler = Unpickler(self.handle)
        magic = self.handle.read(8)
        expected_magic = 'pdko\x00\x00\x00\x01'
        if magic != expected_magic:
            message = 'Magic bytes incorrect. Is %s really a pdko file?' \
                      % self.filename
            raise IndexFormatError, message
        table_offset = read_offset(self.handle)
        self.handle.seek(table_offset)
        self.key_dict = self.unpickler.load()

    def iter_addresses(self, key):
        '''Get a list of pickle addresses for the given key.'''
        try:
            list_offset = self.key_dict[key]
            self.handle.seek(list_offset)
            address_list = self.unpickler.load()
            for addresses in address_list:
                yield addresses
        except KeyError:
            return

    def get(self, key, column):
        '''The columnth object for all object groups under they key.'''
        for addresses in self.iter_addresses(key):
            offset = addresses[column]
            self.handle.seek(offset)
            yield self.unpickler.load()

    def get_all(self, key):
        '''Get the full object group count for the key.'''
        for addresses in self.iter_addresses(key):
            objects = []
            for offset in addresses:
                self.handle.seek(offset)
                objects.append(self.unpickler.load())
            yield tuple(objects)

    def count(self, key):
        '''Get the object group count for the given key.'''
        return len(list(self.iter_addresses(key)))
def readAST(filename):
    with open(filename, 'rb') as f:
        pk = Unpickler(f)
        opcode = pk.load()
        while True:
            try:
                name = pk.load()
                ast = pk.load()
            except EOFError:
                break

        return opcode, name, ast
Beispiel #49
0
 def getCounter(self, key):
     # initialise counter
     try:
         f = open(self.counter_file, 'r+')
         self._counters = Unpickler(f).load()
     except:
         f = open(self.counter_file, 'w+')
         self._counters = {}
     f.close()
     if not self._counters.has_key(key):
         self._counters[key] = 0
     return self._counters[key]
Beispiel #50
0
def retrieve(filename):
    ''' Retrieve a pickled object (e.g. state list or networkx graph) from file
    '''
    filename = os.path.normcase(filename)
    try:
        f = open(filename, 'rb')
        u = Unpickler(f)
        stuff = u.load()
        f.close()
        return stuff
    except IOError:
        raise LogOpeningError("No file found for %s" % filename, filename)
Beispiel #51
0
def unpickle(filename):
    f, obj = None, None
    try:
        f = open(filename, "rb")
        p = Unpickler(f)
        x = p.load()
        f.close()
        f = None
        obj = x
    finally:
        if f: f.close()
    return obj
Beispiel #52
0
def _cpickle_loads(pkl_str):
    """A replacement for cPickle.loads that overrides the find_global
    attribute (Or find_class in Python3)
    """
    pk = Unpickler(BytesIO(pkl_str))
    if hasattr(pk, 'find_class'):
        # This is the method name in Python3
        pk.find_class = _find_global
    else:
        # hasattr doe not really work on the cPickle objects in Python2, so we
        # just assume we're in python2 if hasattr returned false
        pk.find_global = _find_global
    return pk.load()
Beispiel #53
0
def readAST(filename):
    with open(filename, 'rb') as f:
        pk = Unpickler(f)
        opcode = pk.load()
        asts = {}
        while True:
            try:
                name = pk.load()
                ast = pk.load()
                asts[(name, opcode)] = ast
            except EOFError:
                break
        return asts
Beispiel #54
0
 def unpickle_values(self):
     pi = Unpickler( open( self.idata ) )
     self.centerpos = pi.load()
     self.maxhalfwidth = pi.load()
     self.leftmost = pi.load()
     self.rightmost = pi.load()
     self.conv_left = pi.load()
     self.conv_right = pi.load()
     self.middleypos = pi.load()
     self.radius = pi.load()
     self.miny = pi.load()
     self.maxy = pi.load()
     self.extend = pi.load()
Beispiel #55
0
def readAST(filename):
    with open(filename, 'rb') as f:
        pk = Unpickler(f)
        opcode = pk.load()
        print 'opcode: %02x' % opcode
        while True:
            try:
                name = pk.load()
                ast = pk.load()
                print '%s\n%s\n' % (name, str(ast))
            except EOFError:
                break

        return opcode, ast
Beispiel #56
0
 def load_game(self, path):
     """Restores a saved game from the specified path."""
     f = open(path, "rb")
     check_file_magic(f, self.app_magic, "saved game")
     check_file_magic(f, self.save_file_magic, "saved game")
     p = Unpickler(f)
     file_version = p.load()
     app_version = p.load()
     check_file_version(file_version, app_version, self.save_min_version,
                        self.save_file_version, self.save_version_ranges)
     data = p.load()
     f.close()
     self.restore_save_data(data)
     self.set_save_path(path)
     self.unsaved_progress = False
Beispiel #57
0
 def test_collector_implicit_config_pickling(self, testdir):
     from cPickle import Pickler, Unpickler
     tmpdir = testdir.tmpdir
     testdir.chdir()
     testdir.makepyfile(hello="def test_x(): pass")
     config = testdir.parseconfig(tmpdir)
     col = config.getfsnode(config.topdir)
     io = py.std.cStringIO.StringIO()
     pickler = Pickler(io)
     pickler.dump(col)
     io.seek(0)
     unpickler = Unpickler(io)
     col2 = unpickler.load()
     assert col2.name == col.name
     assert col2.listnames() == col.listnames()