Esempio n. 1
0
def write_data_to_file(data, filename, quiet=False):
    """ 
        Writes the data to the given filename. 
        If the data did not change, the file is not touched.
    
    """
    if not isinstance(data, bytes):
        msg = 'Expected "data" to be bytes, not %s.' % type(data).__name__
        raise ValueError(msg)
    if len(filename) > 256:
        msg = 'Invalid argument filename: too long. Did you confuse it with data?'
        raise ValueError(msg)
    
    
    make_sure_dir_exists(filename)
    
    if os.path.exists(filename):
        current = open(filename).read()
        if current == data:
            if not 'assets' in filename:
                if not quiet:
                    logger.debug('already up to date %s' % friendly_path(filename))
            return
         
    with open(filename, 'wb') as f:
        f.write(data)
        
    if not quiet:
        logger.debug('Written to: %s' % friendly_path(filename))
Esempio n. 2
0
    def __setitem__(self, key, value):  # @ReservedAssignment
        if trace_queries:
            logger.debug('W %s' % str(key))

        self.check_existence()

        filename = self.filename_for_key(key)
        dirname = os.path.dirname(filename)
        if not os.path.exists(dirname):
            os.makedirs(dirname)
            
        protocol = pickle.HIGHEST_PROTOCOL
        try:
            paranoid = False
            if paranoid:        
                safe_pickle_dump(value, filename, protocol)
            else:
                with open(filename, 'wb') as f:
                    pickle.dump(value, f, protocol)

        except Exception as e:
            msg = ('Cannot set key %s: cannot pickle object '
                    'of class %s: %s' % (key, value.__class__.__name__, e))
            logger.error(msg)
            logger.exception(e)
            emsg = find_pickling_error(value)
            logger.error(emsg)
            raise SerializationError(msg + '\n' + emsg)
Esempio n. 3
0
    def __setitem__(self, key, value):  # @ReservedAssignment
        if trace_queries:
            logger.debug('W %s' % str(key))

        self.check_existence()

        filename = self.filename_for_key(key)
        protocol = pickle.HIGHEST_PROTOCOL
        try:
#             paranoid = False
#             if paranoid or self.compress:  # safe_pickle_dump can take care of .gz
            safe_pickle_dump(value, filename, protocol)
#             else:
#                 if False:
#                     f = open(filename, 'wb', buffering=-1)
#                     pickle.dump(value, f, protocol)
#                 else:
#                     with open(filename, 'wb', buffering=-1) as f:
#                         pickle.dump(value, f, protocol)

        except Exception as e:
            msg = ('Cannot set key %s: cannot pickle object '
                    'of class %s: %s' % (key, value.__class__.__name__, e))
            logger.error(msg)
            logger.exception(e)
            emsg = find_pickling_error(value)
            logger.error(emsg)
            raise SerializationError(msg + '\n' + emsg)
Esempio n. 4
0
    def __contains__(self, key):
        if trace_queries:
            logger.debug('? %s' % str(key))

        filename = self.filename_for_key(key)
        ex = os.path.exists(filename)

        # logger.debug('? %s %s %s' % (str(key), filename, ex))
        return ex
Esempio n. 5
0
    def __contains__(self, key):
        if trace_queries:
            logger.debug('? %s' % str(key))

        filename = self.filename_for_key(key)
        ex = os.path.exists(filename)

        # logger.debug('? %s %s %s' % (str(key), filename, ex))
        return ex
Esempio n. 6
0
 def __getitem__(self, key):
     if trace_queries:
         logger.debug('R %s' % str(key))
     
     self.check_existence()
     
     filename = self.filename_for_key(key)
     
     if not os.path.exists(filename):
         raise CompmakeException('Could not find key %r.' % key)
     
     try:
         return safe_pickle_load(filename)
     except Exception as e:
         msg = "Could not unpickle file %r." % (filename)
         logger.error(msg)
         logger.exception(e)
         msg += "\n" + traceback.format_exc(e)
         raise CompmakeException(msg)
Esempio n. 7
0
    def __setitem__(self, key, value):  # @ReservedAssignment
        if trace_queries:
            logger.debug('W %s' % str(key))

        self.check_existence()

        filename = self.filename_for_key(key)

        try:
            safe_pickle_dump(value, filename)
            assert os.path.exists(filename)

        except BaseException as e:
            msg = ('Cannot set key %s: cannot pickle object '
                   'of class %s: %s' % (key, value.__class__.__name__, e))
            logger.error(msg)
            logger.exception(e)
            emsg = find_pickling_error(value)
            logger.error(emsg)
            raise SerializationError(msg + '\n' + emsg)
Esempio n. 8
0
    def __setitem__(self, key, value):  # @ReservedAssignment
        if trace_queries:
            logger.debug('W %s' % str(key))

        self.check_existence()

        filename = self.filename_for_key(key)

        try:
            safe_pickle_dump(value, filename)
            assert os.path.exists(filename)
        except KeyboardInterrupt:
            raise
        except BaseException as e:
            msg = ('Cannot set key %s: cannot pickle object '
                   'of class %s: %s' % (key, value.__class__.__name__, e))
            logger.error(msg)
            logger.exception(e)
            emsg = find_pickling_error(value)
            logger.error(emsg)
            raise SerializationError(msg + '\n' + emsg)
Esempio n. 9
0
    def __getitem__(self, key):
        if trace_queries:
            logger.debug('R %s' % str(key))

        self.check_existence()

        filename = self.filename_for_key(key)

        if not os.path.exists(filename):
            msg = 'Could not find key %r.' % key
            msg += '\n file: %s' % filename
            raise CompmakeBug(msg)

        try:
            return safe_pickle_load(filename)
        except Exception as e:
            msg = ("Could not unpickle data for key %r. \n file: %s" %
                   (key, filename))
            logger.error(msg)
            logger.exception(e)
            msg += "\n" + traceback.format_exc()
            raise CompmakeBug(msg)
Esempio n. 10
0
    def __getitem__(self, key):
        if trace_queries:
            logger.debug('R %s' % str(key))

        self.check_existence()

        filename = self.filename_for_key(key)

        if not os.path.exists(filename):
            msg = 'Could not find key %r.' % key
            msg += '\n file: %s' % filename
            raise CompmakeBug(msg)

        try:
            return safe_pickle_load(filename)
        except Exception as e:
            msg = ("Could not unpickle data for key %r. \n file: %s" %
                   (key, filename))
            logger.error(msg)
            logger.exception(e)
            msg += "\n" + traceback.format_exc()
            raise CompmakeBug(msg)