def write_data_to_file(data, filename, quiet=False):
"""
Writes the data to the given filename.
If the data did not change, the file is not touched.
"""
if not isinstance(data, bytes):
msg = 'Expected "data" to be bytes, not %s.' % type(data).__name__
raise ValueError(msg)
if len(filename) > 256:
msg = 'Invalid argument filename: too long. Did you confuse it with data?'
raise ValueError(msg)
make_sure_dir_exists(filename)
if os.path.exists(filename):
current = open(filename).read()
if current == data:
if not 'assets' in filename:
if not quiet:
logger.debug('already up to date %s' % friendly_path(filename))
return
with open(filename, 'wb') as f:
f.write(data)
if not quiet:
logger.debug('Written to: %s' % friendly_path(filename))
def __setitem__(self, key, value): # @ReservedAssignment
if trace_queries:
logger.debug('W %s' % str(key))
self.check_existence()
filename = self.filename_for_key(key)
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
os.makedirs(dirname)
protocol = pickle.HIGHEST_PROTOCOL
try:
paranoid = False
if paranoid:
safe_pickle_dump(value, filename, protocol)
else:
with open(filename, 'wb') as f:
pickle.dump(value, f, protocol)
except Exception as e:
msg = ('Cannot set key %s: cannot pickle object '
'of class %s: %s' % (key, value.__class__.__name__, e))
logger.error(msg)
logger.exception(e)
emsg = find_pickling_error(value)
logger.error(emsg)
raise SerializationError(msg + '\n' + emsg)
def __setitem__(self, key, value): # @ReservedAssignment
if trace_queries:
logger.debug('W %s' % str(key))
self.check_existence()
filename = self.filename_for_key(key)
protocol = pickle.HIGHEST_PROTOCOL
try:
# paranoid = False
# if paranoid or self.compress: # safe_pickle_dump can take care of .gz
safe_pickle_dump(value, filename, protocol)
# else:
# if False:
# f = open(filename, 'wb', buffering=-1)
# pickle.dump(value, f, protocol)
# else:
# with open(filename, 'wb', buffering=-1) as f:
# pickle.dump(value, f, protocol)
except Exception as e:
msg = ('Cannot set key %s: cannot pickle object '
'of class %s: %s' % (key, value.__class__.__name__, e))
logger.error(msg)
logger.exception(e)
emsg = find_pickling_error(value)
logger.error(emsg)
raise SerializationError(msg + '\n' + emsg)
def __contains__(self, key):
if trace_queries:
logger.debug('? %s' % str(key))
filename = self.filename_for_key(key)
ex = os.path.exists(filename)
# logger.debug('? %s %s %s' % (str(key), filename, ex))
return ex
def __contains__(self, key):
if trace_queries:
logger.debug('? %s' % str(key))
filename = self.filename_for_key(key)
ex = os.path.exists(filename)
# logger.debug('? %s %s %s' % (str(key), filename, ex))
return ex
def __getitem__(self, key):
if trace_queries:
logger.debug('R %s' % str(key))
self.check_existence()
filename = self.filename_for_key(key)
if not os.path.exists(filename):
raise CompmakeException('Could not find key %r.' % key)
try:
return safe_pickle_load(filename)
except Exception as e:
msg = "Could not unpickle file %r." % (filename)
logger.error(msg)
logger.exception(e)
msg += "\n" + traceback.format_exc(e)
raise CompmakeException(msg)
def __setitem__(self, key, value): # @ReservedAssignment
if trace_queries:
logger.debug('W %s' % str(key))
self.check_existence()
filename = self.filename_for_key(key)
try:
safe_pickle_dump(value, filename)
assert os.path.exists(filename)
except BaseException as e:
msg = ('Cannot set key %s: cannot pickle object '
'of class %s: %s' % (key, value.__class__.__name__, e))
logger.error(msg)
logger.exception(e)
emsg = find_pickling_error(value)
logger.error(emsg)
raise SerializationError(msg + '\n' + emsg)
def __setitem__(self, key, value): # @ReservedAssignment
if trace_queries:
logger.debug('W %s' % str(key))
self.check_existence()
filename = self.filename_for_key(key)
try:
safe_pickle_dump(value, filename)
assert os.path.exists(filename)
except KeyboardInterrupt:
raise
except BaseException as e:
msg = ('Cannot set key %s: cannot pickle object '
'of class %s: %s' % (key, value.__class__.__name__, e))
logger.error(msg)
logger.exception(e)
emsg = find_pickling_error(value)
logger.error(emsg)
raise SerializationError(msg + '\n' + emsg)
def __getitem__(self, key):
if trace_queries:
logger.debug('R %s' % str(key))
self.check_existence()
filename = self.filename_for_key(key)
if not os.path.exists(filename):
msg = 'Could not find key %r.' % key
msg += '\n file: %s' % filename
raise CompmakeBug(msg)
try:
return safe_pickle_load(filename)
except Exception as e:
msg = ("Could not unpickle data for key %r. \n file: %s" %
(key, filename))
logger.error(msg)
logger.exception(e)
msg += "\n" + traceback.format_exc()
raise CompmakeBug(msg)
def __getitem__(self, key):
if trace_queries:
logger.debug('R %s' % str(key))
self.check_existence()
filename = self.filename_for_key(key)
if not os.path.exists(filename):
msg = 'Could not find key %r.' % key
msg += '\n file: %s' % filename
raise CompmakeBug(msg)
try:
return safe_pickle_load(filename)
except Exception as e:
msg = ("Could not unpickle data for key %r. \n file: %s" %
(key, filename))
logger.error(msg)
logger.exception(e)
msg += "\n" + traceback.format_exc()
raise CompmakeBug(msg)