def _value_for_db_model(self, value, field, field_kind, db_type, lookup):
"""
Converts a field => value mapping received from an
EmbeddedModelField the format chosen for the field storage.
The embedded instance fields' values are also converted /
deconverted using value_for/from_db, so any back-end
conversions will be applied.
Returns (field.column, value) pairs, possibly augmented with
model info (to be able to deconvert the embedded instance for
untyped fields) encoded according to the db_type chosen.
If "dict" db_type is given a Python dict is returned.
If "list db_type is chosen a list with columns and values
interleaved will be returned. Note that just a single level of
the list is flattened, so it still may be nested -- when the
embedded instance holds other embedded models or collections).
Using "bytes" or "string" pickles the mapping using pickle
protocol 0 or 2 respectively.
If an unknown db_type is used a generator yielding (column,
value) pairs with values converted will be returned.
TODO: How should EmbeddedModelField lookups work?
"""
if lookup:
# raise NotImplementedError("Needs specification.")
return value
# Convert using proper instance field's info, change keys from
# fields to columns.
# TODO/XXX: Arguments order due to Python 2.5 compatibility.
value = ((subfield.column,
self._value_for_db(subvalue,
lookup=lookup,
*self._convert_as(subfield, lookup)))
for subfield, subvalue in six.iteritems(value))
# Cast to a dict, interleave columns with values on a list,
# serialize, or return a generator.
if db_type == 'dict':
value = dict(value)
elif db_type == 'list':
value = list(item for pair in value for item in pair)
elif db_type == 'bytes':
value = pickle.dumps(dict(value), protocol=2)
elif db_type == 'string':
value = pickle.dumps(dict(value))
return value
def _value_for_db_model(self, value, field, field_kind, db_type, lookup):
"""
Converts a field => value mapping received from an
EmbeddedModelField the format chosen for the field storage.
The embedded instance fields' values are also converted /
deconverted using value_for/from_db, so any back-end
conversions will be applied.
Returns (field.column, value) pairs, possibly augmented with
model info (to be able to deconvert the embedded instance for
untyped fields) encoded according to the db_type chosen.
If "dict" db_type is given a Python dict is returned.
If "list db_type is chosen a list with columns and values
interleaved will be returned. Note that just a single level of
the list is flattened, so it still may be nested -- when the
embedded instance holds other embedded models or collections).
Using "bytes" or "string" pickles the mapping using pickle
protocol 0 or 2 respectively.
If an unknown db_type is used a generator yielding (column,
value) pairs with values converted will be returned.
TODO: How should EmbeddedModelField lookups work?
"""
if lookup:
# raise NotImplementedError("Needs specification.")
return value
# Convert using proper instance field's info, change keys from
# fields to columns.
# TODO/XXX: Arguments order due to Python 2.5 compatibility.
value = (
(subfield.column, self._value_for_db(
subvalue, lookup=lookup, *self._convert_as(subfield, lookup)))
for subfield, subvalue in value.iteritems())
# Cast to a dict, interleave columns with values on a list,
# serialize, or return a generator.
if db_type == 'dict':
value = dict(value)
elif db_type == 'list':
value = list(item for pair in value for item in pair)
elif db_type == 'bytes':
value = pickle.dumps(dict(value), protocol=2)
elif db_type == 'string':
value = pickle.dumps(dict(value))
return value
def _base_set(self, mode, key, value, timeout=DEFAULT_TIMEOUT):
timeout = self.get_backend_timeout(timeout)
db = router.db_for_write(self.cache_model_class)
table = connections[db].ops.quote_name(self._table)
with connections[db].cursor() as cursor:
cursor.execute("SELECT COUNT(*) FROM %s" % table)
num = cursor.fetchone()[0]
now = timezone.now()
now = now.replace(microsecond=0)
if timeout is None:
exp = datetime.max
elif settings.USE_TZ:
exp = datetime.utcfromtimestamp(timeout)
else:
exp = datetime.fromtimestamp(timeout)
exp = exp.replace(microsecond=0)
if num > self._max_entries:
self._cull(db, cursor, now)
pickled = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
b64encoded = base64.b64encode(pickled)
# The DB column is expecting a string, so make sure the value is a
# string, not bytes. Refs #19274.
if six.PY3:
b64encoded = b64encoded.decode('latin1')
try:
# Note: typecasting for datetimes is needed by some 3rd party
# database backends. All core backends work without typecasting,
# so be careful about changes here - test suite will NOT pick
# regressions.
with transaction.atomic(using=db):
cursor.execute(
"SELECT cache_key, expires FROM %s "
"WHERE cache_key = %%s" % table, [key])
result = cursor.fetchone()
if result:
current_expires = result[1]
if (connections[db].features.needs_datetime_string_cast
and not isinstance(current_expires, datetime)):
current_expires = typecast_timestamp(
str(current_expires))
exp = connections[db].ops.value_to_db_datetime(exp)
if result and (mode == 'set' or
(mode == 'add' and current_expires < now)):
cursor.execute(
"UPDATE %s SET value = %%s, expires = %%s "
"WHERE cache_key = %%s" % table,
[b64encoded, exp, key])
elif not result:
cursor.execute(
"INSERT INTO %s (cache_key, value, expires) "
"VALUES (%%s, %%s, %%s)" % table,
[key, b64encoded, exp])
else:
return False
except DatabaseError:
# To be threadsafe, updates/inserts are allowed to fail silently
return False
else:
return True
def test_pickle(self):
"Testing pickle support."
g1 = OGRGeometry("LINESTRING(1 1 1,2 2 2,3 3 3)", "WGS84")
g2 = pickle.loads(pickle.dumps(g1))
self.assertEqual(g1, g2)
self.assertEqual(4326, g2.srs.srid)
self.assertEqual(g1.srs.wkt, g2.srs.wkt)
def test17_pickle(self):
"Testing pickle support."
g1 = OGRGeometry('LINESTRING(1 1 1,2 2 2,3 3 3)', 'WGS84')
g2 = pickle.loads(pickle.dumps(g1))
self.assertEqual(g1, g2)
self.assertEqual(4326, g2.srs.srid)
self.assertEqual(g1.srs.wkt, g2.srs.wkt)
def test_pickle(self):
"Testing pickling and unpickling support."
# Using both pickle and cPickle -- just 'cause.
from django.utils.six.moves import cPickle
import pickle
# Creating a list of test geometries for pickling,
# and setting the SRID on some of them.
def get_geoms(lst, srid=None):
return [GEOSGeometry(tg.wkt, srid) for tg in lst]
tgeoms = get_geoms(self.geometries.points)
tgeoms.extend(get_geoms(self.geometries.multilinestrings, 4326))
tgeoms.extend(get_geoms(self.geometries.polygons, 3084))
tgeoms.extend(get_geoms(self.geometries.multipolygons, 3857))
# The SRID won't be exported in GEOS 3.0 release candidates.
no_srid = self.null_srid == -1
for geom in tgeoms:
s1, s2 = cPickle.dumps(geom), pickle.dumps(geom)
g1, g2 = cPickle.loads(s1), pickle.loads(s2)
for tmpg in (g1, g2):
self.assertEqual(geom, tmpg)
if not no_srid:
self.assertEqual(geom.srid, tmpg.srid)
def test_pickle(self):
"Testing pickling and unpickling support."
# Using both pickle and cPickle -- just 'cause.
from django.utils.six.moves import cPickle
import pickle
# Creating a list of test geometries for pickling,
# and setting the SRID on some of them.
def get_geoms(lst, srid=None):
return [GEOSGeometry(tg.wkt, srid) for tg in lst]
tgeoms = get_geoms(self.geometries.points)
tgeoms.extend(get_geoms(self.geometries.multilinestrings, 4326))
tgeoms.extend(get_geoms(self.geometries.polygons, 3084))
tgeoms.extend(get_geoms(self.geometries.multipolygons, 900913))
# The SRID won't be exported in GEOS 3.0 release candidates.
no_srid = self.null_srid == -1
for geom in tgeoms:
s1, s2 = cPickle.dumps(geom), pickle.dumps(geom)
g1, g2 = cPickle.loads(s1), pickle.loads(s2)
for tmpg in (g1, g2):
self.assertEqual(geom, tmpg)
if not no_srid:
self.assertEqual(geom.srid, tmpg.srid)
def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
self._createdir() # Cache dir can be deleted at any time.
fname = self._key_to_file(key, version)
self._cull() # make some room if necessary
fd, tmp_path = tempfile.mkstemp(dir=self._dir)
renamed = False
try:
with io.open(fd, 'wb') as f:
expiry = self.get_backend_timeout(timeout)
f.write(pickle.dumps(expiry, -1))
f.write(zlib.compress(pickle.dumps(value), -1))
file_move_safe(tmp_path, fname, allow_overwrite=True)
renamed = True
finally:
if not renamed:
os.remove(tmp_path)
def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
self._createdir() # Cache dir can be deleted at any time.
fname = self._key_to_file(key, version)
self._cull() # make some room if necessary
fd, tmp_path = tempfile.mkstemp(dir=self._dir)
renamed = False
try:
with io.open(fd, 'wb') as f:
expiry = self.get_backend_timeout(timeout)
f.write(pickle.dumps(expiry, pickle.HIGHEST_PROTOCOL))
f.write(zlib.compress(pickle.dumps(value, pickle.HIGHEST_PROTOCOL)))
file_move_safe(tmp_path, fname, allow_overwrite=True)
renamed = True
finally:
if not renamed:
os.remove(tmp_path)
def _base_set(self, mode, key, value, timeout=DEFAULT_TIMEOUT):
timeout = self.get_backend_timeout(timeout)
db = router.db_for_write(self.cache_model_class)
connection = connections[db]
table = connection.ops.quote_name(self._table)
with connection.cursor() as cursor:
cursor.execute("SELECT COUNT(*) FROM %s" % table)
num = cursor.fetchone()[0]
now = timezone.now()
now = now.replace(microsecond=0)
if timeout is None:
exp = datetime.max
elif settings.USE_TZ:
exp = datetime.utcfromtimestamp(timeout)
else:
exp = datetime.fromtimestamp(timeout)
exp = exp.replace(microsecond=0)
if num > self._max_entries:
self._cull(db, cursor, now)
pickled = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
b64encoded = base64.b64encode(pickled)
# The DB column is expecting a string, so make sure the value is a
# string, not bytes. Refs #19274.
if six.PY3:
b64encoded = b64encoded.decode("latin1")
try:
# Note: typecasting for datetimes is needed by some 3rd party
# database backends. All core backends work without typecasting,
# so be careful about changes here - test suite will NOT pick
# regressions.
with transaction.atomic(using=db):
cursor.execute("SELECT cache_key, expires FROM %s " "WHERE cache_key = %%s" % table, [key])
result = cursor.fetchone()
if result:
current_expires = result[1]
expression = models.Expression(output_field=models.DateTimeField())
for converter in connection.ops.get_db_converters(expression) + expression.get_db_converters(
connection
):
current_expires = converter(current_expires, expression, connection, {})
exp = connection.ops.adapt_datetimefield_value(exp)
if result and (mode == "set" or (mode == "add" and current_expires < now)):
cursor.execute(
"UPDATE %s SET value = %%s, expires = %%s " "WHERE cache_key = %%s" % table,
[b64encoded, exp, key],
)
else:
cursor.execute(
"INSERT INTO %s (cache_key, value, expires) " "VALUES (%%s, %%s, %%s)" % table,
[key, b64encoded, exp],
)
except DatabaseError:
# To be threadsafe, updates/inserts are allowed to fail silently
return False
else:
return True
def encode_param(args, kwargs):
"""
序列化参数，保证相同的参数返回相同的值，不同的参数返回不同的值
"""
#return repr((args, kwargs)) # repr 没法区分 Model 类型
value = pickle.dumps((args, kwargs), pickle.HIGHEST_PROTOCOL)
value = md5(value).hexdigest() # 防止返回值太长
return value
def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
key = self.make_key(key, version=version)
self.validate_key(key)
with self._lock.writer():
try:
pickled = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
self._set(key, pickled, timeout)
except pickle.PickleError:
pass
def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
key = self.make_key(key, version=version)
self.validate_key(key)
pickled = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
with self._lock.writer():
if self._has_expired(key):
self._set(key, pickled, timeout)
return True
return False
def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
key = self.make_key(key, version=version)
self.validate_key(key)
with self._lock.writer():
try:
pickled = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
self._set(key, pickled, timeout)
except pickle.PickleError:
pass
def encode(self, session_dict):
data = pickle.dumps(session_dict, pickle.HIGHEST_PROTOCOL)
flag = 0
if conf['COMPRESS_LIB'] and len(data) >= conf['COMPRESS_MIN_LENGTH']:
compressed = compress_lib.compress(data)
if len(compressed) < len(data):
flag |= FLAG_COMPRESSED
data = compressed
return chr(flag) + data
def test_storage_gone(pic1):
product = Product.objects.create(image=pic1['filename'])
assert os.path.exists(pic1['path'])
product.image = 'new.jpg'
product = pickle.loads(pickle.dumps(product))
assert hasattr(product.image, 'storage')
with transaction.atomic(get_using(product)):
product.save()
assert not os.path.exists(pic1['path'])
def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
key = self.make_key(key, version=version)
self.validate_key(key)
pickled = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
with self._lock.writer():
if self._has_expired(key):
self._set(key, pickled, timeout)
return True
return False
def test_storage_gone(pic1):
product = Product.objects.create(image=pic1['filename'])
assert os.path.exists(pic1['path'])
product.image = random_pic()
product = pickle.loads(pickle.dumps(product))
assert hasattr(product.image, 'storage')
with transaction.atomic(get_using(product)):
product.save()
assert not os.path.exists(pic1['path'])
def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
key = self.make_key(key, version=version)
self.validate_key(key)
pickled = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
with self._lock.writer():
exp = self._expire_info.get(key)
if exp is None or exp <= time.time():
self._set(key, pickled, timeout)
return True
return False
def incr(self, key, delta=1, version=None):
with self._lock.writer():
value = self.get(key, version=version, acquire_lock=False)
if value is None:
raise ValueError("Key '%s' not found" % key)
new_value = value + delta
key = self.make_key(key, version=version)
pickled = pickle.dumps(new_value, pickle.HIGHEST_PROTOCOL)
self._cache[key] = pickled
return new_value
def incr(self, key, delta=1, version=None):
with self._lock.writer():
value = self.get(key, version=version, acquire_lock=False)
if value is None:
raise ValueError("Key '%s' not found" % key)
new_value = value + delta
key = self.make_key(key, version=version)
pickled = pickle.dumps(new_value, pickle.HIGHEST_PROTOCOL)
self._cache[key] = pickled
return new_value
def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
key = self.make_key(key, version=version)
self.validate_key(key)
pickled = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
with self._lock.writer():
exp = self._expire_info.get(key)
if exp is None or exp <= time.time():
self._set(key, pickled, timeout)
return True
return False
def _base_set(self, mode, key, value, timeout=DEFAULT_TIMEOUT):
if timeout == DEFAULT_TIMEOUT:
timeout = self.default_timeout
db = router.db_for_write(self.cache_model_class)
table = connections[db].ops.quote_name(self._table)
cursor = connections[db].cursor()
cursor.execute("SELECT COUNT(*) FROM %s" % table)
num = cursor.fetchone()[0]
now = timezone.now()
now = now.replace(microsecond=0)
if timeout is None:
exp = datetime.max
elif settings.USE_TZ:
exp = datetime.utcfromtimestamp(time.time() + timeout)
else:
exp = datetime.fromtimestamp(time.time() + timeout)
exp = exp.replace(microsecond=0)
if num > self._max_entries:
self._cull(db, cursor, now)
pickled = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
b64encoded = base64.b64encode(pickled)
# The DB column is expecting a string, so make sure the value is a
# string, not bytes. Refs #19274.
if six.PY3:
b64encoded = b64encoded.decode('latin1')
try:
# Note: typecasting for datetimes is needed by some 3rd party
# database backends. All core backends work without typecasting,
# so be careful about changes here - test suite will NOT pick
# regressions.
with transaction.atomic(using=db):
cursor.execute("SELECT cache_key, expires FROM %s "
"WHERE cache_key = %%s" % table, [key])
result = cursor.fetchone()
if result:
current_expires = result[1]
if (connections[db].features.needs_datetime_string_cast and not
isinstance(current_expires, datetime)):
current_expires = typecast_timestamp(str(current_expires))
exp = connections[db].ops.value_to_db_datetime(exp)
if result and (mode == 'set' or (mode == 'add' and current_expires < now)):
cursor.execute("UPDATE %s SET value = %%s, expires = %%s "
"WHERE cache_key = %%s" % table,
[b64encoded, exp, key])
else:
cursor.execute("INSERT INTO %s (cache_key, value, expires) "
"VALUES (%%s, %%s, %%s)" % table,
[key, b64encoded, exp])
except DatabaseError:
# To be threadsafe, updates/inserts are allowed to fail silently
return False
else:
return True
def _build_test_chunk_data(self, num_chunks):
"""Build enough test data to fill up the specified number of chunks.
This takes into account the size of the pickle data, and will
get us to exactly the specified number of chunks of data in the cache.
"""
data = 'x' * (CACHE_CHUNK_SIZE * num_chunks - 3 * num_chunks)
pickled_data = pickle.dumps(data)
self.assertEqual(len(pickled_data), CACHE_CHUNK_SIZE * num_chunks)
return data, pickled_data
def test_pickle(self):
letters = 'abcde'
cache.clear()
for num, val in enumerate(letters):
cache.set(val, num)
data = pickle.dumps(cache)
other = pickle.loads(data)
for key in letters:
self.assertEqual(other.get(key), cache.get(key))
def _build_test_chunk_data(self, num_chunks):
"""Build enough test data to fill up the specified number of chunks.
This takes into account the size of the pickle data, and will
get us to exactly the specified number of chunks of data in the cache.
"""
data = 'x' * (CACHE_CHUNK_SIZE * num_chunks - 3 * num_chunks)
pickled_data = pickle.dumps(data)
self.assertEqual(len(pickled_data), CACHE_CHUNK_SIZE * num_chunks)
return data, pickled_data
def incr(self, key, delta=1, version=None):
value = self.get(key, version=version)
if value is None:
raise ValueError("Key '%s' not found" % key)
new_value = value + delta
key = self.make_key(key, version=version)
with self._lock.writer():
try:
pickled = pickle.dumps(new_value, pickle.HIGHEST_PROTOCOL)
self._cache[key] = pickled
except pickle.PickleError:
pass
return new_value
def add(self, key, value, timeout=None, version=None):
key = self.make_key(key, version=version)
self.validate_key(key)
with self._lock.writer():
exp = self._expire_info.get(key)
if exp is None or exp <= time.time():
try:
pickled = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
self._set(key, pickled, timeout)
return True
except pickle.PickleError:
pass
return False
def add(self, key, value, timeout=None, version=None):
key = self.make_key(key, version=version)
self.validate_key(key)
with self._lock.writer():
exp = self._expire_info.get(key)
if exp is None or exp <= time.time():
try:
pickled = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
self._set(key, pickled, timeout)
return True
except pickle.PickleError:
pass
return False
def incr(self, key, delta=1, version=None):
value = self.get(key, version=version)
if value is None:
raise ValueError("Key '%s' not found" % key)
new_value = value + delta
key = self.make_key(key, version=version)
with self._lock.writer():
try:
pickled = pickle.dumps(new_value, pickle.HIGHEST_PROTOCOL)
self._cache[key] = pickled
except pickle.PickleError:
pass
return new_value
def _base_set(self, mode, key, value, timeout=None):
if timeout is None:
timeout = self.default_timeout
db = router.db_for_write(self.cache_model_class)
table = connections[db].ops.quote_name(self._table)
cursor = connections[db].cursor()
cursor.execute("SELECT COUNT(*) FROM %s" % table)
num = cursor.fetchone()[0]
now = timezone.now()
now = now.replace(microsecond=0)
if settings.USE_TZ:
exp = datetime.utcfromtimestamp(time.time() + timeout)
else:
exp = datetime.fromtimestamp(time.time() + timeout)
exp = exp.replace(microsecond=0)
if num > self._max_entries:
self._cull(db, cursor, now)
pickled = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
b64encoded = base64.b64encode(pickled)
# The DB column is expecting a string, so make sure the value is a
# string, not bytes. Refs #19274.
if six.PY3:
b64encoded = b64encoded.decode('latin1')
cursor.execute(
"SELECT cache_key, expires FROM %s "
"WHERE cache_key = %%s" % table, [key])
try:
result = cursor.fetchone()
if result and (mode == 'set' or
(mode == 'add' and result[1] < now)):
cursor.execute(
"UPDATE %s SET value = %%s, expires = %%s "
"WHERE cache_key = %%s" % table, [
b64encoded,
connections[db].ops.value_to_db_datetime(exp), key
])
else:
cursor.execute(
"INSERT INTO %s (cache_key, value, expires) "
"VALUES (%%s, %%s, %%s)" % table, [
key, b64encoded,
connections[db].ops.value_to_db_datetime(exp)
])
except DatabaseError:
# To be threadsafe, updates/inserts are allowed to fail silently
transaction.rollback_unless_managed(using=db)
return False
else:
transaction.commit_unless_managed(using=db)
return True
def dump(self, obj):
"""
Return the pickled representation of the object as a bytes object.
We use ``deepcopy()`` here to avoid a problem with cPickle, where
``dumps()`` can generate different character streams for same lookup
value if they are referenced differently.
The reason this is important is because we do all of our lookups as
simple string matches, thus the character streams must be the same for
the lookups to work properly.
"""
return pickle.dumps(deepcopy(obj), self.protocol)
def encode(self, obj):
"""
Take a Python object and return it as a tuple (value, value_type), a
blob and a one-char code for what type it is
"""
if self._is_valid_mysql_bigint(obj):
return obj, 'i'
value = pickle.dumps(obj, pickle.HIGHEST_PROTOCOL)
value_type = 'p'
if (self._compress_min_length
and len(value) >= self._compress_min_length):
value = zlib.compress(value, self._compress_level)
value_type = 'z'
return value, value_type
def queue(users, label, extra_context=None, sender=None):
"""
Queue the notification in NoticeQueueBatch. This allows for large amounts
of user notifications to be deferred to a seperate process running outside
the webserver.
"""
if extra_context is None:
extra_context = {}
if isinstance(users, QuerySet):
users = [row["pk"] for row in users.values("pk")]
else:
users = [user.pk for user in users]
notices = []
for user in users:
notices.append((user, label, extra_context, sender))
NoticeQueueBatch(pickled_data=base64.b64encode(pickle.dumps(notices))).save()
def queue(users, label, extra_context=None, sender=None):
"""
Queue the notification in NoticeQueueBatch. This allows for large amounts
of user notifications to be deferred to a seperate process running outside
the webserver.
"""
if extra_context is None:
extra_context = {}
if isinstance(users, QuerySet):
users = [row["pk"] for row in users.values("pk")]
else:
users = [user.pk for user in users]
notices = []
for user in users:
notices.append((user, label, extra_context, sender))
NoticeQueueBatch(pickled_data=base64.b64encode(pickle.dumps(notices))).save()
def _cache_store_items(cache, key, items, expiration, compress_large_data):
"""Store items in the cache.
The items will be individually pickled and combined into a binary blob,
which can then optionally be compressed. The resulting data is then
cached over one or more keys, each representing a chunk about 1MB in size.
A main cache key will be set that contains information on the other keys.
"""
results = ((pickle.dumps(item), True, item) for item in items)
if compress_large_data:
results = _cache_compress_pickled_data(results)
for item in _cache_store_chunks(results, key, expiration):
yield item
def pickle_dumps(obj):
"""Return a pickled representation of an object.
This will always use Pickle protocol 0, which is the default on Python 2,
for compatibility across Python 2 and 3.
Args:
obj (object):
The object to dump.
Returns:
unicode:
The Unicode pickled representation of the object, safe for storing
in the database.
"""
return pickle.dumps(obj, protocol=0).decode('latin1')
def test_cache_memoize_large_files_uncompressed_off_by_one(self):
"""Testing cache_memoize with large files without compression and
one byte larger than an even chunk size."""
cache_key = 'abc123'
# This takes into account the size of the pickle data, and will
# get us to just barely 3 chunks of data in cache.
data = self._build_test_chunk_data(num_chunks=2)[0] + 'x'
pickled_data = pickle.dumps(data, protocol=0)
def cache_func():
return data
self.spy_on(cache_func, call_original=True)
result = cache_memoize(cache_key,
cache_func,
large_data=True,
compress_large_data=False)
self.assertEqual(result, data)
self.assertTrue(cache_func.spy.called)
cache_key_0 = make_cache_key('%s-0' % cache_key)
cache_key_1 = make_cache_key('%s-1' % cache_key)
cache_key_2 = make_cache_key('%s-2' % cache_key)
self.assertTrue(make_cache_key(cache_key) in cache)
self.assertTrue(cache_key_0 in cache)
self.assertTrue(cache_key_1 in cache)
self.assertTrue(cache_key_2 in cache)
self.assertFalse(make_cache_key('%s-3' % cache_key) in cache)
# Verify the contents of the stored data.
stored_data = b''.join(
cache.get(cache_key_0) + cache.get(cache_key_1) +
cache.get(cache_key_2))
self.assertEqual(stored_data, pickled_data)
# Try fetching the data we stored.
cache_func.spy.reset_calls()
result = cache_memoize(cache_key,
cache_func,
large_data=True,
compress_large_data=False)
self.assertEqual(result, data)
self.assertFalse(cache_func.spy.called)
def encode(self, obj):
"""
Take a Python object and return it as a tuple (value, value_type), a
blob and a one-char code for what type it is
"""
if self._is_valid_mysql_bigint(obj):
return obj, 'i'
value = pickle.dumps(obj, pickle.HIGHEST_PROTOCOL)
value_type = 'p'
if (
self._compress_min_length and
len(value) >= self._compress_min_length
):
value = zlib.compress(value, self._compress_level)
value_type = 'z'
return value, value_type
def _base_set(self, mode, key, value, timeout=DEFAULT_TIMEOUT, use_tz=False):
if timeout == DEFAULT_TIMEOUT:
timeout = self.default_timeout
db = router.db_for_write(self.cache_model_class)
table = connections[db].ops.quote_name(self._table)
cursor = connections[db].cursor()
cursor.execute("SELECT COUNT(*) FROM %s" % table)
num = cursor.fetchone()[0]
now = timezone.now()
now = now.replace(microsecond=0)
if timeout is None:
exp = datetime.max
elif use_tz:
exp = datetime.utcfromtimestamp(time.time() + timeout)
else:
exp = datetime.fromtimestamp(time.time() + timeout)
exp = exp.replace(microsecond=0)
if num > self._max_entries:
self._cull(db, cursor, now)
pickled = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
b64encoded = base64.b64encode(pickled)
# The DB column is expecting a string, so make sure the value is a
# string, not bytes. Refs #19274.
if six.PY3:
b64encoded = b64encoded.decode('latin1')
try:
with transaction.atomic(using=db):
cursor.execute("SELECT cache_key, expires FROM %s "
"WHERE cache_key = %%s" % table, [key])
result = cursor.fetchone()
exp = connections[db].ops.value_to_db_datetime(exp)
if result and (mode == 'set' or (mode == 'add' and result[1] < now)):
cursor.execute("UPDATE %s SET value = %%s, expires = %%s "
"WHERE cache_key = %%s" % table,
[b64encoded, exp, key])
else:
cursor.execute("INSERT INTO %s (cache_key, value, expires) "
"VALUES (%%s, %%s, %%s)" % table,
[key, b64encoded, exp])
except DatabaseError:
# To be threadsafe, updates/inserts are allowed to fail silently
return False
else:
return True
def _cache_store_items(cache, key, items, expiration, compress_large_data):
"""Store items in the cache.
The items will be individually pickled and combined into a binary blob,
which can then optionally be compressed. The resulting data is then
cached over one or more keys, each representing a chunk about 1MB in size.
A main cache key will be set that contains information on the other keys.
"""
# Note that we want to use pickle protocol 0 in order to be compatible
# across both Python 2 and 3. On Python 2, 0 is the default.
results = ((pickle.dumps(item, protocol=0), True, item) for item in items)
if compress_large_data:
results = _cache_compress_pickled_data(results)
for item in _cache_store_chunks(results, key, expiration):
yield item
def _cache_store_items(cache, key, items, expiration, compress_large_data):
"""Store items in the cache.
The items will be individually pickled and combined into a binary blob,
which can then optionally be compressed. The resulting data is then
cached over one or more keys, each representing a chunk about 1MB in size.
A main cache key will be set that contains information on the other keys.
"""
results = (
(pickle.dumps(item), True, item)
for item in items
)
if compress_large_data:
results = _cache_compress_pickled_data(results)
for item in _cache_store_chunks(results, key, expiration):
yield item
def test_cache_memoize_large_files_uncompressed_off_by_one(self):
"""Testing cache_memoize with large files without compression and
one byte larger than an even chunk size."""
cache_key = 'abc123'
# This takes into account the size of the pickle data, and will
# get us to just barely 3 chunks of data in cache.
data = self._build_test_chunk_data(num_chunks=2)[0] + 'x'
pickled_data = pickle.dumps(data)
def cache_func():
return data
self.spy_on(cache_func, call_original=True)
result = cache_memoize(cache_key, cache_func, large_data=True,
compress_large_data=False)
self.assertEqual(result, data)
self.assertTrue(cache_func.spy.called)
cache_key_0 = make_cache_key('%s-0' % cache_key)
cache_key_1 = make_cache_key('%s-1' % cache_key)
cache_key_2 = make_cache_key('%s-2' % cache_key)
self.assertTrue(make_cache_key(cache_key) in cache)
self.assertTrue(cache_key_0 in cache)
self.assertTrue(cache_key_1 in cache)
self.assertTrue(cache_key_2 in cache)
self.assertFalse(make_cache_key('%s-3' % cache_key) in cache)
# Verify the contents of the stored data.
stored_data = b''.join(cache.get(cache_key_0) +
cache.get(cache_key_1) +
cache.get(cache_key_2))
self.assertEqual(stored_data, pickled_data)
# Try fetching the data we stored.
cache_func.spy.reset_calls()
result = cache_memoize(cache_key, cache_func, large_data=True,
compress_large_data=False)
self.assertEqual(result, data)
self.assertFalse(cache_func.spy.called)
def submit(self, client, job):
# Log tasks to DB, before submitting the batch, as mcpclient then updates them
Task.bulk_log(self.tasks, job)
data = {"tasks": {}}
for task in self.tasks:
task_uuid = six.text_type(task.uuid)
data["tasks"][task_uuid] = self.serialize_task(task)
pickled_data = cPickle.dumps(data)
self.pending = client.submit_job(
task=six.binary_type(job.name),
data=pickled_data,
unique=six.binary_type(self.uuid),
wait_until_complete=False,
background=False,
max_retries=0,
)
logger.debug("Submitted gearman job %s (%s)", self.uuid, job.name)
def _cache_store_items(cache, key, items, expiration, compress_large_data):
"""Store items in the cache.
The items will be individually pickled and combined into a binary blob,
which can then optionally be compressed. The resulting data is then
cached over one or more keys, each representing a chunk about 1MB in size.
A main cache key will be set that contains information on the other keys.
"""
# Note that we want to use pickle protocol 0 in order to be compatible
# across both Python 2 and 3. On Python 2, 0 is the default.
results = (
(pickle.dumps(item, protocol=0), True, item)
for item in items
)
if compress_large_data:
results = _cache_compress_pickled_data(results)
for item in _cache_store_chunks(results, key, expiration):
yield item
def _base_set(self, mode, key, value, timeout=None):
if timeout is None:
timeout = self.default_timeout
db = router.db_for_write(self.cache_model_class)
table = connections[db].ops.quote_name(self._table)
cursor = connections[db].cursor()
cursor.execute("SELECT COUNT(*) FROM %s" % table)
num = cursor.fetchone()[0]
now = timezone.now()
now = now.replace(microsecond=0)
if settings.USE_TZ:
exp = datetime.utcfromtimestamp(time.time() + timeout)
else:
exp = datetime.fromtimestamp(time.time() + timeout)
exp = exp.replace(microsecond=0)
if num > self._max_entries:
self._cull(db, cursor, now)
pickled = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
encoded = base64.b64encode(pickled).strip()
cursor.execute("SELECT cache_key, expires FROM %s "
"WHERE cache_key = %%s" % table, [key])
try:
result = cursor.fetchone()
if result and (mode == 'set' or
(mode == 'add' and result[1] < now)):
cursor.execute("UPDATE %s SET value = %%s, expires = %%s "
"WHERE cache_key = %%s" % table,
[encoded, connections[db].ops.value_to_db_datetime(exp), key])
else:
cursor.execute("INSERT INTO %s (cache_key, value, expires) "
"VALUES (%%s, %%s, %%s)" % table,
[key, encoded, connections[db].ops.value_to_db_datetime(exp)])
except DatabaseError:
# To be threadsafe, updates/inserts are allowed to fail silently
transaction.rollback_unless_managed(using=db)
return False
else:
transaction.commit_unless_managed(using=db)
return True
def dumps(self, obj):
return pickle.dumps(obj, pickle.HIGHEST_PROTOCOL)
def _dump(self, value):
return pickle.dumps(value)
def _dump(self, value):
return self._fernet.encrypt(pickle.dumps(value))
def encode(self, session_dict):
"Returns the given session dictionary pickled and encoded as a string."
pickled = pickle.dumps(session_dict, pickle.HIGHEST_PROTOCOL)
hash = self._hash(pickled)
return base64.b64encode(hash.encode() + b":" + pickled).decode('ascii')
"Thread-safe in-memory cache backend."
def _value_for_db_collection(self, value, field, field_kind, db_type,
lookup):
"""
Recursively converts values from AbstractIterableFields.
Note that collection lookup values are plain values rather than
lists, sets or dicts, but they still should be converted as a
collection item (assuming all items or values are converted in
the same way).
We base the conversion on field class / kind and assume some
knowledge about field internals (e.g. that the field has an
"item_field" property that gives the right subfield for any of
its values), to avoid adding a framework for determination of
parameters for items' conversions; we do the conversion here
rather than inside get_db_prep_save/lookup for symmetry with
deconversion (which can't be in to_python because the method is
also used for data not coming from the database).
Returns a list, set, dict, string or bytes according to the
db_type given.
If the "list" db_type used for DictField, a list with keys and
values interleaved will be returned (list of pairs is not good,
because lists / tuples may need conversion themselves; the list
may still be nested for dicts containing collections).
The "string" and "bytes" db_types use serialization with pickle
protocol 0 or 2 respectively.
If an unknown db_type is specified, returns a generator
yielding converted elements / pairs with converted values.
"""
subfield, subkind, db_subtype = self._convert_as(
field.item_field, lookup)
# Do convert filter parameters.
if lookup:
# Special case where we are looking for an empty list
if lookup == 'exact' and db_type == 'list' and value == u'[]':
return []
value = self._value_for_db(value, subfield, subkind, db_subtype,
lookup)
# Convert list/set items or dict values.
else:
if field_kind == 'DictField':
# Generator yielding pairs with converted values.
value = ((key,
self._value_for_db(subvalue, subfield, subkind,
db_subtype, lookup))
for key, subvalue in six.iteritems(value))
# Return just a dict, a once-flattened list;
if db_type == 'dict':
return dict(value)
elif db_type == 'list':
return list(item for pair in value for item in pair)
else:
# Generator producing converted items.
value = (self._value_for_db(subvalue, subfield, subkind,
db_subtype, lookup)
for subvalue in value)
# "list" may be used for SetField.
if db_type in 'list':
return list(value)
elif db_type == 'set':
# assert field_kind != 'ListField'
return set(value)
# Pickled formats may be used for all collection fields,
# the fields "natural" type is serialized (something
# concrete is needed, pickle can't handle generators :-)
if db_type == 'bytes':
return pickle.dumps(field._type(value), protocol=2)
elif db_type == 'string':
return pickle.dumps(field._type(value))
# If nothing matched, pass the generator to the back-end.
return value
num = cursor.fetchone()[0]
now = timezone.now()
now = now.replace(microsecond=0)
if settings.USE_TZ: #use timezone
exp = datetime.utcfromtimestamp(time.time() + timeout)
else:
exp = datetime.fromtimestamp(time.time() + timeout)
exp = exp.replace(microsecond=0)
最大条目
if num > self._max_entries:
self._cull(db, cursor, now)
pickled = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
b64encoded = base64.b64encode(pickled)
# The DB column is expecting a string, so make sure the value is a
# string, not bytes. Refs #19274.
if six.PY3:
b64encoded = b64encoded.decode('latin1')
cursor.execute("SELECT cache_key, expires FROM %s "
"WHERE cache_key = %%s" % table, [key])
try:
result = cursor.fetchone()
if result and (mode == 'set' or
(mode == 'add' and result[1] < now)): 更新 result[1] < now 表示超时
cursor.execute("UPDATE %s SET value = %%s, expires = %%s "
"WHERE cache_key = %%s" % table,
[b64encoded, connections[db].ops.value_to_db_datetime(exp), key])
def memo(*args, **kwargs):
haxh = cPickle.dumps((args, sorted(kwargs.items())))
if haxh not in memory:
memory[haxh] = fctn(*args, **kwargs)
return memory[haxh]
def dumps(self, obj):
return pickle.dumps(obj, pickle.HIGHEST_PROTOCOL)
