def to_python(self, value):
"""
Resolve the model from the class name and id
"""
if value is not None:
try:
value = dbsafe_decode(value, self.compress)
except:
# If the value is a definite pickle; and an error is raised in
# de-pickling it should be allowed to propogate.
if isinstance(value, PickledObject):
raise
else:
# If the value was encoded (not from cache, convert it here back to our
# desired format)
if value:
if isinstance(value, _ObjectWrapper):
unwrapped_value = value._obj
else:
unwrapped_value = value
return map_dict_to_dict(
lambda key, inner_dict: [
key,
map_dict_to_dict(
lambda inner_key, model_dict: [
inner_key,
self.model_from_class_name_and_pk(
model_dict)
], inner_dict)
], unwrapped_value)
# Value from cache, leave alone
return value
def get_result_group(group_id, failures=False):
# values + decode is 10 times faster than just list comprehension
if failures:
values = Task.objects.filter(group=group_id).values_list('result', flat=True)
else:
values = Task.objects.filter(group=group_id).exclude(success=False).values_list('result', flat=True)
return [dbsafe_decode(t) for t in values]
def copy_data(self):
"""
Copy the data from the it's point of origin, serializing it,
storing it serialized as well as in it's raw form and calculate
a running hash of the serialized representation
"""
hash_function = hashlib.sha256()
self.temporary_file = tempfile.NamedTemporaryFile(mode='w+')
self.copy_file = tempfile.NamedTemporaryFile(mode='w+b')
for row in self.get_data_iteraror():
data = dbsafe_encode(row)
self.temporary_file.write(dbsafe_encode(row))
self.temporary_file.write('\n')
hash_function.update(data)
if isinstance(row, types.StringTypes):
self.copy_file.write(row)
self.temporary_file.seek(0)
self.copy_file.seek(0)
self.new_hash = hash_function.hexdigest()
self.data_iterator = (dbsafe_decode(line[:-1]) for line in self.temporary_file)
# Return the serialized content, an iterator to decode the serialized content, a handler to the raw content and the hash
return self.temporary_file, self.data_iterator, self.copy_file, self.new_hash
def copy_data(self):
"""
Copy the data from the it's point of origin, serializing it,
storing it serialized as well as in it's raw form and calculate
a running hash of the serialized representation
"""
HASH_FUNCTION = hashlib.sha256()
try:
raw_iterator = self.get_binary_iterator()
except AttributeError:
raw_iterator = self.get_non_binary_iterator()
self.copy_file = tempfile.NamedTemporaryFile(mode='w+')
for part in raw_iterator:
encoded_part = dbsafe_encode(part)
self.copy_file.write(encoded_part)
self.copy_file.write('\n')
HASH_FUNCTION.update(encoded_part)
self.copy_file.seek(0)
self.data_iterator = (dbsafe_decode(line)
for line in self.copy_file)
else:
self.copy_file = tempfile.NamedTemporaryFile(mode='w+b')
for part in raw_iterator:
self.copy_file.write(part)
HASH_FUNCTION.update(part)
self.copy_file.seek(0)
self.data_iterator = self.copy_file
self.new_hash = HASH_FUNCTION.hexdigest()
def to_python(self, value):
"""
Resolve the model from the class name and id
"""
if value is not None:
try:
value = dbsafe_decode(value, self.compress)
except:
# If the value is a definite pickle; and an error is raised in
# de-pickling it should be allowed to propogate.
if isinstance(value, PickledObject):
raise
else:
# If the value was encoded (not from cache, convert it here back to our
# desired format)
if value:
if isinstance(value, _ObjectWrapper):
unwrapped_value = value._obj
else:
unwrapped_value = value
return map_dict_to_dict(lambda key, inner_dict:
[key, map_dict_to_dict(lambda inner_key, model_dict:
[inner_key,
self.model_from_class_name_and_pk(model_dict)],
inner_dict
)],
unwrapped_value)
# Value from cache, leave alone
return value
def copy_data(self):
"""
Copy the data from the it's point of origin, serializing it,
storing it serialized as well as in it's raw form and calculate
a running hash of the serialized representation
"""
HASH_FUNCTION = hashlib.sha256()
try:
raw_iterator = self.get_binary_iterator()
except AttributeError:
raw_iterator = self.get_non_binary_iterator()
self.copy_file = tempfile.NamedTemporaryFile(mode='w+')
for part in raw_iterator:
encoded_part = dbsafe_encode(part)
self.copy_file.write(encoded_part)
self.copy_file.write('\n')
HASH_FUNCTION.update(encoded_part)
self.copy_file.seek(0)
self.data_iterator = (dbsafe_decode(line) for line in self.copy_file)
else:
self.copy_file = tempfile.NamedTemporaryFile(mode='w+b')
for part in raw_iterator:
self.copy_file.write(part)
HASH_FUNCTION.update(part)
self.copy_file.seek(0)
self.data_iterator = self.copy_file
self.new_hash = HASH_FUNCTION.hexdigest()
def get_result_group(group_id, failures=False):
# values + decode is 10 times faster than just list comprehension
if failures:
values = Task.objects.filter(group=group_id).values_list('result',
flat=True)
else:
values = Task.objects.filter(group=group_id).exclude(
success=False).values_list('result', flat=True)
return [dbsafe_decode(t) for t in values]
def field_monkey(self, bundle):
result = self.__dehydrate_nomonkey__(bundle)
for f in result.data['fields']:
if 'choices' in f.data:
choices = []
for choice in f.data['choices']:
try:
choice['value'] = dbsafe_decode(choice['value'])
except:
pass
choices.append(choice)
f.data['choices'] = choices
return result
def to_python(self, value):
if value is not None and isinstance(value, six.string_types):
value = decrypt(value)
# The following below is a copypaste of PickledObjectField.to_python of
# v1.0.0 with one change: We re-raise any baseexceptions such as
# signals. 1.0.0 has a bare `except:` which causes issues.
if value is not None:
try:
value = dbsafe_decode(value, self.compress)
except Exception:
# If the value is a definite pickle; and an error is raised in
# de-pickling it should be allowed to propagate.
if isinstance(value, PickledObject):
raise
else:
if isinstance(value, _ObjectWrapper):
return value._obj
return value
def get_installation_status():
from django.db import connection
from picklefield.fields import dbsafe_decode
from core.libs.utility import version_compare
cursor = connection.cursor()
try:
cursor.execute('SELECT value FROM %soptions WHERE name = "jadb_version"' % Core.settings.DB_TABLE_PREFIX)
except Exception:
return -2
version = cursor.fetchone()
if version is None:
return -2
version = dbsafe_decode(version[0])
if version_compare(version, Core.version[0]) != 0:
return -2
return 0
def decode_results(values):
if get_version().split('.')[1] == '7':
# decode values in 1.7
return [dbsafe_decode(v) for v in values]
return values
def decode_results(values):
if get_version().split('.')[1] == '7':
# decode values in 1.7
return [dbsafe_decode(v) for v in values]
return values
def dbsafe_decode_aes(value, *args, **kwargs):
return dbsafe_decode(aes.base64_decrypt(value), *args, **kwargs)
