forked from PerceptumNL/KhanLatest
-
Notifications
You must be signed in to change notification settings - Fork 0
/
object_property.py
160 lines (124 loc) · 5.61 KB
/
object_property.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
"""Contains custom app engine properties (see
developers.google.com/appengine/docs/python/datastore/typesandpropertyclasses
and developers.google.com/appengine/docs/python/datastore/propertyclass).
TODO(david): Rename this file to something more appropriate since it holds more
than just ObjectProperty now, or split it up.
"""
try:
import json
except ImportError:
import simplejson as json
from google.appengine.ext import db
import pickle_util
class ObjectProperty(db.BlobProperty):
"""A property that can be used to store arbitrary objects, via pickling.
IMPORTANT: this uses pickle to serialize the object contents
and, as a result, is fragile when used with objects that
contain non-primitive data, since alterations to their
class definitions (moving from one module to another, etc)
can make the serialized contents no longer deserializable.
It is recommended this is only used with primitive types
or dicts containing primitive types.
From http://kovshenin.com/2010/app-engine-json-objects-google-datastore/
"""
def validate(self, value):
"""Validate that value is pickle-able (raise an exception if not)."""
pickled_value = pickle_util.dump(value)
_ = super(ObjectProperty, self).validate(pickled_value)
return value
def get_value_for_datastore(self, model_instance):
result = (super(ObjectProperty, self)
.get_value_for_datastore(model_instance))
result = pickle_util.dump(result)
return db.Blob(result)
def make_value_from_datastore(self, value):
value = pickle_util.load(str(value))
return super(ObjectProperty, self).make_value_from_datastore(value)
class UnvalidatedObjectProperty(ObjectProperty):
"""Like ObjectProperty but just assumes passed-in values can be pickled."""
def validate(self, value):
# pickle.dumps can be slooooooow, sometimes we just want to
# trust that the item is pickle'able (and that it fills all
# the validity requirements of db.BlobProperty as well.)
return value
class JsonProperty(db.TextProperty):
"""An alternative to ObjectProperty that uses JSON for serialization. Note:
keys and values must all be primitives (eg. datetime objects as keys will
raise an error on put).
JSON is generally faster and smaller in Python (see
http://kovshenin.com/2010/pickle-vs-json-which-is-faster/ and
http://inkdroid.org/journal/2008/10/24/json-vs-pickle/), but can only be
used for serializing primitives. Pickle can be used to serialize classes,
for example, but storing classes is brittle and could cause issues if
changing class names, moving classes to different files, or switching
between Python 2.5 and 2.7.
Modified from
http://kovshenin.com/2010/app-engine-json-objects-google-datastore/
"""
def validate(self, value):
jsoned_value = json.dumps(str(value))
_ = super(JsonProperty, self).validate(jsoned_value)
return value
def get_value_for_datastore(self, model_instance):
result = super(JsonProperty, self).get_value_for_datastore(
model_instance)
result = json.dumps(result)
return db.Text(result)
def make_value_from_datastore(self, value):
value = json.loads(str(value))
return super(JsonProperty, self).make_value_from_datastore(value)
class TsvProperty(db.TextProperty):
'''
An alternative to StringListProperty that serializes lists using a simple
tab-separated format. This is much faster than StringPropertyList, however
elements with tabs are not permitted.
'''
data_type = list
def __init__(self, default=None, **kwds):
if default is None:
default = []
super(TsvProperty, self).__init__(default=default, **kwds)
def get_value_for_datastore(self, model_instance):
value = (super(TsvProperty, self)
.get_value_for_datastore(model_instance))
return db.Text("\t".join(value or []))
def make_value_from_datastore(self, value):
return self.str_to_tsv(value)
@staticmethod
def str_to_tsv(value):
return value.split("\t") if value else []
def empty(self, value):
"""Is list property empty.
[] is not an empty value.
Returns:
True if value is None, else false.
"""
return value is None
def default_value(self):
"""Default value for list.
Because the property supplied to 'default' is a static value,
that value must be shallow copied to prevent all fields with
default values from sharing the same instance.
Returns:
Copy of the default value.
"""
return list(super(TsvProperty, self).default_value())
# the following properties are useful for migrating StringListProperty to
# the faster TsvProperty
class TsvCompatStringListProperty(db.StringListProperty):
"""A StringListProperty that can also read lists serialized as tab
separated strings"""
def make_value_from_datastore(self, value):
if isinstance(value, list):
return (super(TsvCompatStringListProperty, self)
.make_value_from_datastore(value))
else:
return TsvProperty.str_to_tsv(value)
class StringListCompatTsvProperty(TsvProperty):
'A TsvProperty that can also read lists serialized as native Python lists'
def make_value_from_datastore(self, value):
if isinstance(value, list):
return value
else:
return (super(StringListCompatTsvProperty, self)
.make_value_from_datastore(value))