def serialization_id(self):
'''
serialization_id is used to keep items locally sorted and unique
(eg. used redis sorted sets' score or cassandra column names)
serialization_id is also used to select random activities from the feed
(eg. remove activities from feeds must be fast operation)
for this reason the serialization_id should be unique and not change over time
eg:
activity.serialization_id = 1373266755000000000042008
1373266755000 activity creation time as epoch with millisecond resolution
0000000000042 activity left padded object_id (10 digits)
008 left padded activity verb id (3 digits)
:returns: int --the serialization id
'''
if self.object_id >= 10 ** 10 or self.verb.id >= 10 ** 3:
raise TypeError('Fatal: object_id / verb have too many digits !')
if not self.time:
raise TypeError('Cant serialize activities without a time')
milliseconds = str(int(datetime_to_epoch(self.time) * 1000))
serialization_id_str = '%s%0.10d%0.3d' % (
milliseconds, self.object_id, self.verb.id)
serialization_id = int(serialization_id_str)
return serialization_id
def serialization_id(self):
'''
serialization_id is used to keep items locally sorted and unique
(eg. used redis sorted sets' score or cassandra column names)
serialization_id is also used to select random activities from the feed
(eg. remove activities from feeds must be fast operation)
for this reason the serialization_id should be unique and not change over time
eg:
activity.serialization_id = 1373266755000000000042008
1373266755000 activity creation time as epoch with millisecond resolution
0000000000042 activity left padded object_id (10 digits)
008 left padded activity verb id (3 digits)
:returns: int --the serialization id
'''
if self.object_id >= 10 ** 10 or self.verb.id >= 10 ** 3:
raise TypeError('Fatal: object_id / verb have too many digits !')
if not self.time:
raise TypeError('Cant serialize activities without a time')
milliseconds = str(int(datetime_to_epoch(self.time) * 1000))
serialization_id_str = '%s%0.10d%0.3d' % (
milliseconds, self.object_id, self.verb.id)
serialization_id = int(serialization_id_str)
return serialization_id
def dumps(self, activity):
self.check_type(activity)
activity_time = datetime_to_epoch(activity.time)
parts = [activity.actor_id, activity.verb.id,
activity.object_id, activity.target_id or 0]
extra_context = activity.extra_context.copy()
pickle_string = ''
if extra_context:
pickle_string = pickle.dumps(activity.extra_context)
parts += [activity_time, pickle_string]
serialized_activity = ','.join(map(str, parts))
return serialized_activity
def serialization_id(self):
'''
messing around with the serialization id to allow strings
in the object and verb ids
'''
if not self.time:
raise TypeError('Cant serialize activities without a time')
milliseconds = str(int(datetime_to_epoch(self.time) * 1000))
serialization_id_str = '%s_%s_%s' % (
milliseconds, self.object_id, self.verb)
#milliseconds, self.object_id, self.verb.id)
serialization_id = serialization_id_str
return serialization_id
def dumps(self, activity):
self.check_type(activity)
activity_time = datetime_to_epoch(activity.time)
parts = [
activity.actor_id, activity.verb.id, activity.object_id,
activity.target_id or 0
]
extra_context = activity.extra_context.copy()
pickle_string = ''
if extra_context:
pickle_string = pickle.dumps(activity.extra_context)
parts += [activity_time, pickle_string]
serialized_activity = ','.join(map(str, parts))
return serialized_activity
def dumps(self, activity):
self.check_type(activity)
# keep the milliseconds
activity_time = '%.6f' % datetime_to_epoch(activity.time)
parts = [activity.actor_id, activity.verb.id,
activity.object_id, activity.target_id or 0]
extra_context = activity.extra_context.copy()
pickle_string = ''
if extra_context:
pickle_string = pickle.dumps(activity.extra_context)
if six.PY3:
pickle_string = pickle_string.decode('latin1')
parts += [activity_time, pickle_string]
serialized_activity = ','.join(map(str, parts))
return serialized_activity
def post(self, feed_id):
print feed_id
args = parser.parse_args()
j = request.get_json()
#{u'verb': u'posted', u'actor': u'user_6_offu1_my', u'object': u'24iqNvsZDzJ'}
IdObj = namedtuple('IdObj', ['id'])
activity = Activity(
actor=IdObj(j['actor']),
verb=Add,
object=IdObj(j['object']),
time=datetime.datetime.utcnow(),
extra_context={"isn't": "this", "nice": "?", "activity":1}
)
print "json, activity, dir(activity)", j, activity, dir(activity)
feed = manager.feed_classes['normal'](feed_id)
print feed
#j['serialization_id'] = activity.serialization_id
from stream_framework.utils import make_list_unique, datetime_to_epoch
#j['serialization_id'] = '%s%0.10d%0.3d ' % str(int(datetime_to_epoch(activity.time)) * 1000)
j['serialization_id'] = str(int(datetime_to_epoch(activity.time)) * 1000)
print add_operation2(feed, [j])
print "serialization_id", str(int(datetime_to_epoch(activity.time)) * 1000)
#print add_operation2(feed, [activity])
return j
def serialization_id(self):
"""
Shorter serialization id than used by default
"""
if self.object_id >= 10 ** 10 or self.verb.id >= 10 ** 3:
raise TypeError("Fatal: object_id / verb have too many digits !")
if not self.time:
raise TypeError("Cant serialize activities without a time")
milliseconds = str(int(datetime_to_epoch(self.time) * 1000))
# shorter than the default version
serialization_id_str = "%s%0.2d%0.2d" % (milliseconds, self.object_id % 100, self.verb.id)
serialization_id = int(serialization_id_str)
return serialization_id
def serialization_id(self):
'''
Shorter serialization id than used by default
'''
if self.object_id >= 10**10 or self.verb.id >= 10**3:
raise TypeError('Fatal: object_id / verb have too many digits !')
if not self.creationdate:
raise TypeError('Cant serialize activities without a time')
milliseconds = str(int(datetime_to_epoch(self.creationdate) * 1000))
# shorter than the default version
serialization_id_str = '%s%0.2d%0.2d' % (milliseconds, self.object_id %
100, self.verb.id)
serialization_id = int(serialization_id_str)
return serialization_id
def dumps(self, activity):
self.check_type(activity)
# keep the milliseconds
activity_time = '%.6f' % datetime_to_epoch(activity.time)
parts = [
activity.actor_id, activity.verb.id, activity.object_id,
activity.target_id or 0
]
extra_context = activity.extra_context.copy()
pickle_string = ''
if extra_context:
pickle_string = pickle.dumps(activity.extra_context)
if six.PY3:
pickle_string = pickle_string.decode('latin1')
parts += [activity_time, pickle_string]
serialized_activity = ','.join(map(str, parts))
return serialized_activity
def serialization_id(self):
'''
serialization_id is used to keep items locally sorted and unique
(eg. used redis sorted sets' score or cassandra column names)
serialization_id is also used to select random activities from the feed
(eg. remove activities from feeds must be fast operation)
for this reason the serialization_id should be unique and not change over time
eg:
activity.serialization_id = 1373266755000000000042008
1373266755000 activity creation time as epoch with millisecond resolution
0000000000042 activity left padded object_id (10 digits)
008 left padded activity verb id (3 digits)
:returns: int --the serialization id
'''
milliseconds = str(int(datetime_to_epoch(self.updated_at) * 1e6))
return milliseconds
def serialization_id(self):
'''
serialization_id is used to keep items locally sorted and unique
(eg. used redis sorted sets' score or cassandra column names)
serialization_id is also used to select random activities from the feed
(eg. remove activities from feeds must be fast operation)
for this reason the serialization_id should be unique and not change over time
eg:
activity.serialization_id = 1373266755000000000042008
1373266755000 activity creation time as epoch with millisecond resolution
0000000000042 activity left padded object_id (10 digits)
008 left padded activity verb id (3 digits)
:returns: int --the serialization id
'''
milliseconds = str(int(datetime_to_epoch(self.updated_at)))
return milliseconds
def dumps(self, aggregated):
self.check_type(aggregated)
activity_serializer = self.activity_serializer_class(Activity)
# start by storing the group
parts = [aggregated.group]
check_reserved(aggregated.group, [';;'])
# store the dates
for date_field in self.date_fields:
value = getattr(aggregated, date_field)
if value is not None:
# keep the milliseconds
epoch = '%.6f' % datetime_to_epoch(value)
else:
epoch = -1
parts += [epoch]
# add the activities serialization
serialized_activities = []
if self.dehydrate:
if not aggregated.dehydrated:
aggregated = aggregated.get_dehydrated()
serialized_activities = map(str, aggregated._activity_ids)
else:
for activity in aggregated.activities:
serialized = activity_serializer.dumps(activity)
# we use semicolons as delimiter, so need to escape
serialized = serialized.replace(";", "\;")
serialized_activities.append(serialized)
serialized_activities_part = ';'.join(serialized_activities)
parts.append(serialized_activities_part)
# add the minified activities
parts.append(aggregated.minimized_activities)
# stick everything together
serialized_aggregated = ';;'.join(map(str, parts))
serialized = '%s%s' % (self.identifier, serialized_aggregated)
return serialized
def dumps(self, aggregated):
self.check_type(aggregated)
activity_serializer = self.activity_serializer_class(Activity)
# start by storing the group
parts = [aggregated.group]
check_reserved(aggregated.group, [';;'])
# store the dates
for date_field in self.date_fields:
value = getattr(aggregated, date_field)
epoch = datetime_to_epoch(value) if value is not None else -1
parts += [epoch]
# add the activities serialization
serialized_activities = []
if self.dehydrate:
if not aggregated.dehydrated:
aggregated = aggregated.get_dehydrated()
serialized_activities = map(str, aggregated._activity_ids)
else:
for activity in aggregated.activities:
serialized = activity_serializer.dumps(activity)
check_reserved(serialized, [';', ';;'])
serialized_activities.append(serialized)
serialized_activities_part = ';'.join(serialized_activities)
parts.append(serialized_activities_part)
# add the minified activities
parts.append(aggregated.minimized_activities)
# stick everything together
serialized_aggregated = ';;'.join(map(str, parts))
serialized = '%s%s' % (self.identifier, serialized_aggregated)
return serialized
def dumps(self, aggregated):
self.check_type(aggregated)
activity_serializer = self.activity_serializer_class(Activity)
# start by storing the group
parts = [aggregated.group]
check_reserved(aggregated.group, [';;'])
# store the dates
for date_field in self.date_fields:
value = getattr(aggregated, date_field)
epoch = datetime_to_epoch(value) if value is not None else -1
parts += [epoch]
# add the activities serialization
serialized_activities = []
if self.dehydrate:
if not aggregated.dehydrated:
aggregated = aggregated.get_dehydrated()
serialized_activities = map(str, aggregated._activity_ids)
else:
for activity in aggregated.activities:
serialized = activity_serializer.dumps(activity)
check_reserved(serialized, [';', ';;'])
serialized_activities.append(serialized)
serialized_activities_part = ';'.join(serialized_activities)
parts.append(serialized_activities_part)
# add the minified activities
parts.append(aggregated.minimized_activities)
# stick everything together
serialized_aggregated = ';;'.join(map(str, parts))
serialized = '%s%s' % (self.identifier, serialized_aggregated)
return serialized
def test_conversion(self):
source_date = datetime.now()
epoch = datetime_to_epoch(source_date)
converted_date = epoch_to_datetime(epoch)
assert source_date == converted_date
