def serialization_id(self): ''' serialization_id is used to keep items locally sorted and unique (eg. used redis sorted sets' score or cassandra column names) serialization_id is also used to select random activities from the feed (eg. remove activities from feeds must be fast operation) for this reason the serialization_id should be unique and not change over time eg: activity.serialization_id = 1373266755000000000042008 1373266755000 activity creation time as epoch with millisecond resolution 0000000000042 activity left padded object_id (10 digits) 008 left padded activity verb id (3 digits) :returns: int --the serialization id ''' if self.object_id >= 10 ** 10 or self.verb.id >= 10 ** 3: raise TypeError('Fatal: object_id / verb have too many digits !') if not self.time: raise TypeError('Cant serialize activities without a time') milliseconds = str(int(datetime_to_epoch(self.time) * 1000)) serialization_id_str = '%s%0.10d%0.3d' % ( milliseconds, self.object_id, self.verb.id) serialization_id = int(serialization_id_str) return serialization_id
def dumps(self, activity): self.check_type(activity) activity_time = datetime_to_epoch(activity.time) parts = [activity.actor_id, activity.verb.id, activity.object_id, activity.target_id or 0] extra_context = activity.extra_context.copy() pickle_string = '' if extra_context: pickle_string = pickle.dumps(activity.extra_context) parts += [activity_time, pickle_string] serialized_activity = ','.join(map(str, parts)) return serialized_activity
def serialization_id(self): ''' messing around with the serialization id to allow strings in the object and verb ids ''' if not self.time: raise TypeError('Cant serialize activities without a time') milliseconds = str(int(datetime_to_epoch(self.time) * 1000)) serialization_id_str = '%s_%s_%s' % ( milliseconds, self.object_id, self.verb) #milliseconds, self.object_id, self.verb.id) serialization_id = serialization_id_str return serialization_id
def dumps(self, activity): self.check_type(activity) activity_time = datetime_to_epoch(activity.time) parts = [ activity.actor_id, activity.verb.id, activity.object_id, activity.target_id or 0 ] extra_context = activity.extra_context.copy() pickle_string = '' if extra_context: pickle_string = pickle.dumps(activity.extra_context) parts += [activity_time, pickle_string] serialized_activity = ','.join(map(str, parts)) return serialized_activity
def dumps(self, activity): self.check_type(activity) # keep the milliseconds activity_time = '%.6f' % datetime_to_epoch(activity.time) parts = [activity.actor_id, activity.verb.id, activity.object_id, activity.target_id or 0] extra_context = activity.extra_context.copy() pickle_string = '' if extra_context: pickle_string = pickle.dumps(activity.extra_context) if six.PY3: pickle_string = pickle_string.decode('latin1') parts += [activity_time, pickle_string] serialized_activity = ','.join(map(str, parts)) return serialized_activity
def post(self, feed_id): print feed_id args = parser.parse_args() j = request.get_json() #{u'verb': u'posted', u'actor': u'user_6_offu1_my', u'object': u'24iqNvsZDzJ'} IdObj = namedtuple('IdObj', ['id']) activity = Activity( actor=IdObj(j['actor']), verb=Add, object=IdObj(j['object']), time=datetime.datetime.utcnow(), extra_context={"isn't": "this", "nice": "?", "activity":1} ) print "json, activity, dir(activity)", j, activity, dir(activity) feed = manager.feed_classes['normal'](feed_id) print feed #j['serialization_id'] = activity.serialization_id from stream_framework.utils import make_list_unique, datetime_to_epoch #j['serialization_id'] = '%s%0.10d%0.3d ' % str(int(datetime_to_epoch(activity.time)) * 1000) j['serialization_id'] = str(int(datetime_to_epoch(activity.time)) * 1000) print add_operation2(feed, [j]) print "serialization_id", str(int(datetime_to_epoch(activity.time)) * 1000) #print add_operation2(feed, [activity]) return j
def serialization_id(self): """ Shorter serialization id than used by default """ if self.object_id >= 10 ** 10 or self.verb.id >= 10 ** 3: raise TypeError("Fatal: object_id / verb have too many digits !") if not self.time: raise TypeError("Cant serialize activities without a time") milliseconds = str(int(datetime_to_epoch(self.time) * 1000)) # shorter than the default version serialization_id_str = "%s%0.2d%0.2d" % (milliseconds, self.object_id % 100, self.verb.id) serialization_id = int(serialization_id_str) return serialization_id
def serialization_id(self): ''' Shorter serialization id than used by default ''' if self.object_id >= 10**10 or self.verb.id >= 10**3: raise TypeError('Fatal: object_id / verb have too many digits !') if not self.creationdate: raise TypeError('Cant serialize activities without a time') milliseconds = str(int(datetime_to_epoch(self.creationdate) * 1000)) # shorter than the default version serialization_id_str = '%s%0.2d%0.2d' % (milliseconds, self.object_id % 100, self.verb.id) serialization_id = int(serialization_id_str) return serialization_id
def dumps(self, activity): self.check_type(activity) # keep the milliseconds activity_time = '%.6f' % datetime_to_epoch(activity.time) parts = [ activity.actor_id, activity.verb.id, activity.object_id, activity.target_id or 0 ] extra_context = activity.extra_context.copy() pickle_string = '' if extra_context: pickle_string = pickle.dumps(activity.extra_context) if six.PY3: pickle_string = pickle_string.decode('latin1') parts += [activity_time, pickle_string] serialized_activity = ','.join(map(str, parts)) return serialized_activity
def serialization_id(self): ''' serialization_id is used to keep items locally sorted and unique (eg. used redis sorted sets' score or cassandra column names) serialization_id is also used to select random activities from the feed (eg. remove activities from feeds must be fast operation) for this reason the serialization_id should be unique and not change over time eg: activity.serialization_id = 1373266755000000000042008 1373266755000 activity creation time as epoch with millisecond resolution 0000000000042 activity left padded object_id (10 digits) 008 left padded activity verb id (3 digits) :returns: int --the serialization id ''' milliseconds = str(int(datetime_to_epoch(self.updated_at) * 1e6)) return milliseconds
def serialization_id(self): ''' serialization_id is used to keep items locally sorted and unique (eg. used redis sorted sets' score or cassandra column names) serialization_id is also used to select random activities from the feed (eg. remove activities from feeds must be fast operation) for this reason the serialization_id should be unique and not change over time eg: activity.serialization_id = 1373266755000000000042008 1373266755000 activity creation time as epoch with millisecond resolution 0000000000042 activity left padded object_id (10 digits) 008 left padded activity verb id (3 digits) :returns: int --the serialization id ''' milliseconds = str(int(datetime_to_epoch(self.updated_at))) return milliseconds
def dumps(self, aggregated): self.check_type(aggregated) activity_serializer = self.activity_serializer_class(Activity) # start by storing the group parts = [aggregated.group] check_reserved(aggregated.group, [';;']) # store the dates for date_field in self.date_fields: value = getattr(aggregated, date_field) if value is not None: # keep the milliseconds epoch = '%.6f' % datetime_to_epoch(value) else: epoch = -1 parts += [epoch] # add the activities serialization serialized_activities = [] if self.dehydrate: if not aggregated.dehydrated: aggregated = aggregated.get_dehydrated() serialized_activities = map(str, aggregated._activity_ids) else: for activity in aggregated.activities: serialized = activity_serializer.dumps(activity) # we use semicolons as delimiter, so need to escape serialized = serialized.replace(";", "\;") serialized_activities.append(serialized) serialized_activities_part = ';'.join(serialized_activities) parts.append(serialized_activities_part) # add the minified activities parts.append(aggregated.minimized_activities) # stick everything together serialized_aggregated = ';;'.join(map(str, parts)) serialized = '%s%s' % (self.identifier, serialized_aggregated) return serialized
def dumps(self, aggregated): self.check_type(aggregated) activity_serializer = self.activity_serializer_class(Activity) # start by storing the group parts = [aggregated.group] check_reserved(aggregated.group, [';;']) # store the dates for date_field in self.date_fields: value = getattr(aggregated, date_field) epoch = datetime_to_epoch(value) if value is not None else -1 parts += [epoch] # add the activities serialization serialized_activities = [] if self.dehydrate: if not aggregated.dehydrated: aggregated = aggregated.get_dehydrated() serialized_activities = map(str, aggregated._activity_ids) else: for activity in aggregated.activities: serialized = activity_serializer.dumps(activity) check_reserved(serialized, [';', ';;']) serialized_activities.append(serialized) serialized_activities_part = ';'.join(serialized_activities) parts.append(serialized_activities_part) # add the minified activities parts.append(aggregated.minimized_activities) # stick everything together serialized_aggregated = ';;'.join(map(str, parts)) serialized = '%s%s' % (self.identifier, serialized_aggregated) return serialized
def test_conversion(self): source_date = datetime.now() epoch = datetime_to_epoch(source_date) converted_date = epoch_to_datetime(epoch) assert source_date == converted_date