def test_the_timestamp_are_based_on_real_time_milliseconds(self):
before = utils.msec_time()
time.sleep(0.002) # 2 msec
record = self.create_record()
now = record['last_modified']
time.sleep(0.002) # 2 msec
after = utils.msec_time()
self.assertTrue(before < now < after,
'{} < {} < {}'.format(before, now, after))
def on_new_response(event):
response = event.response
request = event.request
# Compute the request processing time in msec (-1 if unknown)
current = utils.msec_time()
duration = current - getattr(request, '_received_at', current - 1)
isotimestamp = datetime.fromtimestamp(current/1000).isoformat()
# Bind infos for request summary logger.
request.log_context(time=isotimestamp,
code=response.status_code,
t=duration)
if summary_logger.level == logging.DEBUG:
request.log_context(response=dict(headers=dict(response.headers),
body=response.body))
try:
# If error response, bind errno.
request.log_context(errno=response.errno)
except AttributeError:
pass
if not hasattr(request, 'parent'):
# Ouput application request summary.
summary_logger.info('', extra=request.log_context())
def bump_timestamp(self, collection_timestamp, record, modified_field,
last_modified):
"""Timestamp are base on current millisecond.
.. note ::
Here it is assumed that if requests from the same user burst in,
the time will slide into the future. It is not problematic since
the timestamp notion is opaque, and behaves like a revision number.
"""
is_specified = record is not None and modified_field in record or last_modified is not None
if is_specified:
# If there is a timestamp in the new record, try to use it.
if last_modified is not None:
current = last_modified
else:
current = record[modified_field]
# If it is equal to current collection timestamp, bump it.
if current == collection_timestamp:
collection_timestamp += 1
current = collection_timestamp
# If it is superior (future), use it as new collection timestamp.
elif current > collection_timestamp:
collection_timestamp = current
# Else (past), do nothing.
else:
# Not specified, use a new one.
current = utils.msec_time()
# If two ops in the same msec, bump it.
if current <= collection_timestamp:
current = collection_timestamp + 1
collection_timestamp = current
return current, collection_timestamp
def on_new_request(event):
request = event.request
# Save the time the request was received by the server.
event.request._received_at = utils.msec_time()
try:
# Pyramid fails if the URL contains invalid UTF-8 characters.
request_path = event.request.path
except UnicodeDecodeError:
raise errors.http_error(
HTTPBadRequest(),
errno=errors.ERRORS.INVALID_PARAMETERS,
message="Invalid URL path.",
)
request.log_context(
agent=request.headers.get("User-Agent"),
path=request_path,
method=request.method,
lang=request.headers.get("Accept-Language"),
errno=0,
)
qs = dict(errors.request_GET(request))
if qs:
request.log_context(querystring=qs)
if summary_logger.level == logging.DEBUG:
request.log_context(headers=dict(request.headers),
body=request.body)
def on_new_request(event):
request = event.request
# Save the time the request was received by the server.
event.request._received_at = utils.msec_time()
try:
# Pyramid fails if the URL contains invalid UTF-8 characters.
request_path = event.request.path
except UnicodeDecodeError:
raise errors.http_error(
HTTPBadRequest(),
errno=errors.ERRORS.INVALID_PARAMETERS,
message='Invalid URL path.')
request.log_context(agent=request.headers.get('User-Agent'),
path=request_path,
method=request.method,
lang=request.headers.get('Accept-Language'),
errno=0)
qs = dict(errors.request_GET(request))
if qs:
request.log_context(querystring=qs)
if summary_logger.level == logging.DEBUG:
request.log_context(headers=dict(request.headers),
body=request.body)
def on_new_response(event):
response = event.response
request = event.request
# Compute the request processing time in msec (-1 if unknown)
current = utils.msec_time()
duration = current - getattr(request, "_received_at", current - 1)
isotimestamp = datetime.fromtimestamp(current / 1000).isoformat()
# Bind infos for request summary logger.
request.log_context(time=isotimestamp,
code=response.status_code,
t=duration)
if summary_logger.level == logging.DEBUG:
request.log_context(response=dict(headers=dict(response.headers),
body=response.body))
try:
# If error response, bind errno.
request.log_context(errno=response.errno)
except AttributeError:
pass
if not hasattr(request, "parent"):
# Ouput application request summary.
summary_logger.info("", extra=request.log_context())
def accumulate_continents(value):
"""
Pick continents from countries and populate a list of dictionaries with unique values
:param value: country to inspect
"""
if value[2] not in [i.code for i in continents]:
continents.append(Continent(**{'code': value[2], 'name': value[3], 'sdate': datetime.datetime(2001, 1, 1),
'parent_id': PARENT_ID, 'last_modified': msec_time()}))
def set(self, key, value, ttl):
if isinstance(value, bytes):
raise TypeError("a string-like object is required, not 'bytes'")
self._clean_expired()
self._clean_oversized()
self.expire(key, ttl)
item_key = self.prefix + key
self._store[item_key] = value
self._created_at[item_key] = msec_time()
self._quota += size_of(item_key, value)
def set(self, key, value, ttl):
if isinstance(value, bytes):
raise TypeError("a string-like object is required, not 'bytes'")
self._clean_expired()
self._clean_oversized()
self.expire(key, ttl)
item_key = self.prefix + key
self._store[item_key] = value
self._created_at[item_key] = msec_time()
self._quota += size_of(item_key, value)
def set(self, key, value, ttl=None):
self._clean_expired()
self._clean_oversized()
if ttl is not None:
self.expire(key, ttl)
else:
logger.warning("No TTL for cache key '{}'".format(key))
item_key = self.prefix + key
self._store[item_key] = value
self._created_at[item_key] = msec_time()
self._quota += size_of(item_key, value)
def map_city_class(value):
"""
Map a csv list to a dictionary useful to create a city instance
:param value: a list of values
:return: a dictionary where the input values has been mapped to keys
"""
return {'country_id': get_country_id(value[4]), 'sd_1_iso_code': value[6],
'sd_1_name': value[7], 'sd_2_iso_code': value[8], 'sd_2_name': value[9],
'name': value[10], 'metro_code': value[11], 'time_zone': value[12],
'parent_id': PARENT_ID, 'last_modified': msec_time()}
def on_new_request(event):
request = event.request
# Save the time the request was received by the server.
event.request._received_at = utils.msec_time()
# New logger context, with infos for request summary logger.
logger.new(agent=request.headers.get('User-Agent'),
path=event.request.path,
method=request.method,
querystring=dict(request.GET),
lang=request.headers.get('Accept-Language'),
uid=None,
authn_type=None,
errno=None)
def on_new_request(event):
request = event.request
# Save the time the request was received by the server.
event.request._received_at = utils.msec_time()
# New logger context, with infos for request summary logger.
logger.new(agent=request.headers.get('User-Agent'),
path=event.request.path,
method=request.method,
querystring=dict(request.GET),
lang=request.headers.get('Accept-Language'),
uid=None,
authn_type=None,
errno=None)
def on_new_response(event):
response = event.response
request = event.request
# Compute the request processing time in msec (-1 if unknown)
current = utils.msec_time()
duration = current - getattr(request, '_received_at', current - 1)
isotimestamp = datetime.fromtimestamp(current / 1000).isoformat()
# Bind infos for request summary logger.
logger.bind(time=isotimestamp, code=response.status_code, t=duration)
# Ouput application request summary.
if not hasattr(request, 'parent'):
logger.info('request.summary')
def _bump_timestamp(self,
collection_id,
parent_id,
record=None,
modified_field=None,
last_modified=None):
key = '{0}.{1}.timestamp'.format(collection_id, parent_id)
while 1:
with self._client.pipeline() as pipe:
try:
pipe.watch(key)
previous = pipe.get(key)
pipe.multi()
# XXX factorize code from memory and redis backends.
is_specified = (record is not None
and modified_field in record
or last_modified is not None)
if is_specified:
# If there is a timestamp in the new record,
# try to use it.
if last_modified is not None:
current = last_modified
else:
current = record[modified_field]
else:
current = utils.msec_time()
if previous and int(previous) >= current:
collection_timestamp = int(previous) + 1
else:
collection_timestamp = current
# Return the newly generated timestamp as the current one
# only if nothing else was specified.
if not is_specified:
current = collection_timestamp
pipe.set(key, collection_timestamp)
pipe.execute()
return current
except redis.WatchError: # pragma: no cover
# Our timestamp has been modified by someone else, let's
# retry.
# XXX: untested.
continue
def on_new_response(event):
response = event.response
request = event.request
# Compute the request processing time in msec (-1 if unknown)
current = utils.msec_time()
duration = current - getattr(request, '_received_at', current - 1)
isotimestamp = datetime.fromtimestamp(current/1000).isoformat()
# Bind infos for request summary logger.
logger.bind(time=isotimestamp,
code=response.status_code,
t=duration)
# Ouput application request summary.
if not hasattr(request, 'parent'):
logger.info('request.summary')
def __call__(self, logger, name, event_dict):
SYSLOG_LEVELS = {
'critical': 0,
'fatal': 0,
'exception': 2,
'error': 2,
'warning': 4,
'info': 6,
'debug': 7,
}
severity = SYSLOG_LEVELS[name]
MSEC_TO_NANOSEC = 1000000
timestamp = utils.msec_time() * MSEC_TO_NANOSEC
event = event_dict.pop('event', '')
defaults = {
'Timestamp': timestamp,
'Logger': self.appname,
'Type': event,
'Hostname': self.hostname,
'Severity': severity,
'Pid': self.pid,
'EnvVersion': self.ENV_VERSION,
'Fields': {}
}
for f, v in defaults.items():
event_dict.setdefault(f, v)
fields = [k for k in event_dict.keys() if k not in defaults]
for f in fields:
value = event_dict.pop(f)
# Heka relies on Protobuf, which doesn't support recursive objects.
if isinstance(value, dict):
value = utils.json.dumps(value)
elif isinstance(value, (list, tuple)):
if not all([isinstance(i, str) for i in value]):
value = utils.json.dumps(value)
event_dict['Fields'][f] = value
return utils.json.dumps(event_dict)
def __call__(self, logger, name, event_dict):
SYSLOG_LEVELS = {
'critical': 0,
'fatal': 0,
'exception': 2,
'error': 2,
'warning': 4,
'info': 6,
'debug': 7,
}
severity = SYSLOG_LEVELS[name]
MSEC_TO_NANOSEC = 1000000
timestamp = utils.msec_time() * MSEC_TO_NANOSEC
event = event_dict.pop('event', '')
defaults = {
'Timestamp': timestamp,
'Logger': self.appname,
'Type': event,
'Hostname': self.hostname,
'Severity': severity,
'Pid': self.pid,
'EnvVersion': self.ENV_VERSION,
'Fields': {}
}
for f, v in defaults.items():
event_dict.setdefault(f, v)
fields = [k for k in event_dict.keys() if k not in defaults]
for f in fields:
value = event_dict.pop(f)
# Heka relies on Protobuf, which doesn't support recursive objects.
if isinstance(value, dict):
value = utils.json.dumps(value)
elif isinstance(value, (list, tuple)):
if not all([isinstance(i, six.string_types) for i in value]):
value = utils.json.dumps(value)
event_dict['Fields'][f] = value
return utils.json.dumps(event_dict)
def _bump_timestamp(self, collection_id, parent_id, record=None,
modified_field=None, last_modified=None):
key = '{0}.{1}.timestamp'.format(collection_id, parent_id)
while 1:
with self._client.pipeline() as pipe:
try:
pipe.watch(key)
previous = pipe.get(key)
pipe.multi()
# XXX factorize code from memory and redis backends.
is_specified = (record is not None and
modified_field in record or
last_modified is not None)
if is_specified:
# If there is a timestamp in the new record,
# try to use it.
if last_modified is not None:
current = last_modified
else:
current = record[modified_field]
else:
current = utils.msec_time()
if previous and int(previous) >= current:
collection_timestamp = int(previous) + 1
else:
collection_timestamp = current
# Return the newly generated timestamp as the current one
# only if nothing else was specified.
is_equal = previous and int(previous) == current
if not is_specified or is_equal:
current = collection_timestamp
pipe.set(key, collection_timestamp)
pipe.execute()
return current
except redis.WatchError: # pragma: no cover
# Our timestamp has been modified by someone else, let's
# retry.
# XXX: untested.
continue
def _bump_timestamp(self,
collection_id,
parent_id,
record=None,
modified_field=None,
last_modified=None):
"""Timestamp are base on current millisecond.
.. note ::
Here it is assumed that if requests from the same user burst in,
the time will slide into the future. It is not problematic since
the timestamp notion is opaque, and behaves like a revision number.
"""
# XXX factorize code from memory and redis backends.
is_specified = (record is not None and modified_field in record
or last_modified is not None)
if is_specified:
# If there is a timestamp in the new record, try to use it.
if last_modified is not None:
current = last_modified
else:
current = record[modified_field]
else:
# Otherwise, use a new one.
current = utils.msec_time()
# Bump the timestamp only if it's more than the previous one.
previous = self._timestamps[parent_id].get(collection_id)
if previous and previous >= current:
collection_timestamp = previous + 1
else:
collection_timestamp = current
# In case the timestamp was specified, the collection timestamp will
# be different from the updated timestamp. As such, we want to return
# the one of the record, and not the collection one.
if not is_specified or previous == current:
current = collection_timestamp
self._timestamps[parent_id][collection_id] = collection_timestamp
return current
def on_new_request(event):
request = event.request
# Save the time the request was received by the server.
event.request._received_at = utils.msec_time()
try:
# Pyramid fails if the URL contains invalid UTF-8 characters.
request_path = event.request.path
except UnicodeDecodeError:
raise errors.http_error(HTTPBadRequest(),
errno=errors.ERRORS.INVALID_PARAMETERS,
message='Invalid URL path.')
request.log_context(agent=request.headers.get('User-Agent'),
path=request_path,
method=request.method,
querystring=dict(errors.request_GET(request)),
lang=request.headers.get('Accept-Language'),
uid=None,
authn_type=None,
errno=None)
def _bump_timestamp(self, collection_id, parent_id, record=None,
modified_field=None, last_modified=None):
"""Timestamp are base on current millisecond.
.. note ::
Here it is assumed that if requests from the same user burst in,
the time will slide into the future. It is not problematic since
the timestamp notion is opaque, and behaves like a revision number.
"""
# XXX factorize code from memory and redis backends.
is_specified = (record is not None and
modified_field in record or
last_modified is not None)
if is_specified:
# If there is a timestamp in the new record, try to use it.
if last_modified is not None:
current = last_modified
else:
current = record[modified_field]
else:
# Otherwise, use a new one.
current = utils.msec_time()
# Bump the timestamp only if it's more than the previous one.
previous = self._timestamps[parent_id].get(collection_id)
if previous and previous >= current:
collection_timestamp = previous + 1
else:
collection_timestamp = current
# In case the timestamp was specified, the collection timestamp will
# be different from the updated timestamp. As such, we want to return
# the one of the record, and not the collection one.
if not is_specified or previous == current:
current = collection_timestamp
self._timestamps[parent_id][collection_id] = collection_timestamp
return current
def on_new_request(event):
request = event.request
# Save the time the request was received by the server.
event.request._received_at = utils.msec_time()
try:
# Pyramid fails if the URL contains invalid UTF-8 characters.
request_path = event.request.path
except UnicodeDecodeError:
raise errors.http_error(
HTTPBadRequest(),
errno=errors.ERRORS.INVALID_PARAMETERS,
message="Invalid URL path.")
# New logger context, with infos for request summary logger.
logger.new(agent=request.headers.get('User-Agent'),
path=request_path,
method=request.method,
querystring=dict(request.GET),
lang=request.headers.get('Accept-Language'),
uid=None,
authn_type=None,
errno=None)
def bump_timestamp(self, collection_timestamp,
record, modified_field, last_modified):
"""Timestamp are base on current millisecond.
.. note ::
Here it is assumed that if requests from the same user burst in,
the time will slide into the future. It is not problematic since
the timestamp notion is opaque, and behaves like a revision number.
"""
is_specified = (record is not None and
modified_field in record or
last_modified is not None)
if is_specified:
# If there is a timestamp in the new record, try to use it.
if last_modified is not None:
current = last_modified
else:
current = record[modified_field]
# If it is equal to current collection timestamp, bump it.
if current == collection_timestamp:
collection_timestamp += 1
current = collection_timestamp
# If it is superior (future), use it as new collection timestamp.
elif current > collection_timestamp:
collection_timestamp = current
# Else (past), do nothing.
else:
# Not specified, use a new one.
current = utils.msec_time()
# If two ops in the same msec, bump it.
if current <= collection_timestamp:
current = collection_timestamp + 1
collection_timestamp = current
return current, collection_timestamp
def ttl(self, key):
ttl = self._ttl.get(self.prefix + key)
if ttl is not None:
return (ttl - msec_time()) / 1000.0
return -1
def deserialize(self, cstruct=colander.null):
if cstruct is colander.null and self.auto_now:
cstruct = msec_time()
return super(TimeStamp, self).deserialize(cstruct)
def timestamp(self):
if not self._entries():
return core_utils.msec_time()
max_value = max([e["last_modified"] for e in self._entries()])
return max_value
def expire(self, key, ttl):
self._ttl[self.prefix + key] = msec_time() + int(ttl * 1000.0)
def ttl(self, key):
ttl = self._ttl.get(self.prefix + key)
if ttl is not None:
return (ttl - msec_time()) / 1000.0
return -1
def _clean_expired(self):
current = msec_time()
expired = [k for k, v in self._ttl.items() if current >= v]
for expired_item_key in expired:
self.delete(expired_item_key[len(self.prefix):])
def timestamp(self):
if not self._entries():
return core_utils.msec_time()
max_value = max([e["last_modified"] for e in self._entries()])
return max_value
def get(self, key):
current = utils.msec_time()
expired = [k for k, v in self._ttl.items() if current >= v]
for expired_item_key in expired:
self.delete(expired_item_key[len(self.prefix):])
return self._store.get(self.prefix + key)
def get(self, key):
current = msec_time()
expired = [k for k, v in self._ttl.items() if current >= v]
for expired_item_key in expired:
self.delete(expired_item_key[len(self.prefix):])
return self._store.get(self.prefix + key)
def _clean_expired(self):
current = msec_time()
expired = [k for k, v in self._ttl.items() if current >= v]
for expired_item_key in expired:
self.delete(expired_item_key[len(self.prefix) :])
def map_country_class(value):
"""
Map a csv list to a dictionary useful to create a country instance
:param value: a list of values
:return: a dictionary where the input values has been mapped to keys
"""
return {'locale': value[1], 'continent_id': value[2], 'code': value[4], 'name': value[5],
'sdate': datetime.datetime(2001, 1, 1), 'parent_id': PARENT_ID, 'last_modified': msec_time()}
def deserialize(self, cstruct=colander.null):
if cstruct is colander.null and self.auto_now:
cstruct = msec_time()
return super(TimeStamp, self).deserialize(cstruct)
def expire(self, key, ttl):
self._ttl[self.prefix + key] = msec_time() + int(ttl * 1000.0)
