def test_the_timestamp_are_based_on_real_time_milliseconds(self): before = utils.msec_time() time.sleep(0.002) # 2 msec record = self.create_record() now = record['last_modified'] time.sleep(0.002) # 2 msec after = utils.msec_time() self.assertTrue(before < now < after, '{} < {} < {}'.format(before, now, after))
def on_new_response(event): response = event.response request = event.request # Compute the request processing time in msec (-1 if unknown) current = utils.msec_time() duration = current - getattr(request, '_received_at', current - 1) isotimestamp = datetime.fromtimestamp(current/1000).isoformat() # Bind infos for request summary logger. request.log_context(time=isotimestamp, code=response.status_code, t=duration) if summary_logger.level == logging.DEBUG: request.log_context(response=dict(headers=dict(response.headers), body=response.body)) try: # If error response, bind errno. request.log_context(errno=response.errno) except AttributeError: pass if not hasattr(request, 'parent'): # Ouput application request summary. summary_logger.info('', extra=request.log_context())
def bump_timestamp(self, collection_timestamp, record, modified_field, last_modified): """Timestamp are base on current millisecond. .. note :: Here it is assumed that if requests from the same user burst in, the time will slide into the future. It is not problematic since the timestamp notion is opaque, and behaves like a revision number. """ is_specified = record is not None and modified_field in record or last_modified is not None if is_specified: # If there is a timestamp in the new record, try to use it. if last_modified is not None: current = last_modified else: current = record[modified_field] # If it is equal to current collection timestamp, bump it. if current == collection_timestamp: collection_timestamp += 1 current = collection_timestamp # If it is superior (future), use it as new collection timestamp. elif current > collection_timestamp: collection_timestamp = current # Else (past), do nothing. else: # Not specified, use a new one. current = utils.msec_time() # If two ops in the same msec, bump it. if current <= collection_timestamp: current = collection_timestamp + 1 collection_timestamp = current return current, collection_timestamp
def on_new_request(event): request = event.request # Save the time the request was received by the server. event.request._received_at = utils.msec_time() try: # Pyramid fails if the URL contains invalid UTF-8 characters. request_path = event.request.path except UnicodeDecodeError: raise errors.http_error( HTTPBadRequest(), errno=errors.ERRORS.INVALID_PARAMETERS, message="Invalid URL path.", ) request.log_context( agent=request.headers.get("User-Agent"), path=request_path, method=request.method, lang=request.headers.get("Accept-Language"), errno=0, ) qs = dict(errors.request_GET(request)) if qs: request.log_context(querystring=qs) if summary_logger.level == logging.DEBUG: request.log_context(headers=dict(request.headers), body=request.body)
def on_new_request(event): request = event.request # Save the time the request was received by the server. event.request._received_at = utils.msec_time() try: # Pyramid fails if the URL contains invalid UTF-8 characters. request_path = event.request.path except UnicodeDecodeError: raise errors.http_error( HTTPBadRequest(), errno=errors.ERRORS.INVALID_PARAMETERS, message='Invalid URL path.') request.log_context(agent=request.headers.get('User-Agent'), path=request_path, method=request.method, lang=request.headers.get('Accept-Language'), errno=0) qs = dict(errors.request_GET(request)) if qs: request.log_context(querystring=qs) if summary_logger.level == logging.DEBUG: request.log_context(headers=dict(request.headers), body=request.body)
def on_new_response(event): response = event.response request = event.request # Compute the request processing time in msec (-1 if unknown) current = utils.msec_time() duration = current - getattr(request, "_received_at", current - 1) isotimestamp = datetime.fromtimestamp(current / 1000).isoformat() # Bind infos for request summary logger. request.log_context(time=isotimestamp, code=response.status_code, t=duration) if summary_logger.level == logging.DEBUG: request.log_context(response=dict(headers=dict(response.headers), body=response.body)) try: # If error response, bind errno. request.log_context(errno=response.errno) except AttributeError: pass if not hasattr(request, "parent"): # Ouput application request summary. summary_logger.info("", extra=request.log_context())
def accumulate_continents(value): """ Pick continents from countries and populate a list of dictionaries with unique values :param value: country to inspect """ if value[2] not in [i.code for i in continents]: continents.append(Continent(**{'code': value[2], 'name': value[3], 'sdate': datetime.datetime(2001, 1, 1), 'parent_id': PARENT_ID, 'last_modified': msec_time()}))
def set(self, key, value, ttl): if isinstance(value, bytes): raise TypeError("a string-like object is required, not 'bytes'") self._clean_expired() self._clean_oversized() self.expire(key, ttl) item_key = self.prefix + key self._store[item_key] = value self._created_at[item_key] = msec_time() self._quota += size_of(item_key, value)
def set(self, key, value, ttl=None): self._clean_expired() self._clean_oversized() if ttl is not None: self.expire(key, ttl) else: logger.warning("No TTL for cache key '{}'".format(key)) item_key = self.prefix + key self._store[item_key] = value self._created_at[item_key] = msec_time() self._quota += size_of(item_key, value)
def map_city_class(value): """ Map a csv list to a dictionary useful to create a city instance :param value: a list of values :return: a dictionary where the input values has been mapped to keys """ return {'country_id': get_country_id(value[4]), 'sd_1_iso_code': value[6], 'sd_1_name': value[7], 'sd_2_iso_code': value[8], 'sd_2_name': value[9], 'name': value[10], 'metro_code': value[11], 'time_zone': value[12], 'parent_id': PARENT_ID, 'last_modified': msec_time()}
def on_new_request(event): request = event.request # Save the time the request was received by the server. event.request._received_at = utils.msec_time() # New logger context, with infos for request summary logger. logger.new(agent=request.headers.get('User-Agent'), path=event.request.path, method=request.method, querystring=dict(request.GET), lang=request.headers.get('Accept-Language'), uid=None, authn_type=None, errno=None)
def on_new_response(event): response = event.response request = event.request # Compute the request processing time in msec (-1 if unknown) current = utils.msec_time() duration = current - getattr(request, '_received_at', current - 1) isotimestamp = datetime.fromtimestamp(current / 1000).isoformat() # Bind infos for request summary logger. logger.bind(time=isotimestamp, code=response.status_code, t=duration) # Ouput application request summary. if not hasattr(request, 'parent'): logger.info('request.summary')
def _bump_timestamp(self, collection_id, parent_id, record=None, modified_field=None, last_modified=None): key = '{0}.{1}.timestamp'.format(collection_id, parent_id) while 1: with self._client.pipeline() as pipe: try: pipe.watch(key) previous = pipe.get(key) pipe.multi() # XXX factorize code from memory and redis backends. is_specified = (record is not None and modified_field in record or last_modified is not None) if is_specified: # If there is a timestamp in the new record, # try to use it. if last_modified is not None: current = last_modified else: current = record[modified_field] else: current = utils.msec_time() if previous and int(previous) >= current: collection_timestamp = int(previous) + 1 else: collection_timestamp = current # Return the newly generated timestamp as the current one # only if nothing else was specified. if not is_specified: current = collection_timestamp pipe.set(key, collection_timestamp) pipe.execute() return current except redis.WatchError: # pragma: no cover # Our timestamp has been modified by someone else, let's # retry. # XXX: untested. continue
def on_new_response(event): response = event.response request = event.request # Compute the request processing time in msec (-1 if unknown) current = utils.msec_time() duration = current - getattr(request, '_received_at', current - 1) isotimestamp = datetime.fromtimestamp(current/1000).isoformat() # Bind infos for request summary logger. logger.bind(time=isotimestamp, code=response.status_code, t=duration) # Ouput application request summary. if not hasattr(request, 'parent'): logger.info('request.summary')
def __call__(self, logger, name, event_dict): SYSLOG_LEVELS = { 'critical': 0, 'fatal': 0, 'exception': 2, 'error': 2, 'warning': 4, 'info': 6, 'debug': 7, } severity = SYSLOG_LEVELS[name] MSEC_TO_NANOSEC = 1000000 timestamp = utils.msec_time() * MSEC_TO_NANOSEC event = event_dict.pop('event', '') defaults = { 'Timestamp': timestamp, 'Logger': self.appname, 'Type': event, 'Hostname': self.hostname, 'Severity': severity, 'Pid': self.pid, 'EnvVersion': self.ENV_VERSION, 'Fields': {} } for f, v in defaults.items(): event_dict.setdefault(f, v) fields = [k for k in event_dict.keys() if k not in defaults] for f in fields: value = event_dict.pop(f) # Heka relies on Protobuf, which doesn't support recursive objects. if isinstance(value, dict): value = utils.json.dumps(value) elif isinstance(value, (list, tuple)): if not all([isinstance(i, str) for i in value]): value = utils.json.dumps(value) event_dict['Fields'][f] = value return utils.json.dumps(event_dict)
def __call__(self, logger, name, event_dict): SYSLOG_LEVELS = { 'critical': 0, 'fatal': 0, 'exception': 2, 'error': 2, 'warning': 4, 'info': 6, 'debug': 7, } severity = SYSLOG_LEVELS[name] MSEC_TO_NANOSEC = 1000000 timestamp = utils.msec_time() * MSEC_TO_NANOSEC event = event_dict.pop('event', '') defaults = { 'Timestamp': timestamp, 'Logger': self.appname, 'Type': event, 'Hostname': self.hostname, 'Severity': severity, 'Pid': self.pid, 'EnvVersion': self.ENV_VERSION, 'Fields': {} } for f, v in defaults.items(): event_dict.setdefault(f, v) fields = [k for k in event_dict.keys() if k not in defaults] for f in fields: value = event_dict.pop(f) # Heka relies on Protobuf, which doesn't support recursive objects. if isinstance(value, dict): value = utils.json.dumps(value) elif isinstance(value, (list, tuple)): if not all([isinstance(i, six.string_types) for i in value]): value = utils.json.dumps(value) event_dict['Fields'][f] = value return utils.json.dumps(event_dict)
def _bump_timestamp(self, collection_id, parent_id, record=None, modified_field=None, last_modified=None): key = '{0}.{1}.timestamp'.format(collection_id, parent_id) while 1: with self._client.pipeline() as pipe: try: pipe.watch(key) previous = pipe.get(key) pipe.multi() # XXX factorize code from memory and redis backends. is_specified = (record is not None and modified_field in record or last_modified is not None) if is_specified: # If there is a timestamp in the new record, # try to use it. if last_modified is not None: current = last_modified else: current = record[modified_field] else: current = utils.msec_time() if previous and int(previous) >= current: collection_timestamp = int(previous) + 1 else: collection_timestamp = current # Return the newly generated timestamp as the current one # only if nothing else was specified. is_equal = previous and int(previous) == current if not is_specified or is_equal: current = collection_timestamp pipe.set(key, collection_timestamp) pipe.execute() return current except redis.WatchError: # pragma: no cover # Our timestamp has been modified by someone else, let's # retry. # XXX: untested. continue
def _bump_timestamp(self, collection_id, parent_id, record=None, modified_field=None, last_modified=None): """Timestamp are base on current millisecond. .. note :: Here it is assumed that if requests from the same user burst in, the time will slide into the future. It is not problematic since the timestamp notion is opaque, and behaves like a revision number. """ # XXX factorize code from memory and redis backends. is_specified = (record is not None and modified_field in record or last_modified is not None) if is_specified: # If there is a timestamp in the new record, try to use it. if last_modified is not None: current = last_modified else: current = record[modified_field] else: # Otherwise, use a new one. current = utils.msec_time() # Bump the timestamp only if it's more than the previous one. previous = self._timestamps[parent_id].get(collection_id) if previous and previous >= current: collection_timestamp = previous + 1 else: collection_timestamp = current # In case the timestamp was specified, the collection timestamp will # be different from the updated timestamp. As such, we want to return # the one of the record, and not the collection one. if not is_specified or previous == current: current = collection_timestamp self._timestamps[parent_id][collection_id] = collection_timestamp return current
def on_new_request(event): request = event.request # Save the time the request was received by the server. event.request._received_at = utils.msec_time() try: # Pyramid fails if the URL contains invalid UTF-8 characters. request_path = event.request.path except UnicodeDecodeError: raise errors.http_error(HTTPBadRequest(), errno=errors.ERRORS.INVALID_PARAMETERS, message='Invalid URL path.') request.log_context(agent=request.headers.get('User-Agent'), path=request_path, method=request.method, querystring=dict(errors.request_GET(request)), lang=request.headers.get('Accept-Language'), uid=None, authn_type=None, errno=None)
def on_new_request(event): request = event.request # Save the time the request was received by the server. event.request._received_at = utils.msec_time() try: # Pyramid fails if the URL contains invalid UTF-8 characters. request_path = event.request.path except UnicodeDecodeError: raise errors.http_error( HTTPBadRequest(), errno=errors.ERRORS.INVALID_PARAMETERS, message="Invalid URL path.") # New logger context, with infos for request summary logger. logger.new(agent=request.headers.get('User-Agent'), path=request_path, method=request.method, querystring=dict(request.GET), lang=request.headers.get('Accept-Language'), uid=None, authn_type=None, errno=None)
def bump_timestamp(self, collection_timestamp, record, modified_field, last_modified): """Timestamp are base on current millisecond. .. note :: Here it is assumed that if requests from the same user burst in, the time will slide into the future. It is not problematic since the timestamp notion is opaque, and behaves like a revision number. """ is_specified = (record is not None and modified_field in record or last_modified is not None) if is_specified: # If there is a timestamp in the new record, try to use it. if last_modified is not None: current = last_modified else: current = record[modified_field] # If it is equal to current collection timestamp, bump it. if current == collection_timestamp: collection_timestamp += 1 current = collection_timestamp # If it is superior (future), use it as new collection timestamp. elif current > collection_timestamp: collection_timestamp = current # Else (past), do nothing. else: # Not specified, use a new one. current = utils.msec_time() # If two ops in the same msec, bump it. if current <= collection_timestamp: current = collection_timestamp + 1 collection_timestamp = current return current, collection_timestamp
def ttl(self, key): ttl = self._ttl.get(self.prefix + key) if ttl is not None: return (ttl - msec_time()) / 1000.0 return -1
def deserialize(self, cstruct=colander.null): if cstruct is colander.null and self.auto_now: cstruct = msec_time() return super(TimeStamp, self).deserialize(cstruct)
def timestamp(self): if not self._entries(): return core_utils.msec_time() max_value = max([e["last_modified"] for e in self._entries()]) return max_value
def expire(self, key, ttl): self._ttl[self.prefix + key] = msec_time() + int(ttl * 1000.0)
def _clean_expired(self): current = msec_time() expired = [k for k, v in self._ttl.items() if current >= v] for expired_item_key in expired: self.delete(expired_item_key[len(self.prefix):])
def get(self, key): current = utils.msec_time() expired = [k for k, v in self._ttl.items() if current >= v] for expired_item_key in expired: self.delete(expired_item_key[len(self.prefix):]) return self._store.get(self.prefix + key)
def get(self, key): current = msec_time() expired = [k for k, v in self._ttl.items() if current >= v] for expired_item_key in expired: self.delete(expired_item_key[len(self.prefix):]) return self._store.get(self.prefix + key)
def _clean_expired(self): current = msec_time() expired = [k for k, v in self._ttl.items() if current >= v] for expired_item_key in expired: self.delete(expired_item_key[len(self.prefix) :])
def map_country_class(value): """ Map a csv list to a dictionary useful to create a country instance :param value: a list of values :return: a dictionary where the input values has been mapped to keys """ return {'locale': value[1], 'continent_id': value[2], 'code': value[4], 'name': value[5], 'sdate': datetime.datetime(2001, 1, 1), 'parent_id': PARENT_ID, 'last_modified': msec_time()}