def test_utcnow():
' args: as_datetime=False, tz_aware=False '
from metriqueu.utils import utcnow
now_date = dt.utcnow().replace(microsecond=0)
now_date_utc = dt.now(utc).replace(microsecond=0)
now_time = lambda x: int(calendar.timegm(x.utctimetuple()))
# FIXME: millisecond resolution?
assert utcnow(drop_micro=True) == now_time(now_date)
assert utcnow(as_datetime=True, drop_micro=True) == now_date
assert utcnow(tz_aware=True, drop_micro=True) == now_date_utc
def save(self, objects, cube=None, owner=None):
'''
Save a list of objects the given metrique.cube.
Returns back a list of object ids (_id|_oid) saved.
:param list objects: list of dictionary-like objects to be stored
:param string cube: cube name
:param string owner: username of cube owner
:rtype: list - list of object ids saved
'''
batch_size = self.config.batch_size
olen = len(objects) if objects else None
if not olen:
self.logger.info("... No objects to save")
return []
else:
self.logger.info("Saving %s objects" % len(objects))
# get 'now' utc timezone aware datetime object
# FIXME IMPORTANT timestamp should be really taken before extract
now = utcnow(tz_aware=True)
cmd = self.get_cmd(owner, cube, 'save')
if (batch_size <= 0) or (olen <= batch_size):
saved = self._post(cmd, objects=objects, mtime=now)
else:
saved = []
k = 0
for batch in batch_gen(objects, batch_size):
_saved = self._post(cmd, objects=batch, mtime=now)
saved.extend(_saved)
k += len(batch)
self.logger.info("... %i of %i posted" % (k, olen))
if saved:
objects = [o for o in objects if o['_oid'] in saved]
# journal locally as well
if self.config.journal:
# journal objects locally if any were saved
for o in sorted(objects, key=itemgetter('_oid')):
dump = {'owner': self.owner, 'name': self.name,
'when': utcnow(), 'object': o}
ojson = json.dumps(dump, default=json_encode,
ensure_ascii=True,
encoding="ISO-8859-1")
self.journal.debug(ojson)
self.logger.info("... Saved %s NEW docs" % len(saved))
return sorted(saved)
def __init__(self, config_file=None, name=None, **kwargs):
# don't assign to {} in class def, define here to avoid
# multiple pyclient objects linking to a shared dict
if self.defaults is None:
self.defaults = {}
if self.fields is None:
self.fields = {}
if self._cache is None:
self._cache = {}
if self.config is None:
self.config = {}
self._config_file = config_file or Config.default_config
# all defaults are loaded, unless specified in
# metrique_config.json
self.load_config(**kwargs)
# cube class defined name
self._cube = type(self).name
utc_str = utcnow(as_datetime=True).strftime('%a%b%d%H%m%S')
# set name if passed in, but don't overwrite default if not
self.name = name or self.name or utc_str
self.config.logdir = os.path.expanduser(self.config.logdir)
if not os.path.exists(self.config.logdir):
os.makedirs(self.config.logdir)
self.config.logfile = os.path.join(self.config.logdir,
self.config.logfile)
# keep logging local to the cube so multiple
# cubes can independently log without interferring
# with each others logging.
self.debug_setup()
def register(self, owner, cube):
'''
Client registration method
Cube registrations is open access. All registered
users can create cubes, assuming their quota has
not been filled already.
Update the cube_profile with new cube defaults values.
Bump the user's total cube count, by 1
Create default cubes indexes.
:param owner: username of cube owner
:param cube: cube name
'''
# FIXME: take out a lock; to avoid situation
# where client tries to create multiple cubes
# simultaneously and we hit race condition
# where user creates more cubes than has quota
# ie, cube create lock...
if self.cube_exists(owner, cube, raise_if_not=False):
self._raise(409, "cube already exists")
# FIXME: move to remaining = self.check_user_cube_quota(...)
quota, own = self.get_user_profile(owner, keys=['cube_quota',
'own'])
if quota is None or self.is_superuser():
remaining = True
else:
own = len(own) if own else 0
quota = quota or 0
remaining = quota - own
if not remaining or remaining <= 0:
self._raise(409, "quota depleted (%s of %s)" % (quota, own))
now_utc = utcnow()
collection = self.cjoin(owner, cube)
doc = {'_id': collection,
'creater': owner,
'created': now_utc,
'read': [],
'write': [],
'admin': [owner]}
self.cube_profile(admin=True).insert(doc)
# push the collection into the list of ones user owns
self.update_user_profile(owner, 'addToSet', 'own', collection)
# run core index
_cube = self.timeline(owner, cube, admin=True)
# ensure basic indices:
_cube.ensure_index([('_hash', 1), ('_end', 1)])
_cube.ensure_index('_oid')
_cube.ensure_index('_start') # used in core_api.get_cube_last_start
_cube.ensure_index('_end') # default for non-historical queries
return remaining
def _prepare_objects(self, objects, autosnap=True):
'''
Validate and normalize objects.
:param _cube: mongodb cube collection proxy
:param obejcts: list of objects to manipulate
'''
start = utcnow()
_exclude_hash = ['_hash', '_id', '_start', '_end']
_end_types_ok = NoneType if autosnap else (NoneType, float, int)
for i, o in enumerate(objects):
# apply default start and typecheck
o = self._obj_start(o, start)
_start = o.get('_start')
if not isinstance(_start, (float, int)):
self._raise(400, "_start must be float/int epoch")
# and apply default end and typecheck
o = self._obj_end(o)
_end = o.get('_end')
if not isinstance(_end, _end_types_ok):
self._raise(400, "_end must be float/int epoch or None")
# give object a unique, constant (referencable) _id
o = self._obj_id(o)
# _hash is of object contents, excluding _metadata
o = self._obj_hash(o, key='_hash', exclude=_exclude_hash)
objects[i] = o
return objects
def get_metrics(self, names=None):
'''
Run metric SQL defined in the cube.
'''
if isinstance(names, basestring):
names = [names]
start = utcnow()
obj = {
'_start': start,
'_end': None,
}
objects = []
for metric, definition in self.metrics.items():
if names and metric not in names:
continue
sql = definition['sql']
fields = definition['fields']
_oid = definition['_oid']
rows = self.proxy.fetchall(sql)
for row in rows:
d = copy(obj)
# derive _oid from metric name and row contents
d['_oid'] = _oid(metric, row)
for i, element in enumerate(row):
d[fields[i]] = element
# append to the local metric result list
objects.append(d)
objects = self.normalize(objects)
return objects
def _prepare_objects(self, _cube, objects, autosnap=True):
'''
Validate and normalize objects.
:param _cube: mongodb cube collection proxy
:param obejcts: list of objects to manipulate
'''
start = utcnow()
_exclude_hash = ['_hash', '_id', '_start', '_end']
_end_types_ok = NoneType if autosnap else (NoneType, float, int)
for o in iter(objects):
o = self._obj_end(o)
_end = o.get('_end')
if not isinstance(_end, _end_types_ok):
self._raise(400, "_end must be float/int epoch or None")
o = self._obj_start(o, start)
_start = o.get('_start')
if not isinstance(_start, (float, int)):
self._raise(400, "_start must be defined, as float/int epoch")
_oid = o.get('_oid')
# give object a unique, constant (referencable) _id
o['_id'] = self._id_hash_gen(_oid, _start, _end)
# _hash is of object contents, excluding metadata
o = self._obj_hash(o, key='_hash', exclude=_exclude_hash)
return objects
def get_objects(self, force=None, last_update=None, parse_timestamp=None,
save=False, autosnap=True):
'''
Extract routine for SQL based cubes.
:param force:
for querying for all objects (True) or only those passed in as list
:param last_update: manual override for 'changed since date'
:param parse_timestamp: flag to convert timestamp timezones in-line
'''
logger.debug('Fetching Objects - Current Values')
objects = []
start = utcnow()
# determine which oids will we query
oids = self._delta_force(force, last_update, parse_timestamp)
# set the 'index' of sql columns so we can extract
# out the sql rows and know which column : field
field_order = tuple(self.fields)
max_workers = self.config.max_workers
batch_size = self.config.sql_batch_size
if max_workers > 1:
with ProcessPoolExecutor(max_workers=max_workers) as ex:
futures = []
kwargs = self.config
kwargs.pop('cube', None) # if in self.config, ignore it
for batch in batch_gen(oids, batch_size):
f = ex.submit(get_objects, cube=self._cube, oids=batch,
field_order=field_order, start=start,
save=save, cube_name=self.name,
autosnap=autosnap, **kwargs)
futures.append(f)
objects = []
for future in as_completed(futures):
try:
objs = future.result()
objects.extend(objs)
except Exception as e:
tb = traceback.format_exc()
logger.error('Extract Error: %s\n%s' % (e, tb))
del tb, e
else:
# respect the global batch size, even if sql batch
# size is not set
for batch in batch_gen(oids, batch_size):
objs = self._get_objects(oids=batch, field_order=field_order,
start=start, save=save,
autosnap=autosnap)
objects.extend(objs)
logger.debug('... current values objects get - done')
return objects
def register(self, owner, cube):
'''
Client registration method
Default behaivor (not currently overridable) is to permit
cube registrations by all registered users.
Update the user__cube __meta__ doc with defaults
Bump the user's total cube count, by 1
'''
# FIXME: take out a lock; to avoid situation
# where client tries to create multiple cubes
# simultaneously and we hit race condition
# where user creates more cubes than has quota
if self.cube_exists(owner, cube, raise_if_not=False):
self._raise(409, "cube already exists")
# FIXME: move to remaining = self.check_user_cube_quota(...)
quota, own = self.get_user_profile(owner, keys=['_cube_quota',
'_own'])
if quota is None:
remaining = True
else:
own = len(own) if own else 0
quota = quota or 0
remaining = quota - own
if not remaining or remaining <= 0:
self._raise(409, "quota depleted (%s of %s)" % (quota, own))
now_utc = utcnow()
collection = self.cjoin(owner, cube)
doc = {'_id': collection,
'owner': owner,
'created': now_utc,
'read': [],
'write': [],
'admin': []}
self.cube_profile(admin=True).insert(doc)
# push the collection into the list of ones user owns
self.update_user_profile(owner, 'addToSet', 'own', collection)
# run core index
_cube = self.timeline(owner, cube, admin=True)
# ensure basic indices:
_cube.ensure_index('_hash')
_cube.ensure_index('_oid')
_cube.ensure_index('_start') # used in core_api.get_cube_last_start
return remaining
def ping(self, auth=None):
user = self.current_user
if auth and not user:
self._raise(401, "authentication required")
else:
logger.debug(
'got ping from %s @ %s' % (user, utcnow(as_datetime=True)))
response = {
'action': 'ping',
'response': 'pong',
'current_user': user,
}
return response
def _normalize(self, objects):
"""
give all these objects the same _start value (if they
don't already have one), and more...
"""
start = utcnow()
for i, o in enumerate(objects):
# normalize fields (alphanumeric characters only, lowercase)
o = self._obj_fields(o)
# convert empty strings to None (null)
o = self._obj_nones(o)
# add object meta data the metriqued requires be set per object
o = self._obj_end(o)
o = self._obj_start(o, start)
objects[i] = o
return objects
def ping(self, auth=None):
'''
Simple ping/pong. Returns back some basic details
of the host:app which caught the ping.
:param auth: flag to force authentication
'''
user = self.current_user
if auth and not user:
self._raise(401, "authentication required")
else:
logger.debug(
'got ping from %s @ %s' % (user, utcnow(as_datetime=True)))
response = {
'action': 'ping',
'current_user': user,
'metriqued': HOSTNAME,
}
return response
def register(self, username, password=None, null_password_ok=False):
if INVALID_USERNAME_RE.search(username):
self._raise(400,
"Invalid username; ascii alpha [a-z] characters only!")
username = username.lower()
if self.user_exists(username):
self._raise(409, "user exists")
passhash = sha256_crypt.encrypt(password) if password else None
if not (passhash or null_password_ok):
self._raise(400, "[%s] no password provided" % username)
doc = {'_id': username,
'_ctime': utcnow(),
'own': [],
'read': [],
'write': [],
'admin': [],
'_cube_quota': CUBE_QUOTA,
'_passhash': passhash,
}
self.user_profile(admin=True).save(doc, upset=True, safe=True)
logger.debug("new user added (%s)" % (username))
return True
def register(self, username, password=None, null_password_ok=False):
'''
Register a given username, if available.
Username's must be ascii alpha characters only (a-z).
Usernames are automatically normalized in the following ways:
* lowercased
:param username: username to register
:param password: password to register for username
:param null_password_ok: flag for whether empty password is ok (krb5)
'''
# FIXME: add a 'cube registration' lock
if INVALID_USERNAME_RE.search(username):
self._raise(
400, "Invalid username; ascii alpha [a-z] characters only!")
username = username.lower()
if self.user_exists(username, raise_if_not=False):
self._raise(409, "[%s] user exists" % username)
passhash = sha256_crypt.encrypt(password) if password else None
if not (passhash or null_password_ok):
self._raise(400, "[%s] no password provided" % username)
cube_quota = self.metrique_config.user_cube_quota
doc = {'_id': username,
'_ctime': utcnow(),
'own': [],
'read': [],
'write': [],
'admin': [],
'cube_quota': cube_quota,
'_passhash': passhash,
}
self.user_profile(admin=True).save(doc, safe=True)
logger.info("new user added (%s)" % (username))
return True
def save_objects(self, owner, cube, objects, mtime=None):
'''
:param str owner: target owner's cube
:param str cube: target cube (collection) to save objects to
:param list objects: list of dictionary-like objects to be stored
:param datetime mtime: datetime to apply as mtime for objects
:rtype: list - list of object ids saved
Get a list of dictionary objects from client and insert
or save them to the timeline.
Apply the given mtime to all objects or apply utcnow(). _mtime
is used to support timebased 'delta' updates.
'''
self.cube_exists(owner, cube)
self.requires_owner_write(owner, cube)
mtime = dt2ts(mtime) if mtime else utcnow()
current_mtime = self.get_cube_last_start(owner, cube)
if current_mtime and mtime and current_mtime > mtime:
# don't fail, but make sure the issue is logged!
# likely, a ntp time sync is required
logger.warn(
"object mtime is < server mtime; %s < %s; " % (mtime,
current_mtime))
_cube = self.timeline(owner, cube, admin=True)
olen_r = len(objects)
logger.debug('[%s.%s] Recieved %s objects' % (owner, cube, olen_r))
objects = self.prepare_objects(_cube, objects, mtime)
logger.debug('[%s.%s] %s objects match their current version in db' % (
owner, cube, olen_r - len(objects)))
if not objects:
logger.debug('[%s.%s] No NEW objects to save' % (owner, cube))
return []
else:
logger.debug('[%s.%s] Saving %s objects' % (owner, cube,
len(objects)))
# End the most recent versions in the db of those objects that
# have newer versionsi (newest version must have _end == None,
# activity import saves objects for which this might not be true):
to_snap = dict([(o['_oid'], o['_start']) for o in objects
if o['_end'] is None])
if to_snap:
db_versions = _cube.find({'_oid': {'$in': to_snap.keys()},
'_end': None},
fields={'_id': 1, '_oid': 1})
snapped = 0
for doc in db_versions:
_cube.update({'_id': doc['_id']},
{'$set': {'_end': to_snap[doc['_oid']]}},
multi=False)
snapped += 1
logger.debug('[%s.%s] Updated %s OLD versions' %
(owner, cube, snapped))
# Insert all new versions:
insert_bulk(_cube, objects)
logger.debug('[%s.%s] Saved %s NEW versions' % (owner, cube,
len(objects)))
# return object ids saved
return [o['_oid'] for o in objects]
def _obj_start(self, obj, default=None):
_start = obj.get("_start", default)
obj["_start"] = _start or utcnow()
return obj
def _obj_start(self, obj, default=None):
_start = obj.get('_start', default)
obj['_start'] = _start or utcnow()
return obj
