def test_utcnow(): ' args: as_datetime=False, tz_aware=False ' from metriqueu.utils import utcnow now_date = dt.utcnow().replace(microsecond=0) now_date_utc = dt.now(utc).replace(microsecond=0) now_time = lambda x: int(calendar.timegm(x.utctimetuple())) # FIXME: millisecond resolution? assert utcnow(drop_micro=True) == now_time(now_date) assert utcnow(as_datetime=True, drop_micro=True) == now_date assert utcnow(tz_aware=True, drop_micro=True) == now_date_utc
def save(self, objects, cube=None, owner=None): ''' Save a list of objects the given metrique.cube. Returns back a list of object ids (_id|_oid) saved. :param list objects: list of dictionary-like objects to be stored :param string cube: cube name :param string owner: username of cube owner :rtype: list - list of object ids saved ''' batch_size = self.config.batch_size olen = len(objects) if objects else None if not olen: self.logger.info("... No objects to save") return [] else: self.logger.info("Saving %s objects" % len(objects)) # get 'now' utc timezone aware datetime object # FIXME IMPORTANT timestamp should be really taken before extract now = utcnow(tz_aware=True) cmd = self.get_cmd(owner, cube, 'save') if (batch_size <= 0) or (olen <= batch_size): saved = self._post(cmd, objects=objects, mtime=now) else: saved = [] k = 0 for batch in batch_gen(objects, batch_size): _saved = self._post(cmd, objects=batch, mtime=now) saved.extend(_saved) k += len(batch) self.logger.info("... %i of %i posted" % (k, olen)) if saved: objects = [o for o in objects if o['_oid'] in saved] # journal locally as well if self.config.journal: # journal objects locally if any were saved for o in sorted(objects, key=itemgetter('_oid')): dump = {'owner': self.owner, 'name': self.name, 'when': utcnow(), 'object': o} ojson = json.dumps(dump, default=json_encode, ensure_ascii=True, encoding="ISO-8859-1") self.journal.debug(ojson) self.logger.info("... Saved %s NEW docs" % len(saved)) return sorted(saved)
def __init__(self, config_file=None, name=None, **kwargs): # don't assign to {} in class def, define here to avoid # multiple pyclient objects linking to a shared dict if self.defaults is None: self.defaults = {} if self.fields is None: self.fields = {} if self._cache is None: self._cache = {} if self.config is None: self.config = {} self._config_file = config_file or Config.default_config # all defaults are loaded, unless specified in # metrique_config.json self.load_config(**kwargs) # cube class defined name self._cube = type(self).name utc_str = utcnow(as_datetime=True).strftime('%a%b%d%H%m%S') # set name if passed in, but don't overwrite default if not self.name = name or self.name or utc_str self.config.logdir = os.path.expanduser(self.config.logdir) if not os.path.exists(self.config.logdir): os.makedirs(self.config.logdir) self.config.logfile = os.path.join(self.config.logdir, self.config.logfile) # keep logging local to the cube so multiple # cubes can independently log without interferring # with each others logging. self.debug_setup()
def register(self, owner, cube): ''' Client registration method Cube registrations is open access. All registered users can create cubes, assuming their quota has not been filled already. Update the cube_profile with new cube defaults values. Bump the user's total cube count, by 1 Create default cubes indexes. :param owner: username of cube owner :param cube: cube name ''' # FIXME: take out a lock; to avoid situation # where client tries to create multiple cubes # simultaneously and we hit race condition # where user creates more cubes than has quota # ie, cube create lock... if self.cube_exists(owner, cube, raise_if_not=False): self._raise(409, "cube already exists") # FIXME: move to remaining = self.check_user_cube_quota(...) quota, own = self.get_user_profile(owner, keys=['cube_quota', 'own']) if quota is None or self.is_superuser(): remaining = True else: own = len(own) if own else 0 quota = quota or 0 remaining = quota - own if not remaining or remaining <= 0: self._raise(409, "quota depleted (%s of %s)" % (quota, own)) now_utc = utcnow() collection = self.cjoin(owner, cube) doc = {'_id': collection, 'creater': owner, 'created': now_utc, 'read': [], 'write': [], 'admin': [owner]} self.cube_profile(admin=True).insert(doc) # push the collection into the list of ones user owns self.update_user_profile(owner, 'addToSet', 'own', collection) # run core index _cube = self.timeline(owner, cube, admin=True) # ensure basic indices: _cube.ensure_index([('_hash', 1), ('_end', 1)]) _cube.ensure_index('_oid') _cube.ensure_index('_start') # used in core_api.get_cube_last_start _cube.ensure_index('_end') # default for non-historical queries return remaining
def _prepare_objects(self, objects, autosnap=True): ''' Validate and normalize objects. :param _cube: mongodb cube collection proxy :param obejcts: list of objects to manipulate ''' start = utcnow() _exclude_hash = ['_hash', '_id', '_start', '_end'] _end_types_ok = NoneType if autosnap else (NoneType, float, int) for i, o in enumerate(objects): # apply default start and typecheck o = self._obj_start(o, start) _start = o.get('_start') if not isinstance(_start, (float, int)): self._raise(400, "_start must be float/int epoch") # and apply default end and typecheck o = self._obj_end(o) _end = o.get('_end') if not isinstance(_end, _end_types_ok): self._raise(400, "_end must be float/int epoch or None") # give object a unique, constant (referencable) _id o = self._obj_id(o) # _hash is of object contents, excluding _metadata o = self._obj_hash(o, key='_hash', exclude=_exclude_hash) objects[i] = o return objects
def get_metrics(self, names=None): ''' Run metric SQL defined in the cube. ''' if isinstance(names, basestring): names = [names] start = utcnow() obj = { '_start': start, '_end': None, } objects = [] for metric, definition in self.metrics.items(): if names and metric not in names: continue sql = definition['sql'] fields = definition['fields'] _oid = definition['_oid'] rows = self.proxy.fetchall(sql) for row in rows: d = copy(obj) # derive _oid from metric name and row contents d['_oid'] = _oid(metric, row) for i, element in enumerate(row): d[fields[i]] = element # append to the local metric result list objects.append(d) objects = self.normalize(objects) return objects
def _prepare_objects(self, _cube, objects, autosnap=True): ''' Validate and normalize objects. :param _cube: mongodb cube collection proxy :param obejcts: list of objects to manipulate ''' start = utcnow() _exclude_hash = ['_hash', '_id', '_start', '_end'] _end_types_ok = NoneType if autosnap else (NoneType, float, int) for o in iter(objects): o = self._obj_end(o) _end = o.get('_end') if not isinstance(_end, _end_types_ok): self._raise(400, "_end must be float/int epoch or None") o = self._obj_start(o, start) _start = o.get('_start') if not isinstance(_start, (float, int)): self._raise(400, "_start must be defined, as float/int epoch") _oid = o.get('_oid') # give object a unique, constant (referencable) _id o['_id'] = self._id_hash_gen(_oid, _start, _end) # _hash is of object contents, excluding metadata o = self._obj_hash(o, key='_hash', exclude=_exclude_hash) return objects
def get_objects(self, force=None, last_update=None, parse_timestamp=None, save=False, autosnap=True): ''' Extract routine for SQL based cubes. :param force: for querying for all objects (True) or only those passed in as list :param last_update: manual override for 'changed since date' :param parse_timestamp: flag to convert timestamp timezones in-line ''' logger.debug('Fetching Objects - Current Values') objects = [] start = utcnow() # determine which oids will we query oids = self._delta_force(force, last_update, parse_timestamp) # set the 'index' of sql columns so we can extract # out the sql rows and know which column : field field_order = tuple(self.fields) max_workers = self.config.max_workers batch_size = self.config.sql_batch_size if max_workers > 1: with ProcessPoolExecutor(max_workers=max_workers) as ex: futures = [] kwargs = self.config kwargs.pop('cube', None) # if in self.config, ignore it for batch in batch_gen(oids, batch_size): f = ex.submit(get_objects, cube=self._cube, oids=batch, field_order=field_order, start=start, save=save, cube_name=self.name, autosnap=autosnap, **kwargs) futures.append(f) objects = [] for future in as_completed(futures): try: objs = future.result() objects.extend(objs) except Exception as e: tb = traceback.format_exc() logger.error('Extract Error: %s\n%s' % (e, tb)) del tb, e else: # respect the global batch size, even if sql batch # size is not set for batch in batch_gen(oids, batch_size): objs = self._get_objects(oids=batch, field_order=field_order, start=start, save=save, autosnap=autosnap) objects.extend(objs) logger.debug('... current values objects get - done') return objects
def register(self, owner, cube): ''' Client registration method Default behaivor (not currently overridable) is to permit cube registrations by all registered users. Update the user__cube __meta__ doc with defaults Bump the user's total cube count, by 1 ''' # FIXME: take out a lock; to avoid situation # where client tries to create multiple cubes # simultaneously and we hit race condition # where user creates more cubes than has quota if self.cube_exists(owner, cube, raise_if_not=False): self._raise(409, "cube already exists") # FIXME: move to remaining = self.check_user_cube_quota(...) quota, own = self.get_user_profile(owner, keys=['_cube_quota', '_own']) if quota is None: remaining = True else: own = len(own) if own else 0 quota = quota or 0 remaining = quota - own if not remaining or remaining <= 0: self._raise(409, "quota depleted (%s of %s)" % (quota, own)) now_utc = utcnow() collection = self.cjoin(owner, cube) doc = {'_id': collection, 'owner': owner, 'created': now_utc, 'read': [], 'write': [], 'admin': []} self.cube_profile(admin=True).insert(doc) # push the collection into the list of ones user owns self.update_user_profile(owner, 'addToSet', 'own', collection) # run core index _cube = self.timeline(owner, cube, admin=True) # ensure basic indices: _cube.ensure_index('_hash') _cube.ensure_index('_oid') _cube.ensure_index('_start') # used in core_api.get_cube_last_start return remaining
def ping(self, auth=None): user = self.current_user if auth and not user: self._raise(401, "authentication required") else: logger.debug( 'got ping from %s @ %s' % (user, utcnow(as_datetime=True))) response = { 'action': 'ping', 'response': 'pong', 'current_user': user, } return response
def _normalize(self, objects): """ give all these objects the same _start value (if they don't already have one), and more... """ start = utcnow() for i, o in enumerate(objects): # normalize fields (alphanumeric characters only, lowercase) o = self._obj_fields(o) # convert empty strings to None (null) o = self._obj_nones(o) # add object meta data the metriqued requires be set per object o = self._obj_end(o) o = self._obj_start(o, start) objects[i] = o return objects
def ping(self, auth=None): ''' Simple ping/pong. Returns back some basic details of the host:app which caught the ping. :param auth: flag to force authentication ''' user = self.current_user if auth and not user: self._raise(401, "authentication required") else: logger.debug( 'got ping from %s @ %s' % (user, utcnow(as_datetime=True))) response = { 'action': 'ping', 'current_user': user, 'metriqued': HOSTNAME, } return response
def register(self, username, password=None, null_password_ok=False): if INVALID_USERNAME_RE.search(username): self._raise(400, "Invalid username; ascii alpha [a-z] characters only!") username = username.lower() if self.user_exists(username): self._raise(409, "user exists") passhash = sha256_crypt.encrypt(password) if password else None if not (passhash or null_password_ok): self._raise(400, "[%s] no password provided" % username) doc = {'_id': username, '_ctime': utcnow(), 'own': [], 'read': [], 'write': [], 'admin': [], '_cube_quota': CUBE_QUOTA, '_passhash': passhash, } self.user_profile(admin=True).save(doc, upset=True, safe=True) logger.debug("new user added (%s)" % (username)) return True
def register(self, username, password=None, null_password_ok=False): ''' Register a given username, if available. Username's must be ascii alpha characters only (a-z). Usernames are automatically normalized in the following ways: * lowercased :param username: username to register :param password: password to register for username :param null_password_ok: flag for whether empty password is ok (krb5) ''' # FIXME: add a 'cube registration' lock if INVALID_USERNAME_RE.search(username): self._raise( 400, "Invalid username; ascii alpha [a-z] characters only!") username = username.lower() if self.user_exists(username, raise_if_not=False): self._raise(409, "[%s] user exists" % username) passhash = sha256_crypt.encrypt(password) if password else None if not (passhash or null_password_ok): self._raise(400, "[%s] no password provided" % username) cube_quota = self.metrique_config.user_cube_quota doc = {'_id': username, '_ctime': utcnow(), 'own': [], 'read': [], 'write': [], 'admin': [], 'cube_quota': cube_quota, '_passhash': passhash, } self.user_profile(admin=True).save(doc, safe=True) logger.info("new user added (%s)" % (username)) return True
def save_objects(self, owner, cube, objects, mtime=None): ''' :param str owner: target owner's cube :param str cube: target cube (collection) to save objects to :param list objects: list of dictionary-like objects to be stored :param datetime mtime: datetime to apply as mtime for objects :rtype: list - list of object ids saved Get a list of dictionary objects from client and insert or save them to the timeline. Apply the given mtime to all objects or apply utcnow(). _mtime is used to support timebased 'delta' updates. ''' self.cube_exists(owner, cube) self.requires_owner_write(owner, cube) mtime = dt2ts(mtime) if mtime else utcnow() current_mtime = self.get_cube_last_start(owner, cube) if current_mtime and mtime and current_mtime > mtime: # don't fail, but make sure the issue is logged! # likely, a ntp time sync is required logger.warn( "object mtime is < server mtime; %s < %s; " % (mtime, current_mtime)) _cube = self.timeline(owner, cube, admin=True) olen_r = len(objects) logger.debug('[%s.%s] Recieved %s objects' % (owner, cube, olen_r)) objects = self.prepare_objects(_cube, objects, mtime) logger.debug('[%s.%s] %s objects match their current version in db' % ( owner, cube, olen_r - len(objects))) if not objects: logger.debug('[%s.%s] No NEW objects to save' % (owner, cube)) return [] else: logger.debug('[%s.%s] Saving %s objects' % (owner, cube, len(objects))) # End the most recent versions in the db of those objects that # have newer versionsi (newest version must have _end == None, # activity import saves objects for which this might not be true): to_snap = dict([(o['_oid'], o['_start']) for o in objects if o['_end'] is None]) if to_snap: db_versions = _cube.find({'_oid': {'$in': to_snap.keys()}, '_end': None}, fields={'_id': 1, '_oid': 1}) snapped = 0 for doc in db_versions: _cube.update({'_id': doc['_id']}, {'$set': {'_end': to_snap[doc['_oid']]}}, multi=False) snapped += 1 logger.debug('[%s.%s] Updated %s OLD versions' % (owner, cube, snapped)) # Insert all new versions: insert_bulk(_cube, objects) logger.debug('[%s.%s] Saved %s NEW versions' % (owner, cube, len(objects))) # return object ids saved return [o['_oid'] for o in objects]
def _obj_start(self, obj, default=None): _start = obj.get("_start", default) obj["_start"] = _start or utcnow() return obj
def _obj_start(self, obj, default=None): _start = obj.get('_start', default) obj['_start'] = _start or utcnow() return obj