def delete_old_runs(expire_age=186, session=None): """Delete all runs and associated metadata older than the provided age :param int expire_age: The number of days into the past to use as the expiration date for deleting the runs :param session: optional session object if one isn't provided a new session """ session = session or get_session() expire_date = datetime.date.today() - datetime.timedelta(days=expire_age) db_utils.model_query(models.Run, session).filter( models.Run.run_at < expire_date).join( models.RunMetadata).delete(synchronize_session='evaluate') db_utils.model_query(models.Run, session).filter( models.Run.run_at < expire_date).delete(synchronize_session='evaluate')
def model_query(session, model, args=None, read_deleted='no', project_id=None): """Query helper that accounts for context's `read_deleted` field. :param session: The session to use, sqlalchemy.orm.session.Session :param model: Model to query. Must be a subclass of ModelBase. :param args: Arguments to query. If None - model is used. :param read_deleted: If not None, overrides context's read_deleted field. Permitted values are 'no', which does not return deleted values; 'only', which only returns deleted values; and 'yes', which does not filter deleted values. :param project_id: tenant id """ query_kwargs = {} if 'no' == read_deleted: query_kwargs['deleted'] = False elif 'only' == read_deleted: query_kwargs['deleted'] = True elif 'yes' == read_deleted: pass else: raise ValueError( _("Unrecognized read_deleted value '%s'") % read_deleted) query = sqlalchemyutils.model_query(model, session, args, **query_kwargs) if project_id: query = query.filter_by(project_id=project_id) return query
def service_destroy(context, id): session = get_session() with session.begin(): count = db_utils.model_query(models.Service, session=session).\ filter_by(id=id).\ soft_delete(synchronize_session=False) return count
def delete_mapping(self, uuid): session = db.get_session() q = utils.model_query(models.HashMapMapping, session) q = q.filter(models.HashMapMapping.mapping_id == uuid) r = q.delete() if not r: raise api.NoSuchMapping(uuid)
def model_query(context, model, args=None, read_deleted=None): """Query helper that accounts for context's `read_deleted` field. :param context: MasakariContext of the query. :param model: Model to query. Must be a subclass of ModelBase. :param args: Arguments to query. If None - model is used. :param read_deleted: If not None, overrides context's read_deleted field. Permitted values are 'no', which does not return deleted values; 'only', which only returns deleted values; and 'yes', which does not filter deleted values. """ if read_deleted is None: read_deleted = context.read_deleted query_kwargs = {} if 'no' == read_deleted: query_kwargs['deleted'] = False elif 'only' == read_deleted: query_kwargs['deleted'] = True elif 'yes' == read_deleted: pass else: raise ValueError( _("Unrecognized read_deleted value '%s'") % read_deleted) query = sqlalchemyutils.model_query(model, context.session, args, **query_kwargs) return query
def get_tenants(self, begin, end): session = db.get_session() q = utils.model_query(self.frame_model, session) q = q.filter(self.frame_model.begin >= begin, self.frame_model.end <= end) tenants = q.distinct().values(self.frame_model.tenant_id) return [tenant.tenant_id for tenant in tenants]
def delete_threshold(self, uuid): session = db.get_session() q = utils.model_query(models.HashMapThreshold, session) q = q.filter(models.HashMapThreshold.threshold_id == uuid) r = q.delete() if not r: raise api.NoSuchThreshold(uuid)
def service_delete(context, id): session = get_session() with session.begin(): count = db_utils.model_query(models.Service, session=session).\ filter_by(id=id).\ delete() return count
def upgrade(): migration_file = ('1679b5bc102c_add_subsecond_columns_to_test_runs_table.' 'mysql_upgrade.sql') migration_dir = os.path.dirname(os.path.realpath(__file__)) sql_path = os.path.join(migration_dir, migration_file) migration_context = context.get_context() if migration_context.dialect.name == 'mysql': with open(sql_path, 'r') as sql_file: op.execute(sql_file.read()) else: op.add_column('test_runs', sa.Column('start_time_microsecond', sa.Integer(), default=0)) op.add_column('test_runs', sa.Column('stop_time_microsecond', sa.Integer(), default=0)) if not CONF.disable_microsecond_data_migration: session = db_api.get_session() query = db_utils.model_query(models.TestRun, session).values( models.TestRun.id, models.TestRun.start_time, models.TestRun.stop_time) for test_run in query: start_micro = test_run[1].microsecond stop_micro = test_run[2].microsecond values = {'start_time_microsecond': start_micro, 'stop_time_microsecond': stop_micro} db_api.update_test_run(values, test_run[0], session) session.close()
def upgrade(): migration_file = ('1679b5bc102c_add_subsecond_columns_to_test_runs_table.' 'mysql_upgrade.sql') migration_dir = os.path.dirname(os.path.realpath(__file__)) sql_path = os.path.join(migration_dir, migration_file) migration_context = context.get_context() if migration_context.dialect.name == 'mysql': with open(sql_path, 'r') as sql_file: op.execute(sql_file.read()) else: op.add_column( 'test_runs', sa.Column('start_time_microsecond', sa.Integer(), default=0)) op.add_column( 'test_runs', sa.Column('stop_time_microsecond', sa.Integer(), default=0)) if not CONF.disable_microsecond_data_migration: session = db_api.get_session() query = db_utils.model_query(models.TestRun, session).values( models.TestRun.id, models.TestRun.start_time, models.TestRun.stop_time) for test_run in query: start_micro = test_run[1].microsecond stop_micro = test_run[2].microsecond values = { 'start_time_microsecond': start_micro, 'stop_time_microsecond': stop_micro } db_api.update_test_run(values, test_run[0], session) session.close()
def get_metadata(self, name): session = db.get_session() q = utils.model_query( models.StateInfo, session) q.filter(models.StateInfo.name == name) return q.value(models.StateInfo.s_metadata)
def model_query(context, model, args=None, read_deleted=None): """Query helper that accounts for context's `read_deleted` field. :param context: MasakariContext of the query. :param model: Model to query. Must be a subclass of ModelBase. :param args: Arguments to query. If None - model is used. :param read_deleted: If not None, overrides context's read_deleted field. Permitted values are 'no', which does not return deleted values; 'only', which only returns deleted values; and 'yes', which does not filter deleted values. """ if read_deleted is None: read_deleted = context.read_deleted query_kwargs = {} if 'no' == read_deleted: query_kwargs['deleted'] = False elif 'only' == read_deleted: query_kwargs['deleted'] = True elif 'yes' == read_deleted: pass else: raise ValueError(_("Unrecognized read_deleted value '%s'") % read_deleted) query = sqlalchemyutils.model_query( model, context.session, args, **query_kwargs) return query
def _get_db_item(self, session, identifier, fetcher=None, collector=None, scope_key=None): fetcher = fetcher or CONF.fetcher.backend collector = collector or CONF.collect.collector scope_key = scope_key or CONF.collect.scope_key q = utils.model_query(self.model, session) r = q.filter(self.model.identifier == identifier). \ filter(self.model.scope_key == scope_key). \ filter(self.model.fetcher == fetcher). \ filter(self.model.collector == collector). \ first() # In case the identifier exists with empty columns, update them if not r: # NOTE(peschk_l): We must use == instead of 'is' because sqlachmey # overloads this operator r = q.filter(self.model.identifier == identifier). \ filter(self.model.scope_key == None). \ filter(self.model.fetcher == None). \ filter(self.model.collector == None). \ first() # noqa if r: r.scope_key = scope_key r.collector = collector r.fetcher = fetcher LOG.info('Updating identifier "{i}" with scope_key "{sk}", ' 'collector "{c}" and fetcher "{f}"'.format( i=identifier, sk=scope_key, c=collector, f=fetcher)) session.commit() return r
def delete_mapping(self, service): session = db.get_session() q = utils.model_query(models.ServiceToCollectorMapping, session) q = q.filter(models.ServiceToCollectorMapping.service == service) r = q.delete() if not r: raise api.NoSuchMapping(service)
def delete_device(self, device_id): session = db.get_session() q = utils.model_query(models.Device, session) q = q.filter(models.Device.device_id == device_id) r = q.delete() if not r: raise api.NoSuchDevice(device_id)
def _get_db_item(self, session, identifier, fetcher=None, collector=None, scope_key=None): fetcher = fetcher or CONF.fetcher.backend collector = collector or CONF.collect.collector scope_key = scope_key or CONF.collect.scope_key q = utils.model_query(self.model, session) r = q.filter(self.model.identifier == identifier). \ filter(self.model.scope_key == scope_key). \ filter(self.model.fetcher == fetcher). \ filter(self.model.collector == collector). \ first() # In case the identifier exists with empty columns, update them if not r: # NOTE(peschk_l): We must use == instead of 'is' because sqlalchemy # overloads this operator r = q.filter(self.model.identifier == identifier). \ filter(self.model.scope_key == None). \ filter(self.model.fetcher == None). \ filter(self.model.collector == None). \ first() # noqa if r: r.scope_key = scope_key r.collector = collector r.fetcher = fetcher LOG.info('Updating identifier "{i}" with scope_key "{sk}", ' 'collector "{c}" and fetcher "{f}"'.format( i=identifier, sk=scope_key, c=collector, f=fetcher)) session.commit() return r
def model_query(context, model, *args, **kwargs): """Query helper that accounts for context's `read_deleted` field. :param context: context to query under :param model: model to query. Must be a subclass of ModelBase. :param session: if present, the session to use :param read_deleted: if present, overrides context's read_deleted field. :param project_only: if present and context is user-type, then restrict query to match the context's project_id. """ session = kwargs.get('session') or get_session() read_deleted = kwargs.get('read_deleted') or context.read_deleted project_only = kwargs.get('project_only') kwargs = dict() if project_only and not context.is_admin: kwargs['project_id'] = context.project_id if read_deleted in ('no', 'n', False): kwargs['deleted'] = False elif read_deleted in ('yes', 'y', True): kwargs['deleted'] = True return db_utils.model_query(model=model, session=session, args=args, **kwargs)
def stop_fixture(self): model = models.RatedDataFrame session = db.get_session() q = utils.model_query( model, session) q.delete()
def get_all(self, identifier=None, fetcher=None, collector=None, scope_key=None, active=1, limit=100, offset=0): """Returns the state of all scopes. This function returns the state of all scopes with support for optional filters. :param identifier: optional scope identifiers to filter on :type identifier: list :param fetcher: optional scope fetchers to filter on :type fetcher: list :param collector: optional collectors to filter on :type collector: list :param fetcher: optional fetchers to filter on :type fetcher: list :param scope_key: optional scope_keys to filter on :type scope_key: list :param active: optional active to filter scopes by status (active/deactivated) :type active: int :param limit: optional to restrict the projection :type limit: int :param offset: optional to shift the projection :type offset: int """ session = db.get_session() session.begin() q = utils.model_query(self.model, session) if identifier: q = q.filter( self.model.identifier.in_(to_list_if_needed(identifier))) if fetcher: q = q.filter(self.model.fetcher.in_(to_list_if_needed(fetcher))) if collector: q = q.filter(self.model.collector.in_( to_list_if_needed(collector))) if scope_key: q = q.filter(self.model.scope_key.in_( to_list_if_needed(scope_key))) if active is not None and active != []: q = q.filter(self.model.active.in_(to_list_if_needed(active))) q = apply_offset_and_limit(limit, offset, q) r = q.all() session.close() for item in r: item.last_processed_timestamp = tzutils.utc_to_local( item.last_processed_timestamp) item.scope_activation_toggle_date = tzutils.utc_to_local( item.scope_activation_toggle_date) return r
def get_test_counts_in_date_range(test_id, start_date=None, stop_date=None, session=None): """Return the number of successes, failures, and skips for a single test. Optionally you can provide a date to filter the results to be within a certain date range :param str start_date: The date to use as the start for counting :param str stop_date: The date to use as the cutoff for counting :param session: optional session object if one isn't provided a new session will be acquired for the duration of this operation :return: a dict containing the number of successes, failures, and skips :rtype: dict """ start_date = datetime.datetime.strptime(start_date, '%b %d %Y') if isinstance(stop_date, str): stop_date = datetime.datetime.strptime(stop_date, '%b %d %Y') session = session or get_session() count_dict = {} success_query = db_utils.model_query(models.TestRun, session).filter_by( test_id=test_id).filter(models.TestRun.status == 'success') fail_query = db_utils.model_query(models.TestRun, session).filter_by( test_id=test_id).filter(models.TestRun.status == 'fail') skip_query = db_utils.model_query(models.TestRun, session).filter_by( test_id=test_id).filter(models.TestRun.status == 'skip') if start_date: success_query = success_query.filter( models.TestRun.start_time > start_date) fail_query = fail_query.filter( models.TestRun.start_time > start_date) skip_query = skip_query.filter( models.TestRun.start_time > start_date) if stop_date: success_query = success_query.filter( models.TestRun.stop_time < stop_date) fail_query = fail_query.filter( models.TestRun.stop_time < stop_date) skip_query = skip_query.filter( models.TestRun.stop_time < stop_date) count_dict['success'] = success_query.count() count_dict['failure'] = fail_query.count() count_dict['skips'] = skip_query.count() return count_dict
def get_state(self, tenant_id=None): session = db.get_session() q = utils.model_query(self.state_model, session) if tenant_id: q = q.filter(self.state_model.tenant_id == tenant_id) q = q.order_by(self.state_model.state.desc()) r = q.first() return ck_utils.dt2ts(r.state) if r else None
def model_query(model, args=None, session=None): if session is None: session = get_session() query = sqlalchemyutils.model_query(model, session, args) return query
def model_query(model, *args, **kwargs): """Query helper for simpler session usage. :param session: if present, the session to use """ with _session_for_read() as session: query = oslo_db_utils.model_query(model, session, *args, **kwargs) return query
def get_state(self, tenant_id=None): session = db.get_session() q = utils.model_query(models.RatedDataFrame, session) if tenant_id: q = q.filter(models.RatedDataFrame.tenant_id == tenant_id) r = q.order_by(models.RatedDataFrame.begin.desc()).first() if r: return ck_utils.dt2ts(r.begin)
def get_mapping(self, service): session = db.get_session() try: q = utils.model_query(models.ServiceToCollectorMapping, session) q = q.filter(models.ServiceToCollectorMapping.service == service) return q.one() except sqlalchemy.orm.exc.NoResultFound: raise api.NoSuchMapping(service)
def list_services(self, collector=None): session = db.get_session() q = utils.model_query(models.ServiceToCollectorMapping, session) if collector: q = q.filter( models.ServiceToCollectorMapping.collector == collector) res = q.distinct().values(models.ServiceToCollectorMapping.service) return res
def list_mappings(self, collector=None): session = db.get_session() q = utils.model_query(models.ServiceToCollectorMapping, session) if collector: q = q.filter( models.ServiceToCollectorMapping.collector == collector) res = q.all() return res
def delete_mapping(self, service): session = db.get_session() q = utils.model_query( models.ServiceToCollectorMapping, session) q = q.filter(models.ServiceToCollectorMapping.service == service) r = q.delete() if not r: raise api.NoSuchMapping(service)
def test_project_filter_allow_none(self): mock_query = utils.model_query(MyModelSoftDeletedProjectId, session=self.session, project_id=(10, None)) self.assertEqual( str(mock_query.filter.call_args[0][0]), 'soft_deleted_project_id_test_model.project_id' ' IN (:project_id_1, NULL)')
def test_deleted_true(self): mock_query = utils.model_query( MyModelSoftDeleted, session=self.session, deleted=True) deleted_filter = mock_query.filter.call_args[0][0] self.assertEqual(str(deleted_filter), 'soft_deleted_test_model.deleted != :deleted_1') self.assertEqual(deleted_filter.right.value, MyModelSoftDeleted.__mapper__.c.deleted.default.arg)
def get_data(_id): model = models.TestData session = get_session(CONF.database) query = sqlalchemyutils.model_query(model, session) obj = query.filter_by(id=_id).first() if obj: return {"id": obj.id, "data": obj.data} else: return None
def get_state(self, tenant_id=None): session = db.get_session() q = utils.model_query(self.frame_model, session) if tenant_id: q = q.filter(self.frame_model.tenant_id == tenant_id) q = q.order_by(self.frame_model.begin.desc()) r = q.first() if r: return ck_utils.dt2ts(r.begin)
def get_priority(self, name): session = db.get_session() q = utils.model_query(models.ModuleStateInfo, session) q = q.filter(models.ModuleStateInfo.name == name) res = q.value(models.ModuleStateInfo.priority) if res: return int(res) else: return 1
def get_state(self, name): session = db.get_session() try: q = utils.model_query(models.ModuleStateInfo, session) q = q.filter(models.ModuleStateInfo.name == name) res = q.value(models.ModuleStateInfo.state) return bool(res) except sqlalchemy.orm.exc.NoResultFound: return None
def delete_mapping(self, uuid): session = db.get_session() q = utils.model_query( models.HashMapMapping, session) q = q.filter(models.HashMapMapping.mapping_id == uuid) r = q.delete() if not r: raise api.NoSuchMapping(uuid)
def delete_threshold(self, uuid): session = db.get_session() q = utils.model_query( models.HashMapThreshold, session) q = q.filter(models.HashMapThreshold.threshold_id == uuid) r = q.delete() if not r: raise api.NoSuchThreshold(uuid)
def list_mappings(self, collector=None): session = db.get_session() q = utils.model_query( models.ServiceToCollectorMapping, session) if collector: q = q.filter( models.ServiceToCollectorMapping.collector == collector) res = q.all() return res
def test_project_filter_allow_none(self): mock_query = utils.model_query( MyModelSoftDeletedProjectId, session=self.session, project_id=(10, None)) self.assertEqual( str(mock_query.filter.call_args[0][0]), 'soft_deleted_project_id_test_model.project_id' ' IN (:project_id_1, NULL)' )
def get_ids_for_all_tests(session=None): """Return a list of ids (uuid primary key) for all tests in the database :param session: optional session object if one isn't provided a new session will be acquired for the duration of this operation :return: The list of all ids for tests in the tests table :rtype: list """ session = session or get_session() return db_utils.model_query(models.Test, session).value(models.Test.id)
def get_state(self, name): session = db.get_session() try: q = utils.model_query( models.ModuleStateInfo, session) q = q.filter(models.ModuleStateInfo.name == name) res = q.value(models.ModuleStateInfo.state) return bool(res) except sqlalchemy.orm.exc.NoResultFound: return None
def get_device(self, device_id=None, name=None): session = db.get_session() try: q = utils.model_query(models.Device, session) if device_id: q = q.filter(models.Device.device_id == device_id) if name: q = q.filter(models.Device.name == name) return q.one() except sqlalchemy.orm.exc.NoResultFound: raise api.NoSuchDevice(device_id)
def get_mapping(self, service): session = db.get_session() try: q = utils.model_query( models.ServiceToCollectorMapping, session) q = q.filter( models.ServiceToCollectorMapping.service == service) return q.one() except sqlalchemy.orm.exc.NoResultFound: raise api.NoSuchMapping(service)
def get_tenants(self, begin, end): session = db.get_session() q = utils.model_query( self.frame_model, session) q = q.filter( self.frame_model.begin >= begin, self.frame_model.end <= end) tenants = q.distinct().values( self.frame_model.tenant_id) return [tenant.tenant_id for tenant in tenants]
def get_priority(self, name): session = db.get_session() q = utils.model_query( models.ModuleStateInfo, session) q = q.filter(models.ModuleStateInfo.name == name) res = q.value(models.ModuleStateInfo.priority) if res: return int(res) else: return 1
def list_services(self, collector=None): session = db.get_session() q = utils.model_query( models.ServiceToCollectorMapping, session) if collector: q = q.filter( models.ServiceToCollectorMapping.collector == collector) res = q.distinct().values( models.ServiceToCollectorMapping.service) return res
def model_query(context, model, *args, **kwargs): """Query helper for model query. :param context: context to query under :param model: model to query. Must be a subclass of ModelBase. :param session: if present, the session to use """ session = kwargs.pop('session') or get_session() return db_utils.model_query(model=model, session=session, args=args, **kwargs)
def get_id_from_test_id(test_id, session=None): """Return the id (uuid primary key) for a test given it's test_id value :param str test_id: :param session: optional session object if one isn't provided a new session will be acquired for the duration of this operation :return: The id for the specified test :rtype: str """ session = session or get_session() return db_utils.model_query(models.Test, session).filter_by( test_id=test_id).value('id')
def test_project_filter(self): project_id = 10 mock_query = utils.model_query( MyModelSoftDeletedProjectId, session=self.session, project_only=True, project_id=project_id) deleted_filter = mock_query.filter.call_args[0][0] self.assertEqual( str(deleted_filter), 'soft_deleted_project_id_test_model.project_id = :project_id_1') self.assertEqual(deleted_filter.right.value, project_id)
def get_all_test_runs(session=None): """Return all test runs from the DB. :param session: optional session object if one isn't provided a new session will be acquired for the duration of this operation :return list: The list of test run objects :rtype: subunit2sql.models.TestRun """ session = session or get_session() query = db_utils.model_query(models.TestRun, session) return query.all()
def delete_script(self, name=None, uuid=None): session = db.get_session() q = utils.model_query(models.PyScriptsScript, session) if name: q = q.filter(models.PyScriptsScript.name == name) elif uuid: q = q.filter(models.PyScriptsScript.script_id == uuid) else: raise ValueError('You must specify either name or uuid.') r = q.delete() if not r: raise api.NoSuchScript(uuid=uuid)
def get_state(self, tenant_id=None): session = db.get_session() q = utils.model_query( self.frame_model, session) if tenant_id: q = q.filter( self.frame_model.tenant_id == tenant_id) q = q.order_by( self.frame_model.begin.desc()) r = q.first() if r: return ck_utils.dt2ts(r.begin)
def get_recent_successful_runs(num_runs=10, session=None): """Return a list of run uuid strings for the most recent successful runs :param int num_runs: The number of runs to return in the list :param session: optional session object if one isn't provided a new session :return list: A list of run uuid strings (the id column in the runs table) for the most recent runs. """ session = session or get_session() results = db_utils.model_query(models.Run, session).order_by( models.Run.run_at.desc()).filter_by(fails=0).limit(num_runs).all() return map(lambda x: x.id, results)
def get_run_by_id(id, session=None): """Get an individual run by it's uuid. :param str id: The uuid for the run (the id field in the DB) :param session: optional session object if one isn't provided a new session will be acquired for the duration of this operation :return: The specified run object :rtype: subunit2sql.models.Run """ session = session or get_session() run = db_utils.model_query(models.Run, session).filter_by(id=id).first() return run
def get_latest_run(session=None): """Return the most recently created run from the DB. :param session: optional session object if one isn't provided a new session will be acquired for the duration of this operation :return: The latest run object :rtype: subunit2sql.models.Run """ session = session or get_session() query = db_utils.model_query(models.Run, session).order_by( models.Run.run_at.desc()) return query.first()
def get_test_run_metadata(test_run_id, session=None): """Return all run metadata objects for associated with a given run. :param str test_run_id: The uuid of the test_run to get all the metadata :param session: optional session object if one isn't provided a new session will be acquired for the duration of this operation :return list: The list of created metadata objects :rtype: subunit2sql.models.RunMeta """ session = session or get_session() query = db_utils.model_query(models.TestRunMetadata, session).filter_by( test_run_id=test_run_id) return query.all()