def register_event_listener(): for event_ in sqlalchemy_events: # listen transient need extra work if event_ == 'transient': event.listens_for(Base, 'init', propagate=True)(transient_state) continue event.listens_for(Session, event_)(state_transition(event_))
def __init__(self, instruments, frequency=Frequency.MINUTE, table_name='bins', file_name='sqlite'): self.__engine = create_engine(f'sqlite:///{file_name}') self.__DBSession = sessionmaker(bind=self.__engine) self.__logger = get_logger(SQLiteFeed.LOGGER_NAME) if table_name in table_names.keys(): self.__bar_model = table_names[table_name] else: self.__bar_model = make_bar_model(table_name) table_names[table_name] = self.__bar_model def before_cursor_execute(conn, cursor, statement, parameters, context, executemany): conn.info.setdefault('query_start_time', []).append(time.time()) self.__logger.debug( f'Start Query: {statement} with params: {parameters}') def after_cursor_execute(conn, cursor, statement, parameters, context, executemany): total = time.time() - conn.info['query_start_time'].pop(-1) self.__logger.debug("Query Complete!") self.__logger.debug("Total Time: %f", total) event.listens_for(self.__engine, "before_cursor_execute")(before_cursor_execute) event.listens_for(self.__engine, "after_cursor_execute")(after_cursor_execute) super(SQLiteFeed, self).__init__(frequency, instruments, None, None) self.bars = []
def setup_ownership_load_event(cls, owner_class, relns): def load_owner_object(target, context): for reln in relns: ls = getattr(target, reln, None) if ls is not None: ls.owner_object = target event.listen(owner_class, "load", load_owner_object, propagate=True) event.listens_for(owner_class, "refresh", load_owner_object, propagate=True) def set_owner_object(target, value, old_value, initiator): if old_value is not None: old_value.owner_object = None if value is not None: value.owner_object = target for reln in relns: cls._owning_relns.append((owner_class, reln)) event.listen(getattr(owner_class, reln), "set", set_owner_object, propagate=True)
def __init__(self, cls, event_name): super().__init__() self.cls = cls self.event_name = event_name self.callbacks = defaultdict(list) def listener(mapper, connection, target): del mapper del connection state = inspect(target) for attr, callbacks in self.callbacks.items(): history = state.get_history(attr, True) if not history.has_changes(): continue for callback in callbacks: old = None if history.deleted and history.deleted[0]: old = history.deleted[0] # not too clear on why this is a list new = None if history.added and history.added[0]: new = history.added[0] # not too clear on why this is a list callback(target=target, new=new, old=old) event.listens_for(self.cls, self.event_name)(listener)
def initialize_logging(settings): # UGLY: I don't know how to count CPU time accurately in a multithreaded environment and I'm too lazy # to implement proper query counting for a multithreaded environment. Not sure how to test this # reliably, I resort to checking if we run under the Werkzeug reloader as a means to decide whether or # not to count CPU time and database queries when logging. Rough, but effective. count_cpu_usage_and_db_queries = 'WERKZEUG_RUN_MAIN' in environ logging_exemptions = () if 'JJ_DEBUG_ASSETS' in environ else ('/static/', '/system/admin/static/', '/system/rq/') query_count_increment = install_request_logger(app, count_cpu_usage_and_db_queries, getLogger('jj.request'), logging_exemptions) if count_cpu_usage_and_db_queries: event.listens_for(Engine, "after_cursor_execute")(query_count_increment)
def enable_time_logging(query_time_threshold): def before_cursor_execute(conn, cursor, statement, params, context, execmany): # pylint: disable=W0613,R0913 conn.info.setdefault('query_start_time', []).append(time.time()) def after_cursor_execute(conn, cursor, statement, params, context, execmany): # pylint: disable=W0613,R0913 total = time.time() - conn.info['query_start_time'].pop(-1) if total > query_time_threshold: RUNTIME_LOGGER.debug(f' SLOW QUERY: {total:.3f} '.center(80, '-')) RUNTIME_LOGGER.debug(f'Params: {params}') # pylint: disable=W1202 RUNTIME_LOGGER.debug(statement) RUNTIME_LOGGER.setLevel(logging.DEBUG) event.listens_for(Engine, 'before_cursor_execute')(before_cursor_execute) event.listens_for(Engine, 'after_cursor_execute')(after_cursor_execute)
def wrapper(model): assert hasattr(model, field) # Discover the hooks defined by the @before and @after decorators hooks = {"before": [], "after": [], "after_save": []} for attr in dir(model): if not attr.startswith("__"): hook = getattr(model, attr) if isinstance(hook, Hook): hooks[hook.hook_type].append(hook) # Re-order the hooks by order of definition using their order attribute for key in hooks: hooks[key] = sorted(hooks[key]) def listener(mapper, connection, target): while target._state_machine_save_queue: target._state_machine_save_queue.pop(0)() event.listens_for(model, "after_insert")(listener) event.listens_for(model, "after_update")(listener) # Save the name of the field, the enum base class and the hook list inside the class setattr(model, "_state_machine_field", field) setattr(model, "_state_machine_hooks", hooks) setattr(model, "_state_machine_save_queue", []) # Generate the `obj.{transition}()` methods for transition in cls.__transitions__: transition = BoundTransition(transition, cls) setattr(model, transition.name, cls._gen_apply(transition)) setattr( model, "can_{name}".format(name=transition.name), cls._gen_can_apply(transition), ) # Generate the `is_{state}` properties for elem in cls: setattr( model, "is_{name}".format(name=str(elem)), cls._gen_is_state(elem, field), ) return model
def database(_database, request): config, engine = _database connection = engine.connect() trans = connection.begin() configure_session(config, bind=connection) if request.node.get_closest_marker("allow_db_rollback"): session.begin_nested() event.listens_for(session, "after_transaction_end")(_restart_savepoint) else: session().rollback = _error_out yield session.remove() trans.rollback() connection.close()
def _configure_creation(self, connection): def do_connect(dbapi_connection, connection_record): # disable pysqlite's emitting of the BEGIN statement entirely. # also stops it from emitting COMMIT before any DDL. iso_level = dbapi_connection.isolation_level dbapi_connection.isolation_level = None try: dbapi_connection.execute("PRAGMA page_size = 5120;") dbapi_connection.execute("PRAGMA cache_size = 12000;") dbapi_connection.execute("PRAGMA foreign_keys = ON;") dbapi_connection.execute("PRAGMA journal_mode = WAL;") except: pass dbapi_connection.isolation_level = iso_level event.listens_for(connection, "connect")(do_connect)
def register_to_celery(celery_broker, celery_config, async_task, max_retries=12, DBSession=None): def send_after_commit_tasks(session): if not hasattr(async_ctx, 'reged_tasks'): return for task in async_ctx.reged_tasks: task.send(async_api) delattr(async_ctx, 'reged_tasks') broker = 'amqp://{user}:{password}@{host}:{port}/{vhost}'.\ format(**celery_broker) app = Celery(broker=broker) app.conf.update(**celery_config) async_api = app.task(max_retries=max_retries, bind=True)(async_task) if DBSession: if event: event.listens_for(DBSession, 'after_commit')(send_after_commit_tasks) else: raise ImportError('You must install sqlalchemy first.') return app, async_api
def test_session_events(self): app = flask.Flask(__name__) app.config['SQLALCHEMY_ENGINE'] = 'sqlite://' app.config['TESTING'] = True db = sqlalchemy.SQLAlchemy(app) from sqlalchemy.event import listens_for seen = [] register = listens_for(db.session, 'after_commit') register(seen.append) db.session.commit() self.assertEqual(seen, [db.session()])
def __init__(self, table_name='bars', db_name='database', db_username='******', db_password='******', connector='mysqldb', db_host='127.0.0.1', db_port=3306): self.__engine = create_engine( f'mysql+{connector}://{db_username}:{db_password}@{db_host}:{db_port}/{db_name}' ) self.__DBSession = sessionmaker(bind=self.__engine) self.__logger = get_logger(MySQLFeed.LOGGER_NAME) self.__bar_model = make_bar_model(table_name) def before_cursor_execute(conn, cursor, statement, parameters, context, executemany): conn.info.setdefault('query_start_time', []).append(time.time()) self.__logger.debug( f'Start Query: {statement} with params: {parameters}') def after_cursor_execute(conn, cursor, statement, parameters, context, executemany): total = time.time() - conn.info['query_start_time'].pop(-1) self.__logger.debug("Query Complete!") self.__logger.debug("Total Time: %f", total) event.listens_for(self.__engine, "before_cursor_execute")(before_cursor_execute) event.listens_for(self.__engine, "after_cursor_execute")(after_cursor_execute) super(MySQLFeed, self).__init__() self.bars = []
def wa(f): if event_name in SQLALCHEMY_MAPPER_EVENTS: event.listens_for(entity, event_name)(f) elif event_name in METADASH_TO_SQLALCHEMY_EVNET_MAP: events = METADASH_TO_SQLALCHEMY_EVNET_MAP[event_name] if isinstance(events, list): for ev in events: event.listens_for(entity, ev)(f) elif isinstance(events, str): event.listens_for(entity, events)(f) return f
def create_schedule(abc_schedule, base, mixins=None, persister=None): """Concrete schedule model factory. :param abc_schedule: Abstract base schedule to use as base. :type abc_schedule: Any ABC schedule from :func:`~create_abc_schedule` factory function. :param base: SQLAlchemy model base to use. :type base: Any SQLAlchemy model base from :func:`sqlalchemy.ext.declarative.declarative_base` factory function :param mixins: Mixins to be mixed into concrete schedule model. :type mixins: Iterable mixin classes. :param persister: Persister to use for the schedule persistence. :type persister: :class:`~news.persistence.ScheduleNotifier` :returns: Concrete schedule model based on given abc schedule. :rtype: :class:`~news.models.AbstractSchedule` SQLAlchemy implementation based on given abc schedule, model base and mixins. """ mixins = mixins or tuple() Schedule = type('Schedule', mixins + (abc_schedule, base), {}) # connect persister if given if persister: event.listens_for(Schedule, 'after_insert')( lambda mapper, connection, target: persister.notify_saved(target, created=True) ) event.listens_for(Schedule, 'after_update')( lambda mapper, connection, target: persister.notify_saved(target, created=False) ) event.listens_for(Schedule, 'after_delete')( lambda mapper, connection, target: persister.notify_deleted(target) ) return Schedule
def enqueue_after_commit(handler_cls, *args, **kwargs): # type: ignore sqlalchemy_event.listens_for( connection, "commit")(lambda _conn: queue.enqueue( async_handler_generic_task, handler_cls, *args, **kwargs))
def setup(self): # heavy load mode is read only mode with a different infobar if self.heavy_load_mode: self.read_only_mode = True if hasattr(signal, 'SIGUSR1'): # not all platforms have user signals signal.signal(signal.SIGUSR1, thread_dump) # initialize caches. Any cache-chains built here must be added # to cache_chains (closed around by reset_caches) so that they # can properly reset their local components localcache_cls = (SelfEmptyingCache if self.running_as_script else LocalCache) num_mc_clients = self.num_mc_clients self.cache_chains = {} # for now, zookeeper will be an optional part of the stack. # if it's not configured, we will grab the expected config from the # [live_config] section of the ini file zk_hosts = self.config.get("zookeeper_connection_string") if zk_hosts: from r2.lib.zookeeper import (connect_to_zookeeper, LiveConfig, LiveList) zk_username = self.config["zookeeper_username"] zk_password = self.config["zookeeper_password"] self.zookeeper = connect_to_zookeeper(zk_hosts, (zk_username, zk_password)) self.live_config = LiveConfig(self.zookeeper, LIVE_CONFIG_NODE) self.throttles = LiveList(self.zookeeper, "/throttles", map_fn=ipaddress.ip_network, reduce_fn=ipaddress.collapse_addresses) else: self.zookeeper = None parser = ConfigParser.RawConfigParser() parser.read([self.config["__file__"]]) self.live_config = extract_live_config(parser, self.plugins) self.throttles = tuple() # immutable since it's not real self.memcache = CMemcache(self.memcaches, num_clients=num_mc_clients) self.lock_cache = CMemcache(self.lockcaches, num_clients=num_mc_clients) self.stats = Stats(self.config.get('statsd_addr'), self.config.get('statsd_sample_rate')) event.listens_for(engine.Engine, 'before_cursor_execute')( self.stats.pg_before_cursor_execute) event.listens_for(engine.Engine, 'after_cursor_execute')( self.stats.pg_after_cursor_execute) self.make_lock = make_lock_factory(self.lock_cache, self.stats) if not self.cassandra_seeds: raise ValueError("cassandra_seeds not set in the .ini") keyspace = "reddit" self.cassandra_pools = { "main": StatsCollectingConnectionPool(keyspace, stats=self.stats, logging_name="main", server_list=self.cassandra_seeds, pool_size=self.cassandra_pool_size, timeout=2, max_retries=3, prefill=False), } perma_memcache = (CMemcache(self.permacache_memcaches, num_clients=num_mc_clients) if self.permacache_memcaches else None) self.permacache = CassandraCacheChain( localcache_cls(), CassandraCache('permacache', self.cassandra_pools[self.cassandra_default_pool], read_consistency_level=self.cassandra_rcl, write_consistency_level=self.cassandra_wcl), memcache=perma_memcache, lock_factory=self.make_lock) self.cache_chains.update(permacache=self.permacache) # hardcache is done after the db info is loaded, and then the # chains are reset to use the appropriate initial entries if self.stalecaches: self.cache = StaleCacheChain( localcache_cls(), CMemcache(self.stalecaches, num_clients=num_mc_clients), self.memcache) else: self.cache = MemcacheChain((localcache_cls(), self.memcache)) self.cache_chains.update(cache=self.cache) self.rendercache = MemcacheChain( (localcache_cls(), CMemcache(self.rendercaches, noreply=True, no_block=True, num_clients=num_mc_clients))) self.cache_chains.update(rendercache=self.rendercache) self.thing_cache = CacheChain((localcache_cls(), )) self.cache_chains.update(thing_cache=self.thing_cache) #load the database info self.dbm = self.load_db_params() # can't do this until load_db_params() has been called self.hardcache = HardcacheChain( (localcache_cls(), self.memcache, HardCache(self)), cache_negative_results=True) self.cache_chains.update(hardcache=self.hardcache) # I know this sucks, but we need non-request-threads to be # able to reset the caches, so we need them be able to close # around 'cache_chains' without being able to call getattr on # 'g' cache_chains = self.cache_chains.copy() def reset_caches(): for name, chain in cache_chains.iteritems(): chain.reset() chain.stats = CacheStats(self.stats, name) self.reset_caches = reset_caches self.reset_caches() # set the modwindow self.MODWINDOW = timedelta(self.MODWINDOW) self.REDDIT_MAIN = bool(os.environ.get('REDDIT_MAIN')) origin_prefix = self.domain_prefix + "." if self.domain_prefix else "" self.origin = "http://" + origin_prefix + self.domain self.secure_domains = set([urlparse(self.payment_domain).netloc]) self.trusted_domains = set([self.domain]) self.trusted_domains.update(self.authorized_cnames) if self.https_endpoint: https_url = urlparse(self.https_endpoint) self.secure_domains.add(https_url.netloc) self.trusted_domains.add(https_url.hostname) if getattr(self, 'oauth_domain', None): self.secure_domains.add(self.oauth_domain) # load the unique hashed names of files under static static_files = os.path.join(self.paths.get('static_files'), 'static') names_file_path = os.path.join(static_files, 'names.json') if os.path.exists(names_file_path): with open(names_file_path) as handle: self.static_names = json.load(handle) else: self.static_names = {} #setup the logger self.log = logging.getLogger('reddit') self.log.addHandler(logging.StreamHandler()) if self.debug: self.log.setLevel(logging.DEBUG) else: self.log.setLevel(logging.INFO) # set log level for pycountry which is chatty logging.getLogger('pycountry.db').setLevel(logging.CRITICAL) if not self.media_domain: self.media_domain = self.domain if self.media_domain == self.domain: print("Warning: g.media_domain == g.domain. " + "This may give untrusted content access to user cookies") self.reddit_host = socket.gethostname() self.reddit_pid = os.getpid() for arg in sys.argv: tokens = arg.split("=") if len(tokens) == 2: k, v = tokens self.log.debug("Overriding g.%s to %s" % (k, v)) setattr(self, k, v) #if we're going to use the query_queue, we need amqp if self.write_query_queue and not self.amqp_host: raise Exception("amqp_host must be defined to use the query queue") # This requirement doesn't *have* to be a requirement, but there are # bugs at the moment that will pop up if you violate it if self.write_query_queue and not self.use_query_cache: raise Exception("write_query_queue requires use_query_cache") # try to set the source control revision numbers self.versions = {} r2_root = os.path.dirname(os.path.dirname(self.paths["root"])) r2_gitdir = os.path.join(r2_root, ".git") self.short_version = self.record_repo_version("r2", r2_gitdir) if I18N_PATH: i18n_git_path = os.path.join(os.path.dirname(I18N_PATH), ".git") self.record_repo_version("i18n", i18n_git_path) if self.log_start: self.log.error("reddit app %s:%s started %s at %s" % (self.reddit_host, self.reddit_pid, self.short_version, datetime.now()))
def setup(self, global_conf): # heavy load mode is read only mode with a different infobar if self.heavy_load_mode: self.read_only_mode = True if hasattr(signal, 'SIGUSR1'): # not all platforms have user signals signal.signal(signal.SIGUSR1, thread_dump) # initialize caches. Any cache-chains built here must be added # to cache_chains (closed around by reset_caches) so that they # can properly reset their local components localcache_cls = (SelfEmptyingCache if self.running_as_script else LocalCache) num_mc_clients = self.num_mc_clients self.cache_chains = {} self.memcache = CMemcache(self.memcaches, num_clients = num_mc_clients) self.make_lock = make_lock_factory(self.memcache) self.stats = Stats(global_conf.get('statsd_addr'), global_conf.get('statsd_sample_rate')) event.listens_for(engine.Engine, 'before_cursor_execute')( self.stats.pg_before_cursor_execute) event.listens_for(engine.Engine, 'after_cursor_execute')( self.stats.pg_after_cursor_execute) if not self.cassandra_seeds: raise ValueError("cassandra_seeds not set in the .ini") keyspace = "reddit" self.cassandra_pools = { "main": StatsCollectingConnectionPool( keyspace, stats=self.stats, logging_name="main", server_list=self.cassandra_seeds, pool_size=self.cassandra_pool_size, timeout=2, max_retries=3, prefill=False ), "noretries": StatsCollectingConnectionPool( keyspace, stats=self.stats, logging_name="noretries", server_list=self.cassandra_seeds, pool_size=len(self.cassandra_seeds), timeout=2, max_retries=0, prefill=False ), } perma_memcache = (CMemcache(self.permacache_memcaches, num_clients = num_mc_clients) if self.permacache_memcaches else None) self.permacache = CassandraCacheChain(localcache_cls(), CassandraCache('permacache', self.cassandra_pools[self.cassandra_default_pool], read_consistency_level = self.cassandra_rcl, write_consistency_level = self.cassandra_wcl), memcache = perma_memcache, lock_factory = self.make_lock) self.cache_chains.update(permacache=self.permacache) # hardcache is done after the db info is loaded, and then the # chains are reset to use the appropriate initial entries if self.stalecaches: self.cache = StaleCacheChain(localcache_cls(), CMemcache(self.stalecaches, num_clients=num_mc_clients), self.memcache) else: self.cache = MemcacheChain((localcache_cls(), self.memcache)) self.cache_chains.update(cache=self.cache) self.rendercache = MemcacheChain((localcache_cls(), CMemcache(self.rendercaches, noreply=True, no_block=True, num_clients = num_mc_clients))) self.cache_chains.update(rendercache=self.rendercache) self.thing_cache = CacheChain((localcache_cls(),)) self.cache_chains.update(thing_cache=self.thing_cache) #load the database info self.dbm = self.load_db_params(global_conf) # can't do this until load_db_params() has been called self.hardcache = HardcacheChain((localcache_cls(), self.memcache, HardCache(self)), cache_negative_results = True) self.cache_chains.update(hardcache=self.hardcache) # I know this sucks, but we need non-request-threads to be # able to reset the caches, so we need them be able to close # around 'cache_chains' without being able to call getattr on # 'g' cache_chains = self.cache_chains.copy() def reset_caches(): for name, chain in cache_chains.iteritems(): chain.reset() chain.stats = CacheStats(self.stats, name) self.reset_caches = reset_caches self.reset_caches() #make a query cache self.stats_collector = QueryStats() # set the modwindow self.MODWINDOW = timedelta(self.MODWINDOW) self.REDDIT_MAIN = bool(os.environ.get('REDDIT_MAIN')) origin_prefix = self.domain_prefix + "." if self.domain_prefix else "" self.origin = "http://" + origin_prefix + self.domain self.secure_domains = set([urlparse(self.payment_domain).netloc]) self.trusted_domains = set([self.domain]) self.trusted_domains.update(self.authorized_cnames) if self.https_endpoint: https_url = urlparse(self.https_endpoint) self.secure_domains.add(https_url.netloc) self.trusted_domains.add(https_url.hostname) # load the unique hashed names of files under static static_files = os.path.join(self.paths.get('static_files'), 'static') names_file_path = os.path.join(static_files, 'names.json') if os.path.exists(names_file_path): with open(names_file_path) as handle: self.static_names = json.load(handle) else: self.static_names = {} #setup the logger self.log = logging.getLogger('reddit') self.log.addHandler(logging.StreamHandler()) if self.debug: self.log.setLevel(logging.DEBUG) else: self.log.setLevel(logging.INFO) # set log level for pycountry which is chatty logging.getLogger('pycountry.db').setLevel(logging.CRITICAL) if not self.media_domain: self.media_domain = self.domain if self.media_domain == self.domain: print ("Warning: g.media_domain == g.domain. " + "This may give untrusted content access to user cookies") self.reddit_host = socket.gethostname() self.reddit_pid = os.getpid() for arg in sys.argv: tokens = arg.split("=") if len(tokens) == 2: k, v = tokens self.log.debug("Overriding g.%s to %s" % (k, v)) setattr(self, k, v) #if we're going to use the query_queue, we need amqp if self.write_query_queue and not self.amqp_host: raise Exception("amqp_host must be defined to use the query queue") # This requirement doesn't *have* to be a requirement, but there are # bugs at the moment that will pop up if you violate it if self.write_query_queue and not self.use_query_cache: raise Exception("write_query_queue requires use_query_cache") # try to set the source control revision number try: self.version = subprocess.check_output(["git", "rev-parse", "HEAD"]) except subprocess.CalledProcessError, e: self.log.info("Couldn't read source revision (%r)" % e) self.version = self.short_version = '(unknown)'
def after_commit(self, func): event.listens_for(self, 'after_commit', once=True)(adapt(func))
from sqlalchemy import event from API import * from flask_sqlalchemy import SQLAlchemy project_path = os.path.dirname(os.path.abspath(__file__)) database_file = "sqlite:///{}".format(os.path.join(project_path, "YCM.db")) app.config["SQLALCHEMY_DATABASE_URI"] = database_file db = SQLAlchemy(app) event.listens_for(db.get_engine(), 'connect') def create_math_functions_on_connect(dbapi_connection, connection_record): dbapi_connection.create_function('sin', 1, math.sin) dbapi_connection.create_function('cos', 1, math.cos) dbapi_connection.create_function('acos', 1, math.acos) dbapi_connection.create_function('radians', 1, math.radians)
self.status = FileStatus.READY def open(self, *args, **kwargs): return self.filesystem.fs.open(self.path, *args, **kwargs) def delete_blob(self): self.status = FileStatus.DELETING self.filesystem.fs.remove(self.path) self.status = FileStatus.DELETED @classmethod def _delete_file_blob(cls, mapper, connection, target): try: target.delete_blob() except Exception as e: current_app.logger.exception(e) event.listens_for(File, 'after_delete', File._delete_file_blob) class FileSchema(BaseSchema): __model__ = File path = fields.Str(required=True, validate=validate.Regexp(_path_regexp)) size = fields.Int(missing=0, validate=validate.Range(min=0)) status = fields.Int(missing=FileStatus.CREATED, validate=validate.OneOf(FileStatus)) filesystem_id = fields.UUID(required=True, load_only=True) filesystem = fields.Nested('FilesystemSchema', dump_only=True)
def make_connector(self, app=None, bind=None): connector = super(SQLAlchemy, self).make_connector(app, bind) engine = connector.get_engine() event.listens_for(engine, 'engine_connect')(ping_connection) return connector
def setup(self): self.queues = queues.declare_queues(self) ################# CONFIGURATION # AMQP is required if not self.amqp_host: raise ValueError("amqp_host not set in the .ini") # This requirement doesn't *have* to be a requirement, but there are # bugs at the moment that will pop up if you violate it # XXX: get rid of these options. new query cache is always on. if self.write_query_queue and not self.use_query_cache: raise Exception("write_query_queue requires use_query_cache") if not self.cassandra_seeds: raise ValueError("cassandra_seeds not set in the .ini") # heavy load mode is read only mode with a different infobar if self.heavy_load_mode: self.read_only_mode = True origin_prefix = self.domain_prefix + "." if self.domain_prefix else "" self.origin = "http://" + origin_prefix + self.domain self.secure_domains = set([urlparse(self.payment_domain).netloc]) self.trusted_domains = set([self.domain]) self.trusted_domains.update(self.authorized_cnames) if self.https_endpoint: https_url = urlparse(self.https_endpoint) self.secure_domains.add(https_url.netloc) self.trusted_domains.add(https_url.hostname) if getattr(self, 'oauth_domain', None): self.secure_domains.add(self.oauth_domain) # load the unique hashed names of files under static static_files = os.path.join(self.paths.get('static_files'), 'static') names_file_path = os.path.join(static_files, 'names.json') if os.path.exists(names_file_path): with open(names_file_path) as handle: self.static_names = json.load(handle) else: self.static_names = {} #setup the logger self.log = logging.getLogger('reddit') self.log.addHandler(logging.StreamHandler()) if self.debug: self.log.setLevel(logging.DEBUG) else: self.log.setLevel(logging.INFO) # set log level for pycountry which is chatty logging.getLogger('pycountry.db').setLevel(logging.CRITICAL) if not self.media_domain: self.media_domain = self.domain if self.media_domain == self.domain: print ("Warning: g.media_domain == g.domain. " + "This may give untrusted content access to user cookies") for arg in sys.argv: tokens = arg.split("=") if len(tokens) == 2: k, v = tokens self.log.debug("Overriding g.%s to %s" % (k, v)) setattr(self, k, v) self.reddit_host = socket.gethostname() self.reddit_pid = os.getpid() if hasattr(signal, 'SIGUSR1'): # not all platforms have user signals signal.signal(signal.SIGUSR1, thread_dump) self.startup_timer.intermediate("configuration") ################# ZOOKEEPER # for now, zookeeper will be an optional part of the stack. # if it's not configured, we will grab the expected config from the # [live_config] section of the ini file zk_hosts = self.config.get("zookeeper_connection_string") if zk_hosts: from r2.lib.zookeeper import (connect_to_zookeeper, LiveConfig, LiveList) zk_username = self.config["zookeeper_username"] zk_password = self.config["zookeeper_password"] self.zookeeper = connect_to_zookeeper(zk_hosts, (zk_username, zk_password)) self.live_config = LiveConfig(self.zookeeper, LIVE_CONFIG_NODE) self.throttles = LiveList(self.zookeeper, "/throttles", map_fn=ipaddress.ip_network, reduce_fn=ipaddress.collapse_addresses) else: self.zookeeper = None parser = ConfigParser.RawConfigParser() parser.read([self.config["__file__"]]) self.live_config = extract_live_config(parser, self.plugins) self.throttles = tuple() # immutable since it's not real self.startup_timer.intermediate("zookeeper") ################# MEMCACHE num_mc_clients = self.num_mc_clients # the main memcache pool. used for most everything. self.memcache = CMemcache(self.memcaches, num_clients=num_mc_clients) # a smaller pool of caches used only for distributed locks. # TODO: move this to ZooKeeper self.lock_cache = CMemcache(self.lockcaches, num_clients=num_mc_clients) self.make_lock = make_lock_factory(self.lock_cache, self.stats) # memcaches used in front of the permacache CF in cassandra. # XXX: this is a legacy thing; permacache was made when C* didn't have # a row cache. if self.permacache_memcaches: permacache_memcaches = CMemcache(self.permacache_memcaches, num_clients=num_mc_clients) else: permacache_memcaches = None # the stalecache is a memcached local to the current app server used # for data that's frequently fetched but doesn't need to be fresh. if self.stalecaches: stalecaches = CMemcache(self.stalecaches, num_clients=num_mc_clients) else: stalecaches = None # rendercache holds rendered partial templates. rendercaches = CMemcache( self.rendercaches, noreply=True, no_block=True, num_clients=num_mc_clients, ) # pagecaches hold fully rendered pages pagecaches = CMemcache( self.pagecaches, noreply=True, no_block=True, num_clients=num_mc_clients, ) self.startup_timer.intermediate("memcache") ################# CASSANDRA keyspace = "reddit" self.cassandra_pools = { "main": StatsCollectingConnectionPool( keyspace, stats=self.stats, logging_name="main", server_list=self.cassandra_seeds, pool_size=self.cassandra_pool_size, timeout=4, max_retries=3, prefill=False ), } permacache_cf = CassandraCache( 'permacache', self.cassandra_pools[self.cassandra_default_pool], read_consistency_level=self.cassandra_rcl, write_consistency_level=self.cassandra_wcl ) self.startup_timer.intermediate("cassandra") ################# POSTGRES event.listens_for(engine.Engine, 'before_cursor_execute')( self.stats.pg_before_cursor_execute) event.listens_for(engine.Engine, 'after_cursor_execute')( self.stats.pg_after_cursor_execute) self.dbm = self.load_db_params() self.startup_timer.intermediate("postgres") ################# CHAINS # initialize caches. Any cache-chains built here must be added # to cache_chains (closed around by reset_caches) so that they # can properly reset their local components self.cache_chains = {} localcache_cls = (SelfEmptyingCache if self.running_as_script else LocalCache) if stalecaches: self.cache = StaleCacheChain( localcache_cls(), stalecaches, self.memcache, ) else: self.cache = MemcacheChain((localcache_cls(), self.memcache)) self.cache_chains.update(cache=self.cache) self.rendercache = MemcacheChain(( localcache_cls(), rendercaches, )) self.cache_chains.update(rendercache=self.rendercache) self.pagecache = MemcacheChain(( localcache_cls(), pagecaches, )) self.cache_chains.update(pagecache=self.pagecache) # the thing_cache is used in tdb_cassandra. self.thing_cache = CacheChain((localcache_cls(),)) self.cache_chains.update(thing_cache=self.thing_cache) self.permacache = CassandraCacheChain( localcache_cls(), permacache_cf, memcache=permacache_memcaches, lock_factory=self.make_lock, ) self.cache_chains.update(permacache=self.permacache) # hardcache is used for various things that tend to expire # TODO: replace hardcache w/ cassandra stuff self.hardcache = HardcacheChain( (localcache_cls(), self.memcache, HardCache(self)), cache_negative_results=True, ) self.cache_chains.update(hardcache=self.hardcache) # I know this sucks, but we need non-request-threads to be # able to reset the caches, so we need them be able to close # around 'cache_chains' without being able to call getattr on # 'g' cache_chains = self.cache_chains.copy() def reset_caches(): for name, chain in cache_chains.iteritems(): chain.reset() chain.stats = CacheStats(self.stats, name) self.reset_caches = reset_caches self.reset_caches() self.startup_timer.intermediate("cache_chains") # try to set the source control revision numbers self.versions = {} r2_root = os.path.dirname(os.path.dirname(self.paths["root"])) r2_gitdir = os.path.join(r2_root, ".git") self.short_version = self.record_repo_version("r2", r2_gitdir) if I18N_PATH: i18n_git_path = os.path.join(os.path.dirname(I18N_PATH), ".git") self.record_repo_version("i18n", i18n_git_path) self.startup_timer.intermediate("revisions")
form = request.form id = int(form.get('good_id')) session['cart'].append(id) flash('добавили в корзину') return redirect(url_for('cart')) else: return redirect(url_for('index')) @app.route('/delete_from_cart', methods=['POST', 'GET']) def delete_from_cart(): if request.method == 'POST': if 'cart' not in session: session['cart'] = [] form = request.form id = int(form.get('good_id')) session['cart'].remove(id) flash('') return redirect(url_for('cart')) #admin decoretor listens_for(Good, 'after_delete')(del_image) admin.add_view(GoodView(Good, db.session)) admin.add_view(CategoryView(Category, db.session)) admin.add_view(OrderView(Order, db.session)) admin.add_view(UserView(User, db.session))
def load_environment(global_conf, app_conf): """Configure the Pylons environment via the ``pylons.config`` object """ # Pylons paths root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) paths = dict(root=root, controllers=os.path.join(root, 'controllers'), static_files=os.path.join(root, 'public'), templates=[os.path.join(root, 'templates')]) # Initialize config with the basic options if is_pylons_0: config = pylons_config else: config = PylonsConfig() config.init_app(global_conf, app_conf, package='fts3rest', paths=paths) config['routes.map'] = make_map(config) config['pylons.app_globals'] = app_globals.Globals(config) config['pylons.h'] = fts3rest.lib.helpers # Setup cache object as early as possible import pylons pylons.cache._push_object(config['pylons.app_globals'].cache) # If fts3.config is set, load configuration from there fts3_config_file = config.get('fts3.config') if fts3_config_file: fts3cfg = fts3_config_load(fts3_config_file) # Let the database be overriden by fts3rest.ini if 'sqlalchemy.url' in config and 'sqlalchemy.url' in fts3cfg: del fts3cfg['sqlalchemy.url'] config.update(fts3cfg) # Setup the SQLAlchemy database engine kwargs = dict() if config['sqlalchemy.url'].startswith('mysql://'): import MySQLdb.cursors kwargs['connect_args'] = {'cursorclass': MySQLdb.cursors.SSCursor} engine = engine_from_config(config, 'sqlalchemy.', pool_recycle=7200, **kwargs) init_model(engine) # Disable for sqlite the isolation level to work around issues with savepoints if config['sqlalchemy.url'].startswith('sqlite'): @event.listens_for(engine, "connect") def do_connect(dbapi_connection, connection_record): dbapi_connection.isolation_level = None # Catch dead connections event.listens_for(engine, 'checkout')(connection_validator) event.listens_for(engine, 'connect')(connection_set_sqlmode) # Mako templating config['pylons.app_globals'].mako_lookup = TemplateLookup( directories=paths['templates'], ) # CONFIGURATION OPTIONS HERE (note: all config options will override # any Pylons config options) return config
def setup(self): self.queues = queues.declare_queues(self) ################# CONFIGURATION # AMQP is required if not self.amqp_host: raise ValueError("amqp_host not set in the .ini") if not self.cassandra_seeds: raise ValueError("cassandra_seeds not set in the .ini") # heavy load mode is read only mode with a different infobar if self.heavy_load_mode: self.read_only_mode = True origin_prefix = self.domain_prefix + "." if self.domain_prefix else "" self.origin = "http://" + origin_prefix + self.domain self.secure_domains = set([urlparse(self.payment_domain).netloc]) self.trusted_domains = set([self.domain]) self.trusted_domains.update(self.authorized_cnames) if self.https_endpoint: https_url = urlparse(self.https_endpoint) self.secure_domains.add(https_url.netloc) self.trusted_domains.add(https_url.hostname) if getattr(self, "oauth_domain", None): self.secure_domains.add(self.oauth_domain) # load the unique hashed names of files under static static_files = os.path.join(self.paths.get("static_files"), "static") names_file_path = os.path.join(static_files, "names.json") if os.path.exists(names_file_path): with open(names_file_path) as handle: self.static_names = json.load(handle) else: self.static_names = {} # if we're a web app running on old uwsgi, force load the logging # config from the file since uwsgi didn't do it for us if not self.running_as_script and self.old_uwsgi_load_logging_config: logging.config.fileConfig(self.config["__file__"]) # make python warnings go through the logging system logging.captureWarnings(capture=True) log = logging.getLogger("reddit") # when we're a script (paster run) just set up super simple logging if self.running_as_script: log.setLevel(logging.INFO) log.addHandler(logging.StreamHandler()) # if in debug mode, override the logging level to DEBUG if self.debug: log.setLevel(logging.DEBUG) # attempt to figure out which pool we're in and add that to the # LogRecords. try: with open("/etc/ec2_asg", "r") as f: pool = f.read().strip() # clean up the pool name since we're putting stuff after "-" pool = pool.partition("-")[0] except IOError: pool = "reddit-app" self.log = logging.LoggerAdapter(log, {"pool": pool}) # make cssutils use the real logging system csslog = logging.getLogger("cssutils") cssutils.log.setLog(csslog) # load the country list countries_file_path = os.path.join(static_files, "countries.json") try: with open(countries_file_path) as handle: self.countries = json.load(handle) self.log.debug("Using countries.json.") except IOError: self.log.warning("Couldn't find countries.json. Using pycountry.") self.countries = get_countries_and_codes() if not self.media_domain: self.media_domain = self.domain if self.media_domain == self.domain: print("Warning: g.media_domain == g.domain. " + "This may give untrusted content access to user cookies") for arg in sys.argv: tokens = arg.split("=") if len(tokens) == 2: k, v = tokens self.log.debug("Overriding g.%s to %s" % (k, v)) setattr(self, k, v) self.reddit_host = socket.gethostname() self.reddit_pid = os.getpid() if hasattr(signal, "SIGUSR1"): # not all platforms have user signals signal.signal(signal.SIGUSR1, thread_dump) self.startup_timer.intermediate("configuration") ################# ZOOKEEPER # for now, zookeeper will be an optional part of the stack. # if it's not configured, we will grab the expected config from the # [live_config] section of the ini file zk_hosts = self.config.get("zookeeper_connection_string") if zk_hosts: from r2.lib.zookeeper import connect_to_zookeeper, LiveConfig, LiveList zk_username = self.config["zookeeper_username"] zk_password = self.config["zookeeper_password"] self.zookeeper = connect_to_zookeeper(zk_hosts, (zk_username, zk_password)) self.live_config = LiveConfig(self.zookeeper, LIVE_CONFIG_NODE) self.throttles = LiveList( self.zookeeper, "/throttles", map_fn=ipaddress.ip_network, reduce_fn=ipaddress.collapse_addresses ) self.banned_domains = LiveDict(self.zookeeper, "/banned-domains", watch=True) else: self.zookeeper = None parser = ConfigParser.RawConfigParser() parser.read([self.config["__file__"]]) self.live_config = extract_live_config(parser, self.plugins) self.throttles = tuple() # immutable since it's not real self.banned_domains = dict() self.startup_timer.intermediate("zookeeper") ################# MEMCACHE num_mc_clients = self.num_mc_clients # the main memcache pool. used for most everything. self.memcache = CMemcache(self.memcaches, num_clients=num_mc_clients) # a smaller pool of caches used only for distributed locks. # TODO: move this to ZooKeeper self.lock_cache = CMemcache(self.lockcaches, num_clients=num_mc_clients) self.make_lock = make_lock_factory(self.lock_cache, self.stats) # memcaches used in front of the permacache CF in cassandra. # XXX: this is a legacy thing; permacache was made when C* didn't have # a row cache. if self.permacache_memcaches: permacache_memcaches = CMemcache(self.permacache_memcaches, num_clients=num_mc_clients) else: permacache_memcaches = None # the stalecache is a memcached local to the current app server used # for data that's frequently fetched but doesn't need to be fresh. if self.stalecaches: stalecaches = CMemcache(self.stalecaches, num_clients=num_mc_clients) else: stalecaches = None # rendercache holds rendered partial templates. rendercaches = CMemcache(self.rendercaches, noreply=True, no_block=True, num_clients=num_mc_clients) # pagecaches hold fully rendered pages pagecaches = CMemcache(self.pagecaches, noreply=True, no_block=True, num_clients=num_mc_clients) self.startup_timer.intermediate("memcache") ################# CASSANDRA keyspace = "reddit" self.cassandra_pools = { "main": StatsCollectingConnectionPool( keyspace, stats=self.stats, logging_name="main", server_list=self.cassandra_seeds, pool_size=self.cassandra_pool_size, timeout=4, max_retries=3, prefill=False, ) } permacache_cf = CassandraCache( "permacache", self.cassandra_pools[self.cassandra_default_pool], read_consistency_level=self.cassandra_rcl, write_consistency_level=self.cassandra_wcl, ) self.startup_timer.intermediate("cassandra") ################# POSTGRES event.listens_for(engine.Engine, "before_cursor_execute")(self.stats.pg_before_cursor_execute) event.listens_for(engine.Engine, "after_cursor_execute")(self.stats.pg_after_cursor_execute) self.dbm = self.load_db_params() self.startup_timer.intermediate("postgres") ################# CHAINS # initialize caches. Any cache-chains built here must be added # to cache_chains (closed around by reset_caches) so that they # can properly reset their local components self.cache_chains = {} localcache_cls = SelfEmptyingCache if self.running_as_script else LocalCache if stalecaches: self.cache = StaleCacheChain(localcache_cls(), stalecaches, self.memcache) else: self.cache = MemcacheChain((localcache_cls(), self.memcache)) self.cache_chains.update(cache=self.cache) self.rendercache = MemcacheChain((localcache_cls(), rendercaches)) self.cache_chains.update(rendercache=self.rendercache) self.pagecache = MemcacheChain((localcache_cls(), pagecaches)) self.cache_chains.update(pagecache=self.pagecache) # the thing_cache is used in tdb_cassandra. self.thing_cache = CacheChain((localcache_cls(),)) self.cache_chains.update(thing_cache=self.thing_cache) self.permacache = CassandraCacheChain( localcache_cls(), permacache_cf, memcache=permacache_memcaches, lock_factory=self.make_lock ) self.cache_chains.update(permacache=self.permacache) # hardcache is used for various things that tend to expire # TODO: replace hardcache w/ cassandra stuff self.hardcache = HardcacheChain((localcache_cls(), self.memcache, HardCache(self)), cache_negative_results=True) self.cache_chains.update(hardcache=self.hardcache) # I know this sucks, but we need non-request-threads to be # able to reset the caches, so we need them be able to close # around 'cache_chains' without being able to call getattr on # 'g' cache_chains = self.cache_chains.copy() def reset_caches(): for name, chain in cache_chains.iteritems(): chain.reset() chain.stats = CacheStats(self.stats, name) self.reset_caches = reset_caches self.reset_caches() self.startup_timer.intermediate("cache_chains") # try to set the source control revision numbers self.versions = {} r2_root = os.path.dirname(os.path.dirname(self.paths["root"])) r2_gitdir = os.path.join(r2_root, ".git") self.short_version = self.record_repo_version("r2", r2_gitdir) if I18N_PATH: i18n_git_path = os.path.join(os.path.dirname(I18N_PATH), ".git") self.record_repo_version("i18n", i18n_git_path) self.startup_timer.intermediate("revisions")
def before_first_req(): event.listens_for(db.engine, 'before_cursor_execute')(receive_before_cursor_execute) event.listens_for(db.engine, 'after_cursor_execute')(receive_after_cursor_execute)
def after_rollback(self, func): event.listens_for(self, 'after_soft_rollback', once=True)(adapt(func))
def setup(self): self.queues = queues.declare_queues(self) ################# CONFIGURATION # AMQP is required if not self.amqp_host: raise ValueError("amqp_host not set in the .ini") # This requirement doesn't *have* to be a requirement, but there are # bugs at the moment that will pop up if you violate it # XXX: get rid of these options. new query cache is always on. if self.write_query_queue and not self.use_query_cache: raise Exception("write_query_queue requires use_query_cache") if not self.cassandra_seeds: raise ValueError("cassandra_seeds not set in the .ini") # heavy load mode is read only mode with a different infobar if self.heavy_load_mode: self.read_only_mode = True origin_prefix = self.domain_prefix + "." if self.domain_prefix else "" self.origin = "http://" + origin_prefix + self.domain self.secure_domains = set([urlparse(self.payment_domain).netloc]) self.trusted_domains = set([self.domain]) self.trusted_domains.update(self.authorized_cnames) if self.https_endpoint: https_url = urlparse(self.https_endpoint) self.secure_domains.add(https_url.netloc) self.trusted_domains.add(https_url.hostname) if getattr(self, 'oauth_domain', None): self.secure_domains.add(self.oauth_domain) # load the unique hashed names of files under static static_files = os.path.join(self.paths.get('static_files'), 'static') names_file_path = os.path.join(static_files, 'names.json') if os.path.exists(names_file_path): with open(names_file_path) as handle: self.static_names = json.load(handle) else: self.static_names = {} #setup the logger self.log = logging.getLogger('reddit') self.log.addHandler(logging.StreamHandler()) if self.debug: self.log.setLevel(logging.DEBUG) else: self.log.setLevel(logging.INFO) # set log level for pycountry which is chatty logging.getLogger('pycountry.db').setLevel(logging.CRITICAL) if not self.media_domain: self.media_domain = self.domain if self.media_domain == self.domain: print ("Warning: g.media_domain == g.domain. " + "This may give untrusted content access to user cookies") for arg in sys.argv: tokens = arg.split("=") if len(tokens) == 2: k, v = tokens self.log.debug("Overriding g.%s to %s" % (k, v)) setattr(self, k, v) self.reddit_host = socket.gethostname() self.reddit_pid = os.getpid() if hasattr(signal, 'SIGUSR1'): # not all platforms have user signals signal.signal(signal.SIGUSR1, thread_dump) self.startup_timer.intermediate("configuration") ################# ZOOKEEPER # for now, zookeeper will be an optional part of the stack. # if it's not configured, we will grab the expected config from the # [live_config] section of the ini file zk_hosts = self.config.get("zookeeper_connection_string") if zk_hosts: from r2.lib.zookeeper import (connect_to_zookeeper, LiveConfig, LiveList) zk_username = self.config["zookeeper_username"] zk_password = self.config["zookeeper_password"] self.zookeeper = connect_to_zookeeper(zk_hosts, (zk_username, zk_password)) self.live_config = LiveConfig(self.zookeeper, LIVE_CONFIG_NODE) self.throttles = LiveList(self.zookeeper, "/throttles", map_fn=ipaddress.ip_network, reduce_fn=ipaddress.collapse_addresses) else: self.zookeeper = None parser = ConfigParser.RawConfigParser() parser.read([self.config["__file__"]]) self.live_config = extract_live_config(parser, self.plugins) self.throttles = tuple() # immutable since it's not real self.startup_timer.intermediate("zookeeper") ################# MEMCACHE num_mc_clients = self.num_mc_clients # the main memcache pool. used for most everything. self.memcache = CMemcache(self.memcaches, num_clients=num_mc_clients) # a smaller pool of caches used only for distributed locks. # TODO: move this to ZooKeeper self.lock_cache = CMemcache(self.lockcaches, num_clients=num_mc_clients) self.make_lock = make_lock_factory(self.lock_cache, self.stats) # memcaches used in front of the permacache CF in cassandra. # XXX: this is a legacy thing; permacache was made when C* didn't have # a row cache. if self.permacache_memcaches: permacache_memcaches = CMemcache(self.permacache_memcaches, num_clients=num_mc_clients) else: permacache_memcaches = None # the stalecache is a memcached local to the current app server used # for data that's frequently fetched but doesn't need to be fresh. if self.stalecaches: stalecaches = CMemcache(self.stalecaches, num_clients=num_mc_clients) else: stalecaches = None # rendercache holds rendered partial templates as well as fully # cached pages. rendercaches = CMemcache( self.rendercaches, noreply=True, no_block=True, num_clients=num_mc_clients, ) self.startup_timer.intermediate("memcache") ################# CASSANDRA keyspace = "reddit" self.cassandra_pools = { "main": StatsCollectingConnectionPool( keyspace, stats=self.stats, logging_name="main", server_list=self.cassandra_seeds, pool_size=self.cassandra_pool_size, timeout=2, max_retries=3, prefill=False ), } permacache_cf = CassandraCache( 'permacache', self.cassandra_pools[self.cassandra_default_pool], read_consistency_level=self.cassandra_rcl, write_consistency_level=self.cassandra_wcl ) self.startup_timer.intermediate("cassandra") ################# POSTGRES event.listens_for(engine.Engine, 'before_cursor_execute')( self.stats.pg_before_cursor_execute) event.listens_for(engine.Engine, 'after_cursor_execute')( self.stats.pg_after_cursor_execute) self.dbm = self.load_db_params() self.startup_timer.intermediate("postgres") ################# CHAINS # initialize caches. Any cache-chains built here must be added # to cache_chains (closed around by reset_caches) so that they # can properly reset their local components self.cache_chains = {} localcache_cls = (SelfEmptyingCache if self.running_as_script else LocalCache) if stalecaches: self.cache = StaleCacheChain( localcache_cls(), stalecaches, self.memcache, ) else: self.cache = MemcacheChain((localcache_cls(), self.memcache)) self.cache_chains.update(cache=self.cache) self.rendercache = MemcacheChain(( localcache_cls(), rendercaches, )) self.cache_chains.update(rendercache=self.rendercache) # the thing_cache is used in tdb_cassandra. self.thing_cache = CacheChain((localcache_cls(),)) self.cache_chains.update(thing_cache=self.thing_cache) self.permacache = CassandraCacheChain( localcache_cls(), permacache_cf, memcache=permacache_memcaches, lock_factory=self.make_lock, ) self.cache_chains.update(permacache=self.permacache) # hardcache is used for various things that tend to expire # TODO: replace hardcache w/ cassandra stuff self.hardcache = HardcacheChain( (localcache_cls(), self.memcache, HardCache(self)), cache_negative_results=True, ) self.cache_chains.update(hardcache=self.hardcache) # I know this sucks, but we need non-request-threads to be # able to reset the caches, so we need them be able to close # around 'cache_chains' without being able to call getattr on # 'g' cache_chains = self.cache_chains.copy() def reset_caches(): for name, chain in cache_chains.iteritems(): chain.reset() chain.stats = CacheStats(self.stats, name) self.reset_caches = reset_caches self.reset_caches() self.startup_timer.intermediate("cache_chains") # try to set the source control revision numbers self.versions = {} r2_root = os.path.dirname(os.path.dirname(self.paths["root"])) r2_gitdir = os.path.join(r2_root, ".git") self.short_version = self.record_repo_version("r2", r2_gitdir) if I18N_PATH: i18n_git_path = os.path.join(os.path.dirname(I18N_PATH), ".git") self.record_repo_version("i18n", i18n_git_path) self.startup_timer.intermediate("revisions")
def __init__(self, engine=None, **kvargs): # setup regex support for sqlalchemy # this hooks up the callback in sqlite listens_for(engine, "begin")(self.hookup_regex)
def __call__(self): connection = create_engine( self.connection_url, connect_args=self.connect_args, **self.engine_args) event.listens_for(connection, 'connect')(self.on_connect) return connection
def create_model(schema_cls, info_logger): # noqa ALLOWED_HOOKS = {'before_create', 'after_create'} name = schema_cls.__name__ methods = dict(schema_cls.__dict__) try: tablename = methods.get('__tablename__', getattr(schema_cls, '__tablename__')) model_methods = {'__tablename__': tablename} except KeyError: raise AttributeError(f'{name} needs to define `__tablename__`') meta = methods.get('Meta') hooks = methods.get('Hooks') declared_fields = methods['_declared_fields'] if hasattr(meta, '__table_args__'): model_methods['__table_args__'] = meta.__table_args__ if hooks: hook_methods = { attr for attr in dir(hooks) if not attr.startswith('__') and callable(getattr(hooks, attr)) } hooks = { attr: getattr(hooks, attr) for attr in hook_methods & ALLOWED_HOOKS } id_constraints = [] indexes = {} for fieldname, field_attrs in declared_fields.items(): if isinstance(field_attrs, fields.Field): if isinstance(field_attrs, postschema_fields.AutoSessionField): perms = getattr(methods.get('Public'), 'permissions', None) if perms and hasattr( perms, 'post') and 'primary_key' in field_attrs.metadata: # auto-injected primary key is based on the session context, # so we can't allow public posts. raise AttributeError( f"{name} can't include 'post' as a public permission attribute" ) metadata = field_attrs.metadata try: field_instance = metadata.pop('sqlfield', None) or metadata['fk'] if not field_instance: continue except KeyError: # skip fields with no sql bindings continue except AttributeError: raise AttributeError( f'Schema field `{fieldname}` needs to define a SQLAlchemy field instance' ) translated = {} default_value = field_attrs.default if default_value != missing: translated['server_default'] = default_value() if callable( default_value) else default_value args = [] if 'fk' in metadata: args.append(metadata['fk']) if 'autoincrement' in metadata: args.append(metadata.pop('autoincrement')) metadict = metadata.copy() metadict.pop('fk', None) metadict.pop('read_only', None) metadict.pop('is_aware', None) if metadict.pop('gist_index', False): indexes[f'{fieldname}_gist_idx'] = [ tablename, fieldname, 'gist' ] if metadict.pop('gin_index', False): indexes[f'{fieldname}_gin_idx'] = [tablename, fieldname, 'gin'] identity_constraint = metadict.pop('identity_constraint', {}) model_methods[fieldname] = sql.Column(field_instance, *args, **metadict, **translated) # parse identity_constraint if identity_constraint: identity_constraint['target_table_local_ref'] = fieldname identity_constraint['tablename'] = tablename id_constraints.append(identity_constraint) modelname = name + 'Model' new_model = type(modelname, (Base, ), model_methods) if hooks: for method, method_fn in hooks.items(): event.listens_for(Base.metadata.tables[tablename], method)(method_fn) for index_name, index_items in indexes.items(): tablename, fieldname, index_type = index_items add_index(Base.metadata, index_name, tablename, fieldname, index_type) for id_constraint in id_constraints: add_identity_triggers(Base.metadata, id_constraint) info_logger.debug(f"- created model `{modelname}`") return new_model
def setup(self): # heavy load mode is read only mode with a different infobar if self.heavy_load_mode: self.read_only_mode = True if hasattr(signal, 'SIGUSR1'): # not all platforms have user signals signal.signal(signal.SIGUSR1, thread_dump) # initialize caches. Any cache-chains built here must be added # to cache_chains (closed around by reset_caches) so that they # can properly reset their local components localcache_cls = (SelfEmptyingCache if self.running_as_script else LocalCache) num_mc_clients = self.num_mc_clients self.cache_chains = {} # for now, zookeeper will be an optional part of the stack. # if it's not configured, we will grab the expected config from the # [live_config] section of the ini file zk_hosts = self.config.get("zookeeper_connection_string") if zk_hosts: from r2.lib.zookeeper import (connect_to_zookeeper, LiveConfig, LiveList) zk_username = self.config["zookeeper_username"] zk_password = self.config["zookeeper_password"] self.zookeeper = connect_to_zookeeper(zk_hosts, (zk_username, zk_password)) self.live_config = LiveConfig(self.zookeeper, LIVE_CONFIG_NODE) self.throttles = LiveList(self.zookeeper, "/throttles", map_fn=ipaddress.ip_network, reduce_fn=ipaddress.collapse_addresses) else: self.zookeeper = None parser = ConfigParser.RawConfigParser() parser.read([self.config["__file__"]]) self.live_config = extract_live_config(parser, self.plugins) self.throttles = tuple() # immutable since it's not real self.memcache = CMemcache(self.memcaches, num_clients = num_mc_clients) self.lock_cache = CMemcache(self.lockcaches, num_clients=num_mc_clients) self.stats = Stats(self.config.get('statsd_addr'), self.config.get('statsd_sample_rate')) event.listens_for(engine.Engine, 'before_cursor_execute')( self.stats.pg_before_cursor_execute) event.listens_for(engine.Engine, 'after_cursor_execute')( self.stats.pg_after_cursor_execute) self.make_lock = make_lock_factory(self.lock_cache, self.stats) if not self.cassandra_seeds: raise ValueError("cassandra_seeds not set in the .ini") keyspace = "reddit" self.cassandra_pools = { "main": StatsCollectingConnectionPool( keyspace, stats=self.stats, logging_name="main", server_list=self.cassandra_seeds, pool_size=self.cassandra_pool_size, timeout=2, max_retries=3, prefill=False ), } perma_memcache = (CMemcache(self.permacache_memcaches, num_clients = num_mc_clients) if self.permacache_memcaches else None) self.permacache = CassandraCacheChain(localcache_cls(), CassandraCache('permacache', self.cassandra_pools[self.cassandra_default_pool], read_consistency_level = self.cassandra_rcl, write_consistency_level = self.cassandra_wcl), memcache = perma_memcache, lock_factory = self.make_lock) self.cache_chains.update(permacache=self.permacache) # hardcache is done after the db info is loaded, and then the # chains are reset to use the appropriate initial entries if self.stalecaches: self.cache = StaleCacheChain(localcache_cls(), CMemcache(self.stalecaches, num_clients=num_mc_clients), self.memcache) else: self.cache = MemcacheChain((localcache_cls(), self.memcache)) self.cache_chains.update(cache=self.cache) self.rendercache = MemcacheChain((localcache_cls(), CMemcache(self.rendercaches, noreply=True, no_block=True, num_clients = num_mc_clients))) self.cache_chains.update(rendercache=self.rendercache) self.thing_cache = CacheChain((localcache_cls(),)) self.cache_chains.update(thing_cache=self.thing_cache) #load the database info self.dbm = self.load_db_params() # can't do this until load_db_params() has been called self.hardcache = HardcacheChain((localcache_cls(), self.memcache, HardCache(self)), cache_negative_results = True) self.cache_chains.update(hardcache=self.hardcache) # I know this sucks, but we need non-request-threads to be # able to reset the caches, so we need them be able to close # around 'cache_chains' without being able to call getattr on # 'g' cache_chains = self.cache_chains.copy() def reset_caches(): for name, chain in cache_chains.iteritems(): chain.reset() chain.stats = CacheStats(self.stats, name) self.reset_caches = reset_caches self.reset_caches() # set the modwindow self.MODWINDOW = timedelta(self.MODWINDOW) self.REDDIT_MAIN = bool(os.environ.get('REDDIT_MAIN')) origin_prefix = self.domain_prefix + "." if self.domain_prefix else "" self.origin = "http://" + origin_prefix + self.domain self.secure_domains = set([urlparse(self.payment_domain).netloc]) self.trusted_domains = set([self.domain]) self.trusted_domains.update(self.authorized_cnames) if self.https_endpoint: https_url = urlparse(self.https_endpoint) self.secure_domains.add(https_url.netloc) self.trusted_domains.add(https_url.hostname) if getattr(self, 'oauth_domain', None): self.secure_domains.add(self.oauth_domain) # load the unique hashed names of files under static static_files = os.path.join(self.paths.get('static_files'), 'static') names_file_path = os.path.join(static_files, 'names.json') if os.path.exists(names_file_path): with open(names_file_path) as handle: self.static_names = json.load(handle) else: self.static_names = {} #setup the logger self.log = logging.getLogger('reddit') self.log.addHandler(logging.StreamHandler()) if self.debug: self.log.setLevel(logging.DEBUG) else: self.log.setLevel(logging.INFO) # set log level for pycountry which is chatty logging.getLogger('pycountry.db').setLevel(logging.CRITICAL) if not self.media_domain: self.media_domain = self.domain if self.media_domain == self.domain: print ("Warning: g.media_domain == g.domain. " + "This may give untrusted content access to user cookies") self.reddit_host = socket.gethostname() self.reddit_pid = os.getpid() for arg in sys.argv: tokens = arg.split("=") if len(tokens) == 2: k, v = tokens self.log.debug("Overriding g.%s to %s" % (k, v)) setattr(self, k, v) #if we're going to use the query_queue, we need amqp if self.write_query_queue and not self.amqp_host: raise Exception("amqp_host must be defined to use the query queue") # This requirement doesn't *have* to be a requirement, but there are # bugs at the moment that will pop up if you violate it if self.write_query_queue and not self.use_query_cache: raise Exception("write_query_queue requires use_query_cache") # try to set the source control revision numbers self.versions = {} r2_root = os.path.dirname(os.path.dirname(self.paths["root"])) r2_gitdir = os.path.join(r2_root, ".git") self.short_version = self.record_repo_version("r2", r2_gitdir) if I18N_PATH: i18n_git_path = os.path.join(os.path.dirname(I18N_PATH), ".git") self.record_repo_version("i18n", i18n_git_path) if self.log_start: self.log.error("reddit app %s:%s started %s at %s" % (self.reddit_host, self.reddit_pid, self.short_version, datetime.now()))
DB_PATH = os.getenv("FUNGPHY_DB") or "fungphy.db" engine = create_engine(f"sqlite:///{DB_PATH}") session = scoped_session( sessionmaker(bind=engine, autocommit=False, autoflush=False)) naming_convention = { "ix": "ix_%(column_0_label)s", "uq": "uq_%(table_name)s_%(column_0_name)s", "ck": "ck_%(table_name)s_%(column_0_name)s", "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s", "pk": "pk_%(table_name)s" } meta = MetaData(naming_convention=naming_convention) Base = declarative_base(metadata=meta) Base.query = session.query_property() def init_db(): import fungphy.models configure_mappers() Base.metadata.create_all(engine) event.listens_for(Engine, "connect") def set_sqlite_pragma(dbapi_connection, connection_record): cursor = dbapi_connection.cursor() cursor.execute("PRAGMA foreign_keys=ON") cursor.close()
def polymorphic_fallback(mapper_klass): event.listens_for(mapper_klass, 'mapper_configured')(receive_mapper_configured) return mapper_klass
def setup(self): self.queues = queues.declare_queues(self) ################# PROVIDERS self.media_provider = select_provider( self.config, self.pkg_resources_working_set, "r2.provider.media", self.media_provider, ) self.startup_timer.intermediate("providers") ################# CONFIGURATION # AMQP is required if not self.amqp_host: raise ValueError("amqp_host not set in the .ini") if not self.cassandra_seeds: raise ValueError("cassandra_seeds not set in the .ini") # heavy load mode is read only mode with a different infobar if self.heavy_load_mode: self.read_only_mode = True origin_prefix = self.domain_prefix + "." if self.domain_prefix else "" self.origin = "http://" + origin_prefix + self.domain self.trusted_domains = set([self.domain]) if self.https_endpoint: https_url = urlparse(self.https_endpoint) self.trusted_domains.add(https_url.hostname) # load the unique hashed names of files under static static_files = os.path.join(self.paths.get('static_files'), 'static') names_file_path = os.path.join(static_files, 'names.json') if os.path.exists(names_file_path): with open(names_file_path) as handle: self.static_names = json.load(handle) else: self.static_names = {} # make python warnings go through the logging system logging.captureWarnings(capture=True) log = logging.getLogger('reddit') # when we're a script (paster run) just set up super simple logging if self.running_as_script: log.setLevel(logging.INFO) log.addHandler(logging.StreamHandler()) # if in debug mode, override the logging level to DEBUG if self.debug: log.setLevel(logging.DEBUG) # attempt to figure out which pool we're in and add that to the # LogRecords. try: with open("/etc/ec2_asg", "r") as f: pool = f.read().strip() # clean up the pool name since we're putting stuff after "-" pool = pool.partition("-")[0] except IOError: pool = "reddit-app" self.log = logging.LoggerAdapter(log, {"pool": pool}) # make cssutils use the real logging system csslog = logging.getLogger("cssutils") cssutils.log.setLog(csslog) # set locations self.locations = {} if not self.media_domain: self.media_domain = self.domain if self.media_domain == self.domain: print >> sys.stderr, ( "Warning: g.media_domain == g.domain. " + "This may give untrusted content access to user cookies") for arg in sys.argv: tokens = arg.split("=") if len(tokens) == 2: k, v = tokens self.log.debug("Overriding g.%s to %s" % (k, v)) setattr(self, k, v) self.reddit_host = socket.gethostname() self.reddit_pid = os.getpid() if hasattr(signal, 'SIGUSR1'): # not all platforms have user signals signal.signal(signal.SIGUSR1, thread_dump) locale.setlocale(locale.LC_ALL, self.locale) # Pre-calculate ratelimit values self.RL_RESET_SECONDS = self.config["RL_RESET_MINUTES"] * 60 self.RL_MAX_REQS = int(self.config["RL_AVG_REQ_PER_SEC"] * self.RL_RESET_SECONDS) self.RL_OAUTH_RESET_SECONDS = self.config["RL_OAUTH_RESET_MINUTES"] * 60 self.RL_OAUTH_MAX_REQS = int(self.config["RL_OAUTH_AVG_REQ_PER_SEC"] * self.RL_OAUTH_RESET_SECONDS) self.startup_timer.intermediate("configuration") ################# ZOOKEEPER # for now, zookeeper will be an optional part of the stack. # if it's not configured, we will grab the expected config from the # [live_config] section of the ini file zk_hosts = self.config.get("zookeeper_connection_string") if zk_hosts: from r2.lib.zookeeper import (connect_to_zookeeper, LiveConfig, LiveList) zk_username = self.config["zookeeper_username"] zk_password = self.config["zookeeper_password"] self.zookeeper = connect_to_zookeeper(zk_hosts, (zk_username, zk_password)) self.live_config = LiveConfig(self.zookeeper, LIVE_CONFIG_NODE) self.secrets = fetch_secrets(self.zookeeper) self.throttles = LiveList(self.zookeeper, "/throttles", map_fn=ipaddress.ip_network, reduce_fn=ipaddress.collapse_addresses) else: self.zookeeper = None parser = ConfigParser.RawConfigParser() parser.optionxform = str parser.read([self.config["__file__"]]) self.live_config = extract_live_config(parser, self.plugins) self.secrets = extract_secrets(parser) self.throttles = tuple() # immutable since it's not real self.startup_timer.intermediate("zookeeper") ################# MEMCACHE num_mc_clients = self.num_mc_clients # the main memcache pool. used for most everything. self.memcache = CMemcache( self.memcaches, min_compress_len=50 * 1024, num_clients=num_mc_clients, ) # a pool just used for @memoize results memoizecaches = CMemcache( self.memoizecaches, min_compress_len=50 * 1024, num_clients=num_mc_clients, ) # a pool just for srmember rels srmembercaches = CMemcache( self.srmembercaches, min_compress_len=50 * 1024, num_clients=num_mc_clients, ) ratelimitcaches = CMemcache( self.ratelimitcaches, min_compress_len=96, num_clients=num_mc_clients, ) # a smaller pool of caches used only for distributed locks. # TODO: move this to ZooKeeper self.lock_cache = CMemcache(self.lockcaches, num_clients=num_mc_clients) self.make_lock = make_lock_factory(self.lock_cache, self.stats) # memcaches used in front of the permacache CF in cassandra. # XXX: this is a legacy thing; permacache was made when C* didn't have # a row cache. if self.permacache_memcaches: permacache_memcaches = CMemcache(self.permacache_memcaches, min_compress_len=50 * 1024, num_clients=num_mc_clients) else: permacache_memcaches = None # the stalecache is a memcached local to the current app server used # for data that's frequently fetched but doesn't need to be fresh. if self.stalecaches: stalecaches = CMemcache(self.stalecaches, num_clients=num_mc_clients) else: stalecaches = None # rendercache holds rendered partial templates. rendercaches = CMemcache( self.rendercaches, noreply=True, no_block=True, num_clients=num_mc_clients, min_compress_len=480, ) # pagecaches hold fully rendered pages pagecaches = CMemcache( self.pagecaches, noreply=True, no_block=True, num_clients=num_mc_clients, min_compress_len=1400, ) self.startup_timer.intermediate("memcache") ################# CASSANDRA keyspace = "reddit" self.cassandra_pools = { "main": StatsCollectingConnectionPool(keyspace, stats=self.stats, logging_name="main", server_list=self.cassandra_seeds, pool_size=self.cassandra_pool_size, timeout=4, max_retries=3, prefill=False), } permacache_cf = CassandraCache( 'permacache', self.cassandra_pools[self.cassandra_default_pool], read_consistency_level=self.cassandra_rcl, write_consistency_level=self.cassandra_wcl) self.startup_timer.intermediate("cassandra") ################# POSTGRES event.listens_for(engine.Engine, 'before_cursor_execute')( self.stats.pg_before_cursor_execute) event.listens_for(engine.Engine, 'after_cursor_execute')( self.stats.pg_after_cursor_execute) self.dbm = self.load_db_params() self.startup_timer.intermediate("postgres") ################# CHAINS # initialize caches. Any cache-chains built here must be added # to cache_chains (closed around by reset_caches) so that they # can properly reset their local components cache_chains = {} localcache_cls = (SelfEmptyingCache if self.running_as_script else LocalCache) if stalecaches: self.cache = StaleCacheChain( localcache_cls(), stalecaches, self.memcache, ) else: self.cache = MemcacheChain((localcache_cls(), self.memcache)) cache_chains.update(cache=self.cache) if stalecaches: self.memoizecache = StaleCacheChain( localcache_cls(), stalecaches, memoizecaches, ) else: self.memoizecache = MemcacheChain( (localcache_cls(), memoizecaches)) cache_chains.update(memoizecache=self.memoizecache) if stalecaches: self.srmembercache = StaleCacheChain( localcache_cls(), stalecaches, srmembercaches, ) else: self.srmembercache = MemcacheChain( (localcache_cls(), srmembercaches)) cache_chains.update(srmembercache=self.srmembercache) self.ratelimitcache = MemcacheChain( (localcache_cls(), ratelimitcaches)) cache_chains.update(ratelimitcaches=self.ratelimitcache) self.rendercache = MemcacheChain(( localcache_cls(), rendercaches, )) cache_chains.update(rendercache=self.rendercache) self.pagecache = MemcacheChain(( localcache_cls(), pagecaches, )) cache_chains.update(pagecache=self.pagecache) # the thing_cache is used in tdb_cassandra. self.thing_cache = CacheChain((localcache_cls(), )) cache_chains.update(thing_cache=self.thing_cache) self.permacache = CassandraCacheChain( localcache_cls(), permacache_cf, memcache=permacache_memcaches, lock_factory=self.make_lock, ) cache_chains.update(permacache=self.permacache) # hardcache is used for various things that tend to expire # TODO: replace hardcache w/ cassandra stuff self.hardcache = HardcacheChain( (localcache_cls(), self.memcache, HardCache(self)), cache_negative_results=True, ) cache_chains.update(hardcache=self.hardcache) # I know this sucks, but we need non-request-threads to be # able to reset the caches, so we need them be able to close # around 'cache_chains' without being able to call getattr on # 'g' def reset_caches(): for name, chain in cache_chains.iteritems(): chain.reset() chain.stats = CacheStats(self.stats, name) self.cache_chains = cache_chains self.reset_caches = reset_caches self.reset_caches() self.startup_timer.intermediate("cache_chains") # try to set the source control revision numbers self.versions = {} r2_root = os.path.dirname(os.path.dirname(self.paths["root"])) r2_gitdir = os.path.join(r2_root, ".git") self.short_version = self.record_repo_version("r2", r2_gitdir) if I18N_PATH: i18n_git_path = os.path.join(os.path.dirname(I18N_PATH), ".git") self.record_repo_version("i18n", i18n_git_path) self.startup_timer.intermediate("revisions")