def command(self): logging.config.fileConfig(self.path_to_ini_file) from pylons import config add_cache(config) engine = engine_from_config(config, 'sqlalchemy.db1.') init_model(engine) index_location = config['index_dir'] repo_location = self.options.repo_location \ if self.options.repo_location else RepoModel().repos_path repo_list = map(strip, self.options.repo_list.split(',')) \ if self.options.repo_list else None load_rcextensions(config['here']) #====================================================================== # WHOOSH DAEMON #====================================================================== from rhodecode.lib.pidlock import LockHeld, DaemonLock from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon try: l = DaemonLock(file_=jn(dn(dn(index_location)), 'make_index.lock')) WhooshIndexingDaemon(index_location=index_location, repo_location=repo_location, repo_list=repo_list,)\ .run(full_index=self.options.full_index) l.release() except LockHeld: sys.exit(1)
def command(self): logging.config.fileConfig(self.path_to_ini_file) from pylons import config add_cache(config) engine = engine_from_config(config, 'sqlalchemy.db1.') init_model(engine) index_location = config['index_dir'] repo_location = self.options.repo_location \ if self.options.repo_location else RepoModel().repos_path repo_list = map(strip, self.options.repo_list.split(',')) \ if self.options.repo_list else None repo_update_list = map(strip, self.options.repo_update_list.split(',')) \ if self.options.repo_update_list else None load_rcextensions(config['here']) #====================================================================== # WHOOSH DAEMON #====================================================================== from rhodecode.lib.pidlock import LockHeld, DaemonLock from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon try: l = DaemonLock(file_=jn(dn(dn(index_location)), 'make_index.lock')) WhooshIndexingDaemon(index_location=index_location, repo_location=repo_location, repo_list=repo_list, repo_update_list=repo_update_list)\ .run(full_index=self.options.full_index) l.release() except LockHeld: sys.exit(1)
def init_db(self, SESSION=None): if SESSION: self.sa = SESSION else: # init new sessions engine = create_engine(self.dburi, echo=self.log_sql) init_model(engine) self.sa = Session()
def load_environment(global_conf, app_conf, initial=False): """Configure the Pylons environment via the ``pylons.config`` object """ config = PylonsConfig() # Pylons paths root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) paths = dict(root=root, controllers=os.path.join(root, 'controllers'), static_files=os.path.join(root, 'public'), templates=[os.path.join(root, 'templates')]) # Initialize config with the basic options config.init_app(global_conf, app_conf, package='rhodecode', paths=paths) config['routes.map'] = make_map(config) config['pylons.app_globals'] = app_globals.Globals(config) config['pylons.h'] = rhodecode.lib.helpers # Setup cache object as early as possible import pylons pylons.cache._push_object(config['pylons.app_globals'].cache) # Create the Mako TemplateLookup, with the default auto-escaping config['pylons.app_globals'].mako_lookup = TemplateLookup( directories=paths['templates'], error_handler=handle_mako_error, module_directory=os.path.join(app_conf['cache_dir'], 'templates'), input_encoding='utf-8', default_filters=['escape'], imports=['from webhelpers.html import escape']) #sets the c attribute access when don't existing attribute are accessed config['pylons.strict_tmpl_context'] = True test = os.path.split(config['__file__'])[-1] == 'test.ini' if test: from rhodecode.lib.utils import create_test_env, create_test_index from rhodecode.tests import TESTS_TMP_PATH create_test_env(TESTS_TMP_PATH, config) create_test_index(TESTS_TMP_PATH, config, True) #MULTIPLE DB configs # Setup the SQLAlchemy database engine sa_engine_db1 = engine_from_config(config, 'sqlalchemy.db1.') init_model(sa_engine_db1) repos_path = make_ui('db').configitems('paths')[0][1] repo2db_mapper(ScmModel().repo_scan(repos_path)) set_available_permissions(config) config['base_path'] = repos_path set_rhodecode_config(config) # CONFIGURATION OPTIONS HERE (note: all config options will override # any Pylons config options) return config
def _init_session(self): """ Inits SqlAlchemy Session """ logging.config.fileConfig(self.path_to_ini_file) from pylons import config from rhodecode.model import init_model from rhodecode.lib.utils2 import engine_from_config #get to remove repos !! add_cache(config) engine = engine_from_config(config, 'sqlalchemy.db1.') init_model(engine)
def command(self): logging.config.fileConfig(self.path_to_ini_file) from pylons import config #get to remove repos !! add_cache(config) engine = engine_from_config(config, 'sqlalchemy.db1.') init_model(engine) repos_location = RhodeCodeUi.get_repos_location() to_remove = [] for dn, dirs, f in os.walk(safe_str(repos_location)): for loc in dirs: if REMOVED_REPO_PAT.match(loc): to_remove.append([loc, self._extract_date(loc)]) #filter older than (if present)! now = datetime.datetime.now() older_than = self.options.older_than if older_than: to_remove_filtered = [] older_than_date = self._parse_older_than(older_than) for name, date_ in to_remove: repo_age = now - date_ if repo_age > older_than_date: to_remove_filtered.append([name, date_]) to_remove = to_remove_filtered print >> sys.stdout, 'removing [%s] deleted repos older than %s[%s]' \ % (len(to_remove), older_than, older_than_date) else: print >> sys.stdout, 'removing all [%s] deleted repos' \ % len(to_remove) if self.options.dont_ask or not to_remove: # don't ask just remove ! remove = True else: remove = ask_ok('are you sure to remove listed repos \n%s [y/n]?' % ', \n'.join(['%s removed on %s' % (safe_str(x[0]), safe_str(x[1])) for x in to_remove])) if remove: for name, date_ in to_remove: print >> sys.stdout, 'removing repository %s' % name shutil.rmtree(os.path.join(repos_location, name)) else: print 'nothing done exiting...' sys.exit(0)
def command(self): logging.config.fileConfig(self.path_to_ini_file) from pylons import config #get to remove repos !! add_cache(config) engine = engine_from_config(config, 'sqlalchemy.db1.') init_model(engine) repo_update_list = map(string.strip, self.options.repo_update_list.split(',')) \ if self.options.repo_update_list else None if repo_update_list: repo_list = Repository.query().filter(Repository.repo_name.in_(repo_update_list)) else: repo_list = Repository.getAll() for repo in repo_list: last_change = repo.scm_instance.last_change repo.update_last_change(last_change)
def init_db(self): engine = create_engine(self.dburi, echo=self.log_sql) init_model(engine) self.sa = Session()
def init_db(self): engine = create_engine(self.dburi, echo=self.log_sql) init_model(engine) self.sa = meta.Session()
def get_session(): engine = engine_from_config(conf, 'sqlalchemy.db1.') init_model(engine) sa = meta.Session() return sa
def handle_git_receive(repo_path, revs, env, hook_type='post'): """ A really hacky method that is runned by git post-receive hook and logs an push action together with pushed revisions. It's executed by subprocess thus needs all info to be able to create a on the fly pylons enviroment, connect to database and run the logging code. Hacky as sh*t but works. :param repo_path: :param revs: :param env: """ from paste.deploy import appconfig from sqlalchemy import engine_from_config from rhodecode.config.environment import load_environment from rhodecode.model import init_model from rhodecode.model.db import RhodeCodeUi from rhodecode.lib.utils import make_ui extras = _extract_extras(env) path, ini_name = os.path.split(extras['config']) conf = appconfig('config:%s' % ini_name, relative_to=path) load_environment(conf.global_conf, conf.local_conf) engine = engine_from_config(conf, 'sqlalchemy.db1.') init_model(engine) baseui = make_ui('db') # fix if it's not a bare repo if repo_path.endswith(os.sep + '.git'): repo_path = repo_path[:-5] repo = Repository.get_by_full_path(repo_path) if not repo: raise OSError('Repository %s not found in database' % (safe_str(repo_path))) _hooks = dict(baseui.configitems('hooks')) or {} if hook_type == 'pre': repo = repo.scm_instance else: #post push shouldn't use the cached instance never repo = repo.scm_instance_no_cache() if hook_type == 'pre': pre_push(baseui, repo) # if push hook is enabled via web interface elif hook_type == 'post' and _hooks.get(RhodeCodeUi.HOOK_PUSH): rev_data = [] for l in revs: old_rev, new_rev, ref = l.split(' ') _ref_data = ref.split('/') if _ref_data[1] in ['tags', 'heads']: rev_data.append({ 'old_rev': old_rev, 'new_rev': new_rev, 'ref': ref, 'type': _ref_data[1], 'name': _ref_data[2].strip() }) git_revs = [] for push_ref in rev_data: _type = push_ref['type'] if _type == 'heads': if push_ref['old_rev'] == EmptyChangeset().raw_id: cmd = "for-each-ref --format='%(refname)' 'refs/heads/*'" heads = repo.run_git_command(cmd)[0] heads = heads.replace(push_ref['ref'], '') heads = ' '.join( map(lambda c: c.strip('\n').strip(), heads.splitlines())) cmd = (('log %(new_rev)s' % push_ref) + ' --reverse --pretty=format:"%H" --not ' + heads) git_revs += repo.run_git_command(cmd)[0].splitlines() elif push_ref['new_rev'] == EmptyChangeset().raw_id: #delete branch case git_revs += ['delete_branch=>%s' % push_ref['name']] else: cmd = (('log %(old_rev)s..%(new_rev)s' % push_ref) + ' --reverse --pretty=format:"%H"') git_revs += repo.run_git_command(cmd)[0].splitlines() elif _type == 'tags': git_revs += ['tag=>%s' % push_ref['name']] log_push_action(baseui, repo, _git_revs=git_revs)
def load_environment(global_conf, app_conf, initial=False): """ Configure the Pylons environment via the ``pylons.config`` object """ config = PylonsConfig() # Pylons paths root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) paths = dict(root=root, controllers=os.path.join(root, 'controllers'), static_files=os.path.join(root, 'public'), templates=[os.path.join(root, 'templates')]) # Initialize config with the basic options config.init_app(global_conf, app_conf, package='rhodecode', paths=paths) # store some globals into rhodecode rhodecode.CELERY_ON = str2bool(config['app_conf'].get('use_celery')) rhodecode.CELERY_EAGER = str2bool( config['app_conf'].get('celery.always.eager')) config['routes.map'] = make_map(config) config['pylons.app_globals'] = app_globals.Globals(config) config['pylons.h'] = helpers rhodecode.CONFIG = config load_rcextensions(root_path=config['here']) # Setup cache object as early as possible import pylons pylons.cache._push_object(config['pylons.app_globals'].cache) # Create the Mako TemplateLookup, with the default auto-escaping config['pylons.app_globals'].mako_lookup = TemplateLookup( directories=paths['templates'], error_handler=handle_mako_error, module_directory=os.path.join(app_conf['cache_dir'], 'templates'), input_encoding='utf-8', default_filters=['escape'], imports=['from webhelpers.html import escape']) # sets the c attribute access when don't existing attribute are accessed config['pylons.strict_tmpl_context'] = True test = os.path.split(config['__file__'])[-1] == 'test.ini' if test: if os.environ.get('TEST_DB'): # swap config if we pass enviroment variable config['sqlalchemy.db1.url'] = os.environ.get('TEST_DB') from rhodecode.lib.utils import create_test_env, create_test_index from rhodecode.tests import TESTS_TMP_PATH # set RC_NO_TMP_PATH=1 to disable re-creating the database and # test repos if not int(os.environ.get('RC_NO_TMP_PATH', 0)): create_test_env(TESTS_TMP_PATH, config) # set RC_WHOOSH_TEST_DISABLE=1 to disable whoosh index during tests if not int(os.environ.get('RC_WHOOSH_TEST_DISABLE', 0)): create_test_index(TESTS_TMP_PATH, config, True) DbManage.check_waitress() # MULTIPLE DB configs # Setup the SQLAlchemy database engine sa_engine_db1 = engine_from_config(config, 'sqlalchemy.db1.') init_model(sa_engine_db1) set_available_permissions(config) repos_path = make_ui('db').configitems('paths')[0][1] config['base_path'] = repos_path set_rhodecode_config(config) instance_id = rhodecode.CONFIG.get('instance_id') if instance_id == '*': instance_id = '%s-%s' % (os.uname()[1], os.getpid()) rhodecode.CONFIG['instance_id'] = instance_id # CONFIGURATION OPTIONS HERE (note: all config options will override # any Pylons config options) # store config reference into our module to skip import magic of # pylons rhodecode.CONFIG.update(config) set_vcs_config(rhodecode.CONFIG) #check git version check_git_version() if str2bool(config.get('initial_repo_scan', True)): repo2db_mapper(ScmModel().repo_scan(repos_path), remove_obsolete=False, install_git_hook=False) return config
def get_session(): engine = engine_from_config(conf, 'sqlalchemy.db1.') init_model(engine) sa = meta.Session return sa
def load_environment(global_conf, app_conf, initial=False): """ Configure the Pylons environment via the ``pylons.config`` object """ config = PylonsConfig() # Pylons paths root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) paths = dict(root=root, controllers=os.path.join(root, 'controllers'), static_files=os.path.join(root, 'public'), templates=[os.path.join(root, 'templates')]) # Initialize config with the basic options config.init_app(global_conf, app_conf, package='rhodecode', paths=paths) # store some globals into rhodecode rhodecode.CELERY_ON = str2bool(config['app_conf'].get('use_celery')) rhodecode.CELERY_EAGER = str2bool( config['app_conf'].get('celery.always.eager')) config['routes.map'] = make_map(config) config['pylons.app_globals'] = app_globals.Globals(config) config['pylons.h'] = helpers rhodecode.CONFIG = config load_rcextensions(root_path=config['here']) # Setup cache object as early as possible import pylons pylons.cache._push_object(config['pylons.app_globals'].cache) # Create the Mako TemplateLookup, with the default auto-escaping config['pylons.app_globals'].mako_lookup = TemplateLookup( directories=paths['templates'], error_handler=handle_mako_error, module_directory=os.path.join(app_conf['cache_dir'], 'templates'), input_encoding='utf-8', default_filters=['escape'], imports=['from webhelpers.html import escape']) # sets the c attribute access when don't existing attribute are accessed config['pylons.strict_tmpl_context'] = True test = os.path.split(config['__file__'])[-1] == 'test.ini' if test: from rhodecode.lib.utils import create_test_env, create_test_index from rhodecode.tests import TESTS_TMP_PATH create_test_env(TESTS_TMP_PATH, config) create_test_index(TESTS_TMP_PATH, config, True) # MULTIPLE DB configs # Setup the SQLAlchemy database engine sa_engine_db1 = engine_from_config(config, 'sqlalchemy.db1.') init_model(sa_engine_db1) repos_path = make_ui('db').configitems('paths')[0][1] repo2db_mapper(ScmModel().repo_scan(repos_path)) set_available_permissions(config) config['base_path'] = repos_path set_rhodecode_config(config) # CONFIGURATION OPTIONS HERE (note: all config options will override # any Pylons config options) # store config reference into our module to skip import magic of # pylons rhodecode.CONFIG.update(config) return config
rel_path = dn(dn(dn(os.path.abspath(__file__)))) conf = appconfig('config:%s' % sys.argv[1], relative_to=rel_path) load_environment(conf.global_conf, conf.local_conf) add_cache(conf) USER = '******' PASS = '******' HOST = '127.0.0.1:5000' DEBUG = False print 'DEBUG:', DEBUG log = logging.getLogger(__name__) engine = engine_from_config(conf, 'sqlalchemy.db1.') init_model(engine) sa = meta.Session class Command(object): def __init__(self, cwd): self.cwd = cwd def execute(self, cmd, *args): """Runs command on the system with given ``args``. """ command = cmd + ' ' + ' '.join(args) log.debug('Executing %s' % command) if DEBUG: print command
def get_session(): if CELERY_ON: engine = engine_from_config(config, 'sqlalchemy.db1.') init_model(engine) sa = meta.Session() return sa
def load_environment(global_conf, app_conf, initial=False): """ Configure the Pylons environment via the ``pylons.config`` object """ config = PylonsConfig() # Pylons paths root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) paths = dict( root=root, controllers=os.path.join(root, 'controllers'), static_files=os.path.join(root, 'public'), templates=[os.path.join(root, 'templates')] ) # Initialize config with the basic options config.init_app(global_conf, app_conf, package='rhodecode', paths=paths) # store some globals into rhodecode rhodecode.CELERY_ON = str2bool(config['app_conf'].get('use_celery')) rhodecode.CELERY_EAGER = str2bool(config['app_conf'].get('celery.always.eager')) config['routes.map'] = make_map(config) config['pylons.app_globals'] = app_globals.Globals(config) config['pylons.h'] = helpers rhodecode.CONFIG = config load_rcextensions(root_path=config['here']) # Setup cache object as early as possible import pylons pylons.cache._push_object(config['pylons.app_globals'].cache) # Create the Mako TemplateLookup, with the default auto-escaping config['pylons.app_globals'].mako_lookup = TemplateLookup( directories=paths['templates'], error_handler=handle_mako_error, module_directory=os.path.join(app_conf['cache_dir'], 'templates'), input_encoding='utf-8', default_filters=['escape'], imports=['from webhelpers.html import escape']) # sets the c attribute access when don't existing attribute are accessed config['pylons.strict_tmpl_context'] = True test = os.path.split(config['__file__'])[-1] == 'test.ini' if test: if os.environ.get('TEST_DB'): # swap config if we pass enviroment variable config['sqlalchemy.db1.url'] = os.environ.get('TEST_DB') from rhodecode.lib.utils import create_test_env, create_test_index from rhodecode.tests import TESTS_TMP_PATH # set RC_NO_TMP_PATH=1 to disable re-creating the database and # test repos if not int(os.environ.get('RC_NO_TMP_PATH', 0)): create_test_env(TESTS_TMP_PATH, config) # set RC_WHOOSH_TEST_DISABLE=1 to disable whoosh index during tests if not int(os.environ.get('RC_WHOOSH_TEST_DISABLE', 0)): create_test_index(TESTS_TMP_PATH, config, True) #check git version check_git_version() DbManage.check_waitress() # MULTIPLE DB configs # Setup the SQLAlchemy database engine sa_engine_db1 = engine_from_config(config, 'sqlalchemy.db1.') init_model(sa_engine_db1) repos_path = make_ui('db').configitems('paths')[0][1] repo2db_mapper(ScmModel().repo_scan(repos_path), remove_obsolete=False, install_git_hook=False) set_available_permissions(config) config['base_path'] = repos_path set_rhodecode_config(config) instance_id = rhodecode.CONFIG.get('instance_id') if instance_id == '*': instance_id = '%s-%s' % (os.uname()[1], os.getpid()) rhodecode.CONFIG['instance_id'] = instance_id # CONFIGURATION OPTIONS HERE (note: all config options will override # any Pylons config options) # store config reference into our module to skip import magic of # pylons rhodecode.CONFIG.update(config) return config
def load_environment(global_conf, app_conf, initial=False): """ Configure the Pylons environment via the ``pylons.config`` object """ config = PylonsConfig() # Pylons paths root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) paths = dict( root=root, controllers=os.path.join(root, "controllers"), static_files=os.path.join(root, "public"), templates=[os.path.join(root, "templates")], ) # Initialize config with the basic options config.init_app(global_conf, app_conf, package="rhodecode", paths=paths) # store some globals into rhodecode rhodecode.CELERY_ON = str2bool(config["app_conf"].get("use_celery")) rhodecode.CELERY_EAGER = str2bool(config["app_conf"].get("celery.always.eager")) config["routes.map"] = make_map(config) config["pylons.app_globals"] = app_globals.Globals(config) config["pylons.h"] = helpers rhodecode.CONFIG = config load_rcextensions(root_path=config["here"]) # Setup cache object as early as possible import pylons pylons.cache._push_object(config["pylons.app_globals"].cache) # Create the Mako TemplateLookup, with the default auto-escaping config["pylons.app_globals"].mako_lookup = TemplateLookup( directories=paths["templates"], error_handler=handle_mako_error, module_directory=os.path.join(app_conf["cache_dir"], "templates"), input_encoding="utf-8", default_filters=["escape"], imports=["from webhelpers.html import escape"], ) # sets the c attribute access when don't existing attribute are accessed config["pylons.strict_tmpl_context"] = True test = os.path.split(config["__file__"])[-1] == "test.ini" if test: from rhodecode.lib.utils import create_test_env, create_test_index from rhodecode.tests import TESTS_TMP_PATH create_test_env(TESTS_TMP_PATH, config) create_test_index(TESTS_TMP_PATH, config, True) # MULTIPLE DB configs # Setup the SQLAlchemy database engine sa_engine_db1 = engine_from_config(config, "sqlalchemy.db1.") init_model(sa_engine_db1) repos_path = make_ui("db").configitems("paths")[0][1] repo2db_mapper(ScmModel().repo_scan(repos_path)) set_available_permissions(config) config["base_path"] = repos_path set_rhodecode_config(config) # CONFIGURATION OPTIONS HERE (note: all config options will override # any Pylons config options) # store config reference into our module to skip import magic of # pylons rhodecode.CONFIG.update(config) return config
def handle_git_receive(repo_path, revs, env, hook_type='post'): """ A really hacky method that is runned by git post-receive hook and logs an push action together with pushed revisions. It's executed by subprocess thus needs all info to be able to create a on the fly pylons enviroment, connect to database and run the logging code. Hacky as sh*t but works. :param repo_path: :param revs: :param env: """ from paste.deploy import appconfig from sqlalchemy import engine_from_config from rhodecode.config.environment import load_environment from rhodecode.model import init_model from rhodecode.model.db import RhodeCodeUi from rhodecode.lib.utils import make_ui extras = _extract_extras(env) path, ini_name = os.path.split(extras['config']) conf = appconfig('config:%s' % ini_name, relative_to=path) load_environment(conf.global_conf, conf.local_conf) engine = engine_from_config(conf, 'sqlalchemy.db1.') init_model(engine) baseui = make_ui('db') # fix if it's not a bare repo if repo_path.endswith(os.sep + '.git'): repo_path = repo_path[:-5] repo = Repository.get_by_full_path(repo_path) if not repo: raise OSError('Repository %s not found in database' % (safe_str(repo_path))) _hooks = dict(baseui.configitems('hooks')) or {} if hook_type == 'pre': repo = repo.scm_instance else: #post push shouldn't use the cached instance never repo = repo.scm_instance_no_cache() if hook_type == 'pre': pre_push(baseui, repo) # if push hook is enabled via web interface elif hook_type == 'post' and _hooks.get(RhodeCodeUi.HOOK_PUSH): rev_data = [] for l in revs: old_rev, new_rev, ref = l.split(' ') _ref_data = ref.split('/') if _ref_data[1] in ['tags', 'heads']: rev_data.append({'old_rev': old_rev, 'new_rev': new_rev, 'ref': ref, 'type': _ref_data[1], 'name': _ref_data[2].strip()}) git_revs = [] for push_ref in rev_data: _type = push_ref['type'] if _type == 'heads': if push_ref['old_rev'] == EmptyChangeset().raw_id: cmd = "for-each-ref --format='%(refname)' 'refs/heads/*'" heads = repo.run_git_command(cmd)[0] heads = heads.replace(push_ref['ref'], '') heads = ' '.join(map(lambda c: c.strip('\n').strip(), heads.splitlines())) cmd = (('log %(new_rev)s' % push_ref) + ' --reverse --pretty=format:"%H" --not ' + heads) git_revs += repo.run_git_command(cmd)[0].splitlines() elif push_ref['new_rev'] == EmptyChangeset().raw_id: #delete branch case git_revs += ['delete_branch=>%s' % push_ref['name']] else: cmd = (('log %(old_rev)s..%(new_rev)s' % push_ref) + ' --reverse --pretty=format:"%H"') git_revs += repo.run_git_command(cmd)[0].splitlines() elif _type == 'tags': git_revs += ['tag=>%s' % push_ref['name']] log_push_action(baseui, repo, _git_revs=git_revs)
def get_session(): if CELERY_ON: engine = engine_from_config(config, "sqlalchemy.db1.") init_model(engine) sa = meta.Session() return sa
def initialize_database(config): from rhodecode.lib.utils2 import engine_from_config, get_encryption_key engine = engine_from_config(config, 'sqlalchemy.db1.') init_model(engine, encryption_key=get_encryption_key(config))