def check_indexes(cnx):
"""Check indexes of a system database: output missing expected indexes as well as unexpected ones.
Return 0 if there is no differences, else 1.
"""
schema_indexes = expected_indexes(cnx)
db_indexes = database_indexes(cnx)
missing_indexes = set(schema_indexes) - db_indexes
if missing_indexes:
print(underline_title('Missing indexes'))
print('index expected by the schema but not found in the database:\n')
missing = ['{0} ON {1[0]} {1[1]}'.format(idx, schema_indexes[idx])
for idx in missing_indexes]
print('\n'.join(sorted(missing)))
print()
status = 1
additional_indexes = db_indexes - set(schema_indexes)
if additional_indexes:
print(underline_title('Additional indexes'))
print('index in the database but not expected by the schema:\n')
print('\n'.join(sorted(additional_indexes)))
print()
status = 1
if not (missing_indexes or additional_indexes):
print('Everything is Ok')
status = 0
return status
def bootstrap(self, cubes, automatic=False, inputlevel=0):
"""create an instance by copying files from the given cube and by asking
information necessary to build required configuration files
"""
config = self.config
if not automatic:
print(underline_title('Configuring the repository'))
config.input_config('email', inputlevel)
print('\n' + underline_title('Configuring the sources'))
# hack to make Method('default_instance_id') usable in db option defs
# (in native.py)
sconfig = SourceConfiguration(config,
options=SOURCE_TYPES['native'].options)
if not automatic:
sconfig.input_config(inputlevel=inputlevel)
print()
sourcescfg = {'system': sconfig}
if automatic:
# XXX modify a copy
password = generate_password()
print('-> set administrator account to admin / %s' % password)
USER_OPTIONS[1][1]['default'] = password
sconfig = Configuration(options=USER_OPTIONS)
else:
sconfig = Configuration(options=USER_OPTIONS)
sconfig.input_config(inputlevel=inputlevel)
sourcescfg['admin'] = sconfig
config.write_sources_file(sourcescfg)
# remember selected cubes for later initialization of the database
config.write_bootstrap_cubes_file(cubes)
def run(self, args):
check_options_consistency(self.config)
print('\n' + underline_title('Initializing the system database'))
from cubicweb.server import init_repository
appid = args[0]
config = ServerConfiguration.config_for(appid)
try:
system = config.system_source_config
extra_args = system.get('db-extra-arguments')
extra = extra_args and {'extra_args': extra_args} or {}
get_connection(system['db-driver'],
database=system['db-name'],
host=system.get('db-host'),
port=system.get('db-port'),
user=system.get('db-user') or '',
password=system.get('db-password') or '',
schema=system.get('db-namespace'),
**extra)
except Exception as ex:
raise ConfigurationError(
'You seem to have provided wrong connection information in '
'the %s file. Resolve this first (error: %s).' %
(config.sources_file(), str(ex).strip()))
init_repository(config, drop=self.config.drop)
if not self.config.automatic:
while ASK.confirm('Enter another source ?', default_is_yes=False):
CWCTL.run([
'source-add', '--config-level',
str(self.config.config_level), config.appid
])
def upgrade_instance(self, appid):
print('\n' + underline_title('Upgrading the instance %s' % appid))
from logilab.common.changelog import Version
config = cwcfg.config_for(appid)
config.repairing = True # notice we're not starting the server
config.verbosity = self.config.verbosity
set_sources_mode = getattr(config, 'set_sources_mode', None)
if set_sources_mode is not None:
set_sources_mode(self.config.ext_sources or ('migration', ))
# get instance and installed versions for the server and the componants
mih = config.migration_handler()
repo = mih.repo
vcconf = repo.get_versions()
helper = self.config_helper(config, required=False)
if self.config.force_cube_version:
for cube, version in self.config.force_cube_version.items():
vcconf[cube] = Version(version)
toupgrade = []
for cube in config.cubes():
installedversion = config.cube_version(cube)
try:
applversion = vcconf[cube]
except KeyError:
config.error('no version information for %s' % cube)
continue
if installedversion > applversion:
toupgrade.append((cube, applversion, installedversion))
cubicwebversion = config.cubicweb_version()
if self.config.force_cubicweb_version:
applcubicwebversion = Version(self.config.force_cubicweb_version)
vcconf['cubicweb'] = applcubicwebversion
else:
applcubicwebversion = vcconf.get('cubicweb')
if cubicwebversion > applcubicwebversion:
toupgrade.append(
('cubicweb', applcubicwebversion, cubicwebversion))
# run cubicweb/componants migration scripts
if self.config.fs_only or toupgrade:
for cube, fromversion, toversion in toupgrade:
print('-> migration needed from %s to %s for %s' %
(fromversion, toversion, cube))
with mih.cnx:
with mih.cnx.security_enabled(False, False):
mih.migrate(vcconf, reversed(toupgrade), self.config)
clear_cache(config, 'instance_md5_version')
else:
print('-> no data migration needed for instance %s.' % appid)
# rewrite main configuration file
if not self.config.no_config_update:
mih.rewrite_configuration()
mih.shutdown()
# handle i18n upgrade
if not self.i18nupgrade(config):
return
print()
if helper:
helper.postupgrade(repo)
print('-> instance migrated.')
print()
def bootstrap(self, cubes, automatic=False, inputlevel=0):
"""bootstrap this configuration"""
if not automatic:
print('\n' + underline_title('Generic web configuration'))
config = self.config
config.input_config('web', inputlevel)
if ASK.confirm('Allow anonymous access ?', False):
config.global_set_option('anonymous-user', 'anon')
config.global_set_option('anonymous-password', 'anon')
def update_cube_catalogs(cubedir):
cubedir = osp.abspath(osp.normpath(cubedir))
workdir = tempfile.mkdtemp()
try:
cubename = osp.basename(cubedir)
if cubename.startswith('cubicweb_'): # new layout
distname = cubename
cubename = cubename[len('cubicweb_'):]
else:
distname = 'cubicweb_' + cubename
print('cubedir', cubedir)
extract_cls = I18nCubeMessageExtractor
try:
extract_cls = pkg_resources.load_entry_point(
distname, 'cubicweb.i18ncube', cubename)
except (pkg_resources.DistributionNotFound, ImportError):
pass # no customization found
print(underline_title('Updating i18n catalogs for cube %s' % cubename))
chdir(cubedir)
extractor = extract_cls(workdir, cubedir)
potfile = extractor.generate_pot_file()
if potfile is None:
print('no message catalog for cube', cubename,
'nothing to translate')
return ()
print('-> merging main pot file with existing translations:', end=' ')
chdir('i18n')
toedit = []
for lang in CubicWebNoAppConfiguration.cw_languages():
print(lang, end=' ')
cubepo = '%s.po' % lang
if not osp.exists(cubepo):
shutil.copy(potfile, cubepo)
else:
cmd = [
'msgmerge', '-N', '-s', '-o', cubepo + 'new', cubepo,
potfile
]
execute2(cmd)
ensure_fs_mode(cubepo)
shutil.move('%snew' % cubepo, cubepo)
toedit.append(osp.abspath(cubepo))
print()
return toedit
finally:
# cleanup
shutil.rmtree(workdir)
def run(self, args):
"""run the command with its specific arguments"""
from logilab.common.textutils import splitstrip
check_options_consistency(self.config)
configname = self.config.config
cubes, appid = args
cubes = splitstrip(cubes)
# get the configuration and helper
config = cwcfg.config_for(appid, configname, creating=True)
cubes = config.expand_cubes(cubes)
config.init_cubes(cubes)
helper = self.config_helper(config)
# check the cube exists
try:
templdirs = [cwcfg.cube_dir(cube)
for cube in cubes]
except ConfigurationError as ex:
print(ex)
print('\navailable cubes:', end=' ')
print(', '.join(available_cube_names(cwcfg)))
return
# create the registry directory for this instance
print('\n' + underline_title('Creating the instance %s' % appid))
create_dir(config.apphome)
# cubicweb-ctl configuration
if not self.config.automatic:
print('\n' + underline_title('Configuring the instance (%s.conf)'
% configname))
config.input_config('main', self.config.config_level)
# configuration'specific stuff
print()
helper.bootstrap(cubes, self.config.automatic, self.config.config_level)
# input for cubes specific options
if not self.config.automatic:
sections = set(sect.lower() for sect, opt, odict in config.all_options()
if 'type' in odict
and odict.get('level', 0) <= self.config.config_level)
for section in sections:
if section not in ('main', 'email', 'web'):
print('\n' + underline_title('%s options' % section))
config.input_config(section, self.config.config_level)
# write down configuration
config.save()
print('-> generated config %s' % config.main_config_file())
# handle i18n files structure
# in the first cube given
from cubicweb import i18n
langs = [lang for lang, _ in i18n.available_catalogs(join(templdirs[0], 'i18n'))]
errors = config.i18ncompile(langs)
if errors:
print('\n'.join(errors))
if self.config.automatic \
or not ASK.confirm('error while compiling message catalogs, '
'continue anyway ?'):
print('creation not completed')
return
# create the additional data directory for this instance
if config.appdatahome != config.apphome: # true in dev mode
create_dir(config.appdatahome)
create_dir(join(config.appdatahome, 'backup'))
if config['uid']:
from logilab.common.shellutils import chown
# this directory should be owned by the uid of the server process
print('set %s as owner of the data directory' % config['uid'])
chown(config.appdatahome, config['uid'])
print('\n-> creation done for %s\n' % repr(config.apphome)[1:-1])
if not self.config.no_db_create:
helper.postcreate(self.config.automatic, self.config.config_level)
def run(self, args):
"""run the command with its specific arguments"""
check_options_consistency(self.config)
automatic = self.get('automatic')
appid = args.pop()
config = ServerConfiguration.config_for(appid)
source = config.system_source_config
dbname = source['db-name']
driver = source['db-driver']
helper = get_db_helper(driver)
if driver == 'sqlite':
if os.path.exists(dbname) and (automatic or ASK.confirm(
'Database %s already exists. Drop it?' % dbname)):
os.unlink(dbname)
elif self.config.create_db:
print('\n' + underline_title('Creating the system database'))
# connect on the dbms system base to create our base
dbcnx = _db_sys_cnx(source,
'CREATE/DROP DATABASE and / or USER',
interactive=not automatic)
cursor = dbcnx.cursor()
try:
if helper.users_support:
user = source['db-user']
if not helper.user_exists(cursor, user) and (
automatic
or ASK.confirm('Create db user %s ?' % user,
default_is_yes=False)):
helper.create_user(source['db-user'],
source.get('db-password'))
print('-> user %s created.' % user)
if dbname in helper.list_databases(cursor):
if automatic or ASK.confirm(
'Database %s already exists -- '
'do you want to drop it ?' % dbname):
cursor.execute('DROP DATABASE "%s"' % dbname)
else:
print('you may want to run "cubicweb-ctl db-init '
'--drop %s" manually to continue.' %
config.appid)
return
createdb(helper, source, dbcnx, cursor)
dbcnx.commit()
print('-> database %s created.' % dbname)
except BaseException:
dbcnx.rollback()
raise
cnx = system_source_cnx(source,
special_privs='CREATE LANGUAGE/SCHEMA',
interactive=not automatic)
cursor = cnx.cursor()
helper.init_fti_extensions(cursor)
namespace = source.get('db-namespace')
if namespace and ASK.confirm('Create schema %s in database %s ?' %
(namespace, dbname)):
helper.create_schema(cursor, namespace)
cnx.commit()
# postgres specific stuff
if driver == 'postgres':
# install plpythonu/plpgsql languages
langs = ('plpythonu', 'plpgsql')
for extlang in langs:
if automatic or ASK.confirm('Create language %s ?' % extlang):
try:
helper.create_language(cursor, extlang)
except Exception as exc:
print('-> ERROR:', exc)
print('-> could not create language %s, '
'some stored procedures might be unusable' %
extlang)
cnx.rollback()
else:
cnx.commit()
print(
'-> database for instance %s created and necessary extensions installed.'
% appid)
print()
if automatic:
CWCTL.run([
'db-init', '--automatic', '--config-level', '0', config.appid
])
elif ASK.confirm('Run db-init to initialize the system database ?'):
CWCTL.run([
'db-init', '--config-level',
str(self.config.config_level), config.appid
])
else:
print('-> nevermind, you can do it later with '
'"cubicweb-ctl db-init %s".' % config.appid)