def init_db(settings: Dict[str, Any]) -> None:
"""
Initialize the SQLAlchemy Session.
First test the connection, on wen environment it should be OK, with the command line we should get
an exception ind initialise the connection.
"""
try:
from c2cgeoportal_commons.models import DBSession # pylint: disable=import-outside-toplevel
from c2cgeoportal_commons.models.main import Theme # pylint: disable=import-outside-toplevel
session = DBSession()
session.query(Theme).count()
except: # pylint: disable=bare-except
# Init db sessions
class R:
settings: Dict[str, Any] = {}
class C:
registry = R()
def get_settings(self) -> Dict[str, Any]:
return self.registry.settings
def add_tween(self, *args: Any, **kwargs: Any) -> None:
pass
config_ = C()
config_.registry.settings = settings
c2cgeoportal_geoportal.init_db_sessions(settings, config_)
def __init__(self, options):
self.options = options
self.imported = set()
self.layers = []
settings = {}
with open("config.yaml") as f:
settings = yaml.load(f)
self.languages = settings["vars"]["available_locale_names"]
exluded_themes_string = settings["vars"]["excluded_themes_from_search"]
exluded_themes = []
if exluded_themes_string is not None:
exluded_themes = exluded_themes_string.split(",")
# must be done only once we have loaded the project config
from c2cgeoportal_commons.models import DBSession
from c2cgeoportal_commons.models.main import Interface, Theme, Role
self.session = DBSession()
self._ = {}
self.metadata_service_url = base_url = os.environ[
"GEONETWORK_BASE_URL"]
with bootstrap(self.options.app_config,
options=escape_variables(os.environ)) as env:
registry = env['registry']
request = env['request']
self.es_layer_index = get_index(request) + '_layers'
self.tdirs = registry.queryUtility(ITranslationDirectories, default=[])
self.tsf = TranslationStringFactory('geoportailv3_geoportal-client')
self.interfaces = self.session.query(Interface).filter(
Interface.name.in_(options.interfaces)).all()
self.public_theme = {}
self.public_group = {}
for interface in self.interfaces:
self.public_theme[interface.id] = []
self.public_group[interface.id] = []
for theme in self.session.query(Theme).filter_by(public=True).all():
if theme.name not in exluded_themes:
self._add_theme(theme)
for role in self.session.query(Role).all():
for theme in self.session.query(Theme).all():
if theme.name not in exluded_themes:
self._add_theme(theme, role)
ensure_index(get_elasticsearch(request), self.es_layer_index,
options.recreate_index)
try:
helpers.bulk(actions=self.layers,
client=get_elasticsearch(request),
raise_on_error=True)
except (BulkIndexError, ConnectionTimeout) as e:
statuslog("\n %s" % e)
def setup_method(self, _):
# Always see the diff
# https://docs.python.org/2/library/unittest.html#unittest.TestCase.maxDiff
self.maxDiff = None # pylint: disable=invalid-name
self._tables = []
from c2cgeoportal_commons.models import DBSession
from c2cgeoportal_commons.models.static import OAuth2Client, User
user1 = User(username="******", password="******")
user1.is_password_changed = True
user1.email = "*****@*****.**"
user2 = User(username="******", password="******")
user2.email = "*****@*****.**"
client = OAuth2Client()
client.client_id = "qgis"
client.secret = "1234"
client.redirect_uri = "http://127.0.0.1:7070/"
session = DBSession()
session.add_all([user1, user2, client])
session.flush()
transaction.commit()
def get_session(app_config, app_name):
environ = escape_variables(os.environ)
fileConfig(app_config, defaults=environ)
lux_get_app(app_config, app_name)
# must be done afterwe have loaded the project config
from c2cgeoportal_commons.models import DBSession
return DBSession()
def main():
parser = argparse.ArgumentParser(
description="Create and populate the database tables.")
parser.add_argument('-i',
'--iniconfig',
default='production.ini',
help='project .ini config file')
parser.add_argument(
'-n',
'--app-name',
default="app",
help='The application name (optional, default is "app")')
options = parser.parse_args()
# read the configuration
env = {}
env.update(os.environ)
env["LOG_LEVEL"] = "INFO"
env["GUNICORN_ACCESS_LOG_LEVEL"] = "INFO"
env["C2CGEOPORTAL_LOG_LEVEL"] = "WARN"
fileConfig(options.iniconfig, defaults=env)
get_app(options.iniconfig, options.app_name, options=env)
from c2cgeoportal_commons.models import DBSession
from c2cgeoportal_commons.models.main import Interface, OGCServer, Theme, LayerGroup, LayerWMS
session = DBSession()
interfaces = session.query(Interface).all()
ogc_server = session.query(OGCServer).filter(
OGCServer.name == "source for image/png").one()
layer_borders = LayerWMS("Borders", "borders")
layer_borders.interfaces = interfaces
layer_borders.ogc_server = ogc_server
layer_density = LayerWMS("Density", "density")
layer_density.interfaces = interfaces
layer_density.ogc_server = ogc_server
group = LayerGroup("Demo")
group.children = [layer_borders, layer_density]
theme = Theme("Demo")
theme.children = [group]
theme.interfaces = interfaces
session.add(theme)
transaction.commit()
print("Successfully added the demo theme")
def __init__(self, options):
self.options = options
self.imported = set()
settings = get_config("geoportal/config.yaml")
package = settings["package"]
self.fts_languages = settings["fulltextsearch"]["languages"]
self.languages = settings["available_locale_names"]
# must be done only once we have loaded the project config
from c2cgeoportal_commons.models import DBSession
from c2cgeoportal_commons.models.main import FullTextSearch, Interface, Theme, Role
self.session = DBSession()
self.session.execute(FullTextSearch.__table__.delete().where(
FullTextSearch.from_theme == True)) # noqa
self._ = {}
for lang in self.languages:
self._[lang] = translation(
"{}_geoportal-client".format(package),
os.path.join("geoportal", "{}_geoportal".format(package),
"locale/"), [lang])
query = self.session.query(Interface)
if options.interfaces is not None:
query = query.filter(Interface.name.in_(options.interfaces))
self.interfaces = query.all()
self.public_theme = {}
self.public_group = {}
for interface in self.interfaces:
self.public_theme[interface.id] = []
self.public_group[interface.id] = []
for theme in self.session.query(Theme).filter_by(public=True).all():
self._add_theme(theme)
for role in self.session.query(Role).all():
for theme in self.session.query(Theme).all():
self._add_theme(theme, role)
transaction.commit()
def main():
"""
Emergency user create and password reset script
example, reset toto password to foobar:
./docker-compose-run manage_users -p foobar toto
example, create user foo with password bar and role admin:
./docker-compose-run manage_users -c -r role_admin -p bar foo
to get the options list, do:
./docker-compose-run manage_users -h
"""
usage = """Usage: %prog [options] USERNAME
Reset a user password.
The username is used as password if the password is not provided with the corresponding option.
User can be created if it does not exist yet."""
parser = argparse.ArgumentParser(description=usage)
parser.add_argument(
"-i",
"--app-config",
default="geoportal/production.ini",
dest="app_config",
help="The application .ini config file (optional, default is "
"'production.ini')")
parser.add_argument(
"-n",
"--app-name",
default="app",
dest="app_name",
help="The application name (optional, default is 'app')")
parser.add_argument(
"-p",
"--password",
help="Set password (if not set, username is used as password")
parser.add_argument("-c",
"--create",
action="store_true",
default=False,
help="Create user if it does not already exist")
parser.add_argument("-r",
"--rolename",
default="role_admin",
help="The role name which must exist in the database")
parser.add_argument("-e", "--email", default=None, help="The user email")
parser.add_argument('user', nargs='1', help="The user")
options = parser.parse_args()
username = options.user
app_config = options.app_config
app_name = options.app_name
if app_name is None and "#" in app_config:
app_config, app_name = app_config.split("#", 1)
if not os.path.isfile(app_config):
parser.error("Cannot find config file: {0!s}".format(app_config))
# loading schema name from config and setting its value to the
# corresponding global variable from c2cgeoportal_geoportal
# Ignores pyramid deprecation warnings
warnings.simplefilter("ignore", DeprecationWarning)
fileConfig(app_config, defaults=os.environ)
get_app(app_name, options.app_name, options=os.environ)
# must be done only once we have loaded the project config
from c2cgeoportal_commons.models import DBSession, main, static
print("\n")
# check that user exists
sess = DBSession()
query = sess.query(
static.User).filter_by(username="******".format(username))
result = query.count()
if result == 0:
if not options.create:
# if doesn"t exist and no -c option, throw error
raise Exception(
"User {0!s} does not exist in database".format(username))
else:
print(("User {0!s} does not exist in database, creating".format(
username)))
# if does not exist and -c option, create user
password = options.password if options.password is not None else username
email = options.email if options.email is not None else username
# get roles
query_role = sess.query(main.Role).filter(
main.Role.name == "{0!s}".format(options.rolename))
if query_role.count() == 0:
# role not found in db?
raise Exception(
"Role matching {0!s} does not exist in database".format(
options.rolename))
role = query_role.first()
user = static.User(username="******".format(username),
password="******".format(password),
email="{0!s}".format(email),
role=role)
sess.add(user)
transaction.commit()
print((
"User {0!s} created with password {1!s} and role {2!s}".format(
username, password, options.rolename)))
else:
# if user exists (assuming username are unique)
user = query.first()
if options.password is not None:
print(("Password set to: {0!s}".format(options.password)))
user.password = "******".format(options.password)
if options.email is not None:
user.email = options.email
sess.add(user)
transaction.commit()
print(("Password reset for user {0!s}".format(username)))
def main():
parser = ArgumentParser(
prog=sys.argv[0],
add_help=True,
description=
"Tool used to migrate your old layers from the old structure to the new one.",
)
parser.add_argument(
"-i",
"--app-config",
default="geoportal/production.ini",
dest="app_config",
help=
"the application .ini config file (optional, default is 'production.ini')"
)
parser.add_argument(
"-n",
"--app-name",
default="app",
dest="app_name",
help="the application name (optional, default is 'app')")
parser.add_argument("--no-layers",
dest="layers",
action="store_false",
help="do not import the layers")
parser.add_argument("--no-groups",
dest="groups",
action="store_false",
help="do not import the groups")
parser.add_argument("--no-themes",
dest="themes",
action="store_false",
help="do not import the themes")
options = parser.parse_args()
app_config = options.app_config
app_name = options.app_name
if app_name is None and "#" in app_config:
app_config, app_name = app_config.split("#", 1)
env = {}
env.update(os.environ)
env["LOG_LEVEL"] = "INFO"
env["GUNICORN_ACCESS_LOG_LEVEL"] = "INFO"
env["C2CGEOPORTAL_LOG_LEVEL"] = "WARN"
fileConfig(app_config, defaults=env)
app = get_app(app_config, app_name, options=env)
# must be done only once we have loaded the project config
from c2cgeoportal_commons.models import DBSession
from c2cgeoportal_commons.models.main import OGCServer, Theme, LayerWMS, LayerWMTS, LayerV1, LayerGroup
session = DBSession()
if options.layers:
table_list = [LayerWMTS, LayerWMS, OGCServer]
for table in table_list:
print(("Emptying table {0!s}.".format(table.__table__)))
# must be done exactly this way otherwise the cascade config in the
# models are not used
for t in session.query(table).all():
session.delete(t)
# list and create all distinct ogc_server
ogc_server(session, app.registry.settings)
print("Converting layerv1.")
for layer in session.query(LayerV1).all():
layer_v1tov2(session, layer)
if options.groups:
print("Converting layer groups.")
for group in session.query(LayerGroup).all():
layergroup_v1tov2(session, group)
if options.themes:
print("Converting themes.")
for theme in session.query(Theme).all():
theme_v1tov2(session, theme)
transaction.commit()
def _collect_app_config(self, filename: str) -> List[Message]:
config.init(filename)
settings = config.get_config()
# Collect raster layers names
raster = [
Message(None, raster_layer, None, [], "", "",
(filename, "raster/{}".format(raster_layer)))
for raster_layer in list(settings.get("raster", {}).keys())
]
# Init db sessions
class R:
settings: Dict[str, Any] = {}
class C:
registry = R()
def get_settings(self) -> Dict[str, Any]:
return self.registry.settings
def add_tween(self, *args: Any, **kwargs: Any) -> None:
pass
config_ = C()
config_.registry.settings = settings
c2cgeoportal_geoportal.init_dbsessions(settings, config_)
# Collect layers enum values (for filters)
from c2cgeoportal_commons.models import DBSessions # pylint: disable=import-outside-toplevel
from c2cgeoportal_commons.models.main import Metadata # pylint: disable=import-outside-toplevel
enums = []
enum_layers = settings.get("layers", {}).get("enum", {})
for layername in list(enum_layers.keys()):
layerinfos = enum_layers.get(layername, {})
attributes = layerinfos.get("attributes", {})
for fieldname in list(attributes.keys()):
values = self._enumerate_attributes_values(
DBSessions, layerinfos, fieldname)
for (value, ) in values:
if isinstance(value, str) and value != "":
msgid = value
location = "/layers/{}/values/{}/{}".format(
layername,
fieldname,
value.encode("ascii",
errors="replace").decode("ascii"),
)
enums.append(
Message(None, msgid, None, [], "", "",
(filename, location)))
metadata_list = []
defs = config["admin_interface"]["available_metadata"] # pylint: disable=unsubscriptable-object
names = [e["name"] for e in defs if e.get("translate", False)]
if names:
engine = sqlalchemy.create_engine(config["sqlalchemy.url"])
DBSession = sqlalchemy.orm.session.sessionmaker(
) # noqa: disable=N806
DBSession.configure(bind=engine)
session = DBSession()
query = session.query(Metadata).filter(Metadata.name.in_(names)) # pylint: disable=no-member
for metadata in query.all():
location = "metadata/{}/{}".format(metadata.name, metadata.id)
metadata_list.append(
Message(None, metadata.value, None, [], "", "",
(filename, location)))
interfaces_messages = []
for interface, interface_config in config["interfaces_config"].items():
for ds_index, datasource in enumerate(
interface_config.get("constants",
{}).get("gmfSearchOptions",
{}).get("datasources", [])):
for a_index, action in enumerate(
datasource.get("groupActions", [])):
location = (
"interfaces_config/{}/constants/gmfSearchOptions/datasources[{}]/"
"groupActions[{}]/title".format(
interface, ds_index, a_index))
interfaces_messages.append(
Message(None, action["title"], None, [], "", "",
(filename, location)))
for merge_tab in (interface_config.get("constants", {}).get(
"gmfDisplayQueryGridOptions", {}).get("mergeTabs",
{}).keys()):
location = "interfaces_config/{}/constants/gmfDisplayQueryGridOptions/mergeTabs/{}/".format(
interface, merge_tab)
interfaces_messages.append(
Message(None, merge_tab, None, [], "", "",
(filename, location)))
return raster + enums + metadata_list + interfaces_messages
def __call__(self,
filename: str,
options: Dict[str, Any],
fileobj: Optional[str] = None,
lineno: int = 0) -> List[Message]:
del fileobj, lineno
print(f"Running {self.__class__.__name__} on {filename}")
messages: List[Message] = []
try:
init_db(self.config)
from c2cgeoportal_commons.models import DBSession # pylint: disable=import-outside-toplevel
db_session = DBSession()
try:
from c2cgeoportal_commons.models.main import ( # pylint: disable=import-outside-toplevel
FullTextSearch, LayerGroup, LayerWMS, LayerWMTS, Theme,
)
self._import(Theme,
messages,
name_regex=_get_config_str("THEME_REGEX", ".*"))
self._import(
LayerGroup,
messages,
name_regex=_get_config_str("GROUP_REGEX", ".*"),
has_interfaces=False,
)
self._import(
LayerWMS,
messages,
self._import_layer_wms,
name_regex=_get_config_str("WMSLAYER_REGEX", ".*"),
)
self._import(
LayerWMTS,
messages,
self._import_layer_wmts,
name_regex=_get_config_str("WMTSLAYER_REGEX", ".*"),
)
for (layer_name, ) in db_session.query(
FullTextSearch.layer_name).distinct().all():
if layer_name is not None and layer_name != "":
assert layer_name is not None
messages.append(
Message(
None,
layer_name,
None,
[],
"",
"",
("fts",
layer_name.encode("ascii", errors="replace")),
))
for (actions, ) in db_session.query(
FullTextSearch.actions).distinct().all():
if actions is not None and actions != "":
for action in actions:
assert action["data"] is not None
messages.append(
Message(
None,
action["data"],
None,
[],
"",
"",
("fts", action["data"].encode(
"ascii", errors="replace")),
))
except ProgrammingError as e:
print(
colorize(
"ERROR! The database is probably not up to date "
"(should be ignored when happen during the upgrade)",
Color.RED,
))
print(colorize(e, Color.RED))
if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
raise
except NoSuchTableError as e:
print(
colorize(
"ERROR! The schema didn't seem to exists "
"(should be ignored when happen during the deploy)",
Color.RED,
))
print(colorize(e, Color.RED))
if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
raise
except OperationalError as e:
print(
colorize(
"ERROR! The database didn't seem to exists "
"(should be ignored when happen during the deploy)",
Color.RED,
))
print(colorize(e, Color.RED))
if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
raise
return messages
def _collect_app_config(self, filename: str) -> List[Message]:
config.init(filename)
settings = config.get_config()
assert not [
raster_layer
for raster_layer in list(settings.get("raster", {}).keys())
if raster_layer is None
]
# Collect raster layers names
raster = [
Message(None, raster_layer, None, [], "", "",
(filename, f"raster/{raster_layer}"))
for raster_layer in list(settings.get("raster", {}).keys())
]
init_db(settings)
# Collect layers enum values (for filters)
from c2cgeoportal_commons.models import ( # pylint: disable=import-outside-toplevel
DBSession, DBSessions,
)
from c2cgeoportal_commons.models.main import Metadata # pylint: disable=import-outside-toplevel
enums = []
enum_layers = settings.get("layers", {}).get("enum", {})
for layername in list(enum_layers.keys()):
layerinfos = enum_layers.get(layername, {})
attributes = layerinfos.get("attributes", {})
for fieldname in list(attributes.keys()):
values = self._enumerate_attributes_values(
DBSessions, layerinfos, fieldname)
for (value, ) in values:
if isinstance(value, str) and value != "":
msgid = value
location = (
f"/layers/{layername}/values/{fieldname}/"
f"{value.encode('ascii', errors='replace').decode('ascii')}"
)
assert msgid is not None
enums.append(
Message(None, msgid, None, [], "", "",
(filename, location)))
metadata_list = []
defs = config["admin_interface"]["available_metadata"] # pylint: disable=unsubscriptable-object
names = [e["name"] for e in defs if e.get("translate", False)]
if names:
session = DBSession()
query = session.query(Metadata).filter(Metadata.name.in_(names))
for metadata in query.all():
location = f"metadata/{metadata.name}/{metadata.id}"
assert metadata.value is not None
metadata_list.append(
Message(None, metadata.value, None, [], "", "",
(filename, location)))
interfaces_messages = []
for interface, interface_config in config["interfaces_config"].items():
for ds_index, datasource in enumerate(
interface_config.get("constants",
{}).get("gmfSearchOptions",
{}).get("datasources", [])):
for a_index, action in enumerate(
datasource.get("groupActions", [])):
location = (
f"interfaces_config/{interface}/constants/gmfSearchOptions/datasources[{ds_index}]/"
f"groupActions[{a_index}]/title")
assert action["title"] is not None
interfaces_messages.append(
Message(None, action["title"], None, [], "", "",
(filename, location)))
for merge_tab in (interface_config.get("constants", {}).get(
"gmfDisplayQueryGridOptions", {}).get("mergeTabs",
{}).keys()):
location = (
f"interfaces_config/{interface}/constants/gmfDisplayQueryGridOptions/"
f"mergeTabs/{merge_tab}/")
assert merge_tab is not None
interfaces_messages.append(
Message(None, merge_tab, None, [], "", "",
(filename, location)))
return raster + enums + metadata_list + interfaces_messages
