def _import_layer_attributes(self, url, layer, item_type, name, messages):
attributes, layers = self._layer_attributes(url, layer)
for sub_layer in layers:
messages.append(
Message(
None,
sub_layer,
None,
[],
"",
"",
(".".join([item_type, name]), sub_layer.encode("ascii", "replace")),
)
)
for attribute in attributes:
messages.append(
Message(
None,
attribute,
None,
[],
"",
"",
(".".join([item_type, name]), layer.encode("ascii", "replace")),
)
)
def _import_layer_attributes(self, url: str, layer: "main.Layer",
item_type: str, name: str,
messages: List[str]) -> None:
attributes, layers = self._layer_attributes(url, layer)
for sub_layer in layers:
messages.append(
Message(
None,
sub_layer,
None,
[],
"",
"",
(".".join([item_type, name
]), sub_layer.encode("ascii", "replace")),
))
for attribute in attributes:
messages.append(
Message(
None,
attribute,
None,
[],
"",
"",
(".".join([item_type, name
]), layer.encode("ascii", "replace")),
))
def __call__(self, filename, options):
with open(filename) as config_file:
from c2cgeoportal.lib.print_ import * # noqa
config = yaml.load(config_file)
# for application config (.build/config.yaml)
if "vars" in config:
return [
Message(None, raster_layer, None, [], u"", u"",
(filename, u"raster/%s" % raster_layer)) for
raster_layer in config["vars"].get("raster", {}).keys()
]
# for the print config
elif "templates" in config:
result = []
for template in config.get("templates").keys():
result.append(
Message(None, template, None, [], u"", u"",
(filename, u"template/%s" % template)))
result += [
Message(None, attribute, None, [], u"", u"",
(filename, u"template/%s/%s" %
(template, attribute))) for attribute in
config.get("templates")[template].attributes.keys()
]
return result
else:
raise Exception("Not a known config file")
def _collect_app_config(cls, config, filename):
# Collect raster layers names
raster = [
Message(None, raster_layer, None, [], "", "",
(filename, "raster/{}".format(raster_layer)))
for raster_layer in list(config["vars"].get("raster", {}).keys())
]
# Collect layers enum values (for filters)
settings = get_config("config.yaml")
from c2cgeoportal.pyramid_ import init_dbsessions
init_dbsessions(settings)
from c2cgeoportal.models import DBSessions
from c2cgeoportal.views.layers import Layers
enums = []
enum_layers = config["vars"].get("layers", {}).get("enum", {})
for layername in list(enum_layers.keys()):
layerinfos = enum_layers.get(layername, {})
attributes = layerinfos.get("attributes", {})
for fieldname in list(attributes.keys()):
values = cls._enumerate_attributes_values(
DBSessions, Layers, layerinfos, fieldname)
for value, in values:
if value != "":
msgid = value if isinstance(value, str) else value
location = "/layers/{}/values/{}/{}".format(
layername, fieldname,
value.encode("ascii", errors="replace")
if isinstance(value, str) else value)
enums.append(
Message(None, msgid, None, [], "", "",
(filename, location)))
return raster + enums
def _collect_print_config(print_config: Dict[str, Any],
filename: str) -> List[Message]:
result = []
for template_ in list(
cast(Dict[str, Any], print_config.get("templates")).keys()):
assert template_ is not None
result.append(
Message(None, template_, None, [], "", "",
(filename, f"template/{template_}")))
assert not [
attribute for attribute in list(
print_config["templates"][template_]["attributes"].keys())
if attribute is None
]
result += [
Message(
None,
attribute,
None,
[],
"",
"",
(filename, f"template/{template_}/{attribute}"),
) for attribute in list(print_config["templates"][template_]
["attributes"].keys())
]
return result
def __call__(self, filename, options):
messages = []
try:
self.env = bootstrap(filename, options=os.environ)
try:
from c2cgeoportal_commons.models import DBSession
from c2cgeoportal_commons.models.main import Theme, LayerGroup, \
LayerWMS, LayerWMTS, FullTextSearch
self._import(Theme, messages)
self._import(LayerGroup, messages)
self._import(LayerWMS, messages, self._import_layer_wms)
self._import(LayerWMTS, messages, self._import_layer_wmts)
for ln, in DBSession.query(
FullTextSearch.layer_name).distinct().all():
if ln is not None and ln != "":
messages.append(
Message(
None, ln, None, [], "", "",
("fts", ln.encode("ascii", errors="replace"))))
for ln, in DBSession.query(
FullTextSearch.actions).distinct().all():
if ln is not None and ln != "":
action = json.loads(ln)
messages.append(
Message(None, action['data'], None, [], "", "",
("fts", action['data'].encode(
"ascii", errors="replace"))))
except ProgrammingError as e:
print(
colorize(
"ERROR! The database is probably not up to date "
"(should be ignored when happen during the upgrade)",
RED))
print(colorize(e, RED))
if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
raise
except NoSuchTableError as e:
print(
colorize(
"ERROR! The schema didn't seem to exists "
"(should be ignored when happen during the deploy)", RED))
print(colorize(e, RED))
if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
raise
except OperationalError as e:
print(
colorize(
"ERROR! The database didn't seem to exists "
"(should be ignored when happen during the deploy)", RED))
print(colorize(e, RED))
if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
raise
return messages
def _collect_print_config(cls, config, filename):
result = []
for template_ in list(config.get("templates").keys()):
result.append(
Message(None, template_, None, [], "", "",
(filename, "template/{}".format(template_))))
result += [
Message(
None, attribute, None, [], "", "",
(filename, "template/{}/{}".format(template_, attribute)))
for attribute in list(
config.get("templates")[template_].attributes.keys())
]
return result
def __call__(self, filename, options, fileobj=None, lineno=0):
factories = list()
trim = lambda x: x[1:-1]
with open(filename, 'r+') as f:
data = f.read()
for match in REGEX_FACTORY.finditer(data):
var, domain = match.groups()
domain = trim(domain)
if options.domain is None or options.domain == domain:
factories.append(var)
# pre calculation to now with lineno the string was found
s = 0
linesum = list()
for line in data.split('\n'):
s += len(line)
linesum.append(s)
for factory in factories:
regex = re.compile(REGEX_MSG % factory)
for match in regex.finditer(data):
msgid, default = match.groups()
msgid, default = trim(msgid), trim(default)
# calculate lineno
start = match.start()
for i, s in enumerate(linesum):
lineno = i + 1
if start < s:
break
yield Message(None, msgid, None, [], 'Default: %s' % default,
'', (filename, lineno))
def process_python(self, code, code_lineno, translator_strings):
source = code.getvalue().strip()
if source.endswith(compat.b(":")):
if source in (
compat.b("try:"),
compat.b("else:"),
) or source.startswith(compat.b("except")):
source = compat.b("") # Ignore try/except and else
elif source.startswith(compat.b("elif")):
source = source[2:] # Replace "elif" with "if"
source += compat.b("pass")
code = io.BytesIO(source)
for msg in self.python_extractor(
self.filename, self.options, code, code_lineno - 1
):
if translator_strings:
msg = Message(
msg.msgctxt,
msg.msgid,
msg.msgid_plural,
msg.flags,
compat.u(" ").join(translator_strings + [msg.comment]),
msg.tcomment,
msg.location,
)
yield msg
def _collect_app_config(self, filename):
config.init(filename)
settings = config.get_config()
# Collect raster layers names
raster = [
Message(None, raster_layer, None, [], "", "",
(filename, "raster/{}".format(raster_layer)))
for raster_layer in list(settings.get("raster", {}).keys())
]
# Collect layers enum values (for filters)
class R:
settings = None
class C:
registry = R()
config_ = C()
config_.registry.settings = settings
init_dbsessions(settings, config_)
from c2cgeoportal_commons import models
c2cgeoportal_geoportal.init_dbsessions(settings, config_)
from c2cgeoportal_geoportal.views.layers import Layers
enums = []
enum_layers = settings.get("layers", {}).get("enum", {})
for layername in list(enum_layers.keys()):
layerinfos = enum_layers.get(layername, {})
attributes = layerinfos.get("attributes", {})
for fieldname in list(attributes.keys()):
values = self._enumerate_attributes_values(
models.DBSessions, Layers, layerinfos, fieldname)
for value, in values:
if value != "":
msgid = value if isinstance(value, str) else value
location = "/layers/{}/values/{}/{}".format(
layername, fieldname,
value.encode("ascii", errors="replace")
if isinstance(value, str) else value)
enums.append(
Message(None, msgid, None, [], "", "",
(filename, location)))
return raster + enums
def _import(self, object_type, messages, callback=None):
from c2cgeoportal.models import DBSession
items = DBSession.query(object_type)
for item in items:
messages.append(
Message(None, item.name, None, [], u"", u"",
(".".join([item.item_type, item.name]), item.id)))
if callback is not None:
callback(item, messages)
def _collect_print_config(print_config: Dict[str, Any],
filename: str) -> List[Message]:
result = []
for template_ in list(
cast(Dict[str, Any], print_config.get("templates")).keys()):
result.append(
Message(None, template_, None, [], "", "",
(filename, "template/{}".format(template_))))
result += [
Message(
None,
attribute,
None,
[],
"",
"",
(filename, "template/{}/{}".format(template_, attribute)),
) for attribute in list(print_config["templates"][template_]
["attributes"].keys())
]
return result
def _import(object_type, messages, callback=None):
from c2cgeoportal.models import DBSession
items = DBSession.query(object_type)
for item in items:
messages.append(
Message(None, item.name, None, [], "", "",
(item.item_type,
item.name.encode("ascii", errors="replace"))))
if callback is not None:
callback(item, messages)
def process_python(self, code, code_lineno, translator_strings):
source = code.getvalue().strip()
if source.endswith(compat.b(':')):
source += compat.b(' pass')
code = io.BytesIO(source)
for msg in self.python_extractor(self.filename, self.options, code,
code_lineno):
if translator_strings:
msg = Message(
msg.msgctxt, msg.msgid, msg.msgid_plural, msg.flags,
compat.u(' ').join(translator_strings + [msg.comment]),
msg.tcomment, msg.location)
yield msg
def _import_layer_wms(self, layer: "main.Layer",
messages: List[str]) -> None:
server = layer.ogc_server
url = server.url_wfs or server.url
if url is None:
return
if layer.ogc_server.wfs_support:
for wms_layer in layer.layer.split(","):
self._import_layer_attributes(url, wms_layer, layer.item_type,
layer.name, messages)
if layer.geo_table is not None and layer.geo_table != "":
try:
cls = get_layer_class(layer, with_last_update_columns=True)
for column_property in class_mapper(cls).iterate_properties:
if isinstance(column_property, ColumnProperty) and len(
column_property.columns) == 1:
column = column_property.columns[0]
if not column.primary_key and not isinstance(
column.type, Geometry):
if column.foreign_keys:
if column.name == "type_id":
name = "type_"
elif column.name.endswith("_id"):
name = column.name[:-3]
else:
name = column.name + "_"
else:
name = column_property.key
messages.append(
Message(
None,
name,
None,
[],
"",
"",
(".".join([
"edit", layer.item_type,
str(layer.id)
]), layer.name),
))
except NoSuchTableError:
print(
colorize(
"ERROR! No such table '{}' for layer '{}'.".format(
layer.geo_table, layer.name),
Color.RED,
))
print(colorize(traceback.format_exc(), Color.RED))
if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
raise
def __call__(
self,
filename: str,
options: Dict[str, Any],
fileobj: Optional[Dict[str, Any]] = None,
lineno: int = 0,
) -> List[Message]:
del options, fileobj, lineno
print(f"Running {self.__class__.__name__} on {filename}")
with open(filename, encoding="utf8") as config_file:
data = config_file.read()
settings = yaml.load(
data.replace("{{cookiecutter.geomapfish_main_version}}",
os.environ["MAJOR_VERSION"]),
Loader=yaml.SafeLoader,
)
admin_interface = settings.get("vars", {}).get("admin_interface", {})
available_metadata = []
for elem in admin_interface.get("available_metadata", []):
if "description" in elem:
location = f"admin_interface/available_metadata/{elem.get('name', '')}"
available_metadata.append(
Message(None, elem["description"].strip(), None, [], "",
"", (filename, location)))
available_functionalities = []
for elem in admin_interface.get("available_functionalities", []):
if "description" in elem:
location = f"admin_interface/available_functionalities/{elem.get('name', '')}"
available_functionalities.append(
Message(None, elem["description"].strip(), None, [], "",
"", (filename, location)))
return available_metadata + available_functionalities
def _insert_attribute(self, attribute, location):
if attribute not in self.fields:
print(f'Adding attribute {attribute}')
self.fields.add(attribute)
self.messages.append(
Message(
None,
u'%(name)s' % {'name': attribute},
None,
[],
"",
"",
location,
))
def __call__(self, filename, options):
message_str = subprocess.check_output(
["node", "tools/extract-messages.js", filename])
try:
messages = loads(message_str)
return [
Message(None, message, None, [], u"", u"", context.split(":"))
for context, message in messages
]
except:
print("An error occurred")
print(message_str)
print("------")
raise
def __call__(self, filename, options):
message_str = subprocess.check_output(
["node", "tools/extract-messages.js", filename])
try:
messages = []
for contexts, message in loads(message_str):
for context in contexts.split(", "):
messages.append(
Message(None, message, None, [], u"", u"",
context.split(":")))
return messages
except:
print(colorize("An error occurred", RED))
print(colorize(message_str, RED))
print("------")
raise
def _import_layer_wms(self, layer, messages):
server = layer.ogc_server
url = server.url_wfs or server.url
if url is None:
return
for wms_layer in layer.layer.split(","):
self._import_layer_attributes(url, wms_layer, layer.item_type,
layer.name, messages)
if layer.geo_table is not None and layer.geo_table != "":
exclude = [] if layer.exclude_properties is None else layer.exclude_properties.split(
",")
last_update_date = layer.get_metadatas("lastUpdateDateColumn")
if len(last_update_date) == 1:
exclude.append(last_update_date[0].value)
last_update_user = layer.get_metadatas("lastUpdateUserColumn")
if len(last_update_user) == 1:
exclude.append(last_update_user[0].value)
try:
cls = get_class(layer.geo_table, exclude_properties=exclude)
for column_property in class_mapper(cls).iterate_properties:
if isinstance(column_property, ColumnProperty) and len(
column_property.columns) == 1:
column = column_property.columns[0]
if not column.primary_key and not isinstance(
column.type, Geometry):
if column.foreign_keys:
name = "type_" if column.name == "type_id" else \
column.name[0:column.name.rindex("_id")]
else:
name = column_property.key
messages.append(
Message(None, name, None, [], "", "",
(".".join([
"edit", layer.item_type,
str(layer.id)
]), layer.name)))
except NoSuchTableError:
print(
colorize(
"ERROR! No such table '{}' for layer '{}'.".format(
layer.geo_table, layer.name), RED))
print(colorize(traceback.format_exc(), RED))
if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
raise
def __call__(self, filename, options, fileobj=None, lineno=0):
self.filename = filename
if fileobj is None:
fileobj = io.open(filename, encoding='utf-8')
parser_func = parsers.get(self.syntax)
source = fileobj.read()
if isinstance(source, bytes):
source = source.decode('UTF-8')
tree = parser_func(filename, source, translatable=True)
for node in tree:
if isinstance(node, TranslatableText):
yield Message(None, node.text, None, [], u'', u'',
(filename, lineno + (node.position[0] or 1)))
elif isinstance(node, Expression):
for m in self.parse_python(node, filename, lineno, options):
yield m
def _import(object_type, messages, callback=None):
from c2cgeoportal_commons.models import DBSession # pylint: disable=import-outside-toplevel
items = DBSession.query(object_type)
for item in items:
messages.append(
Message(
None,
item.name,
None,
[],
"",
"",
(item.item_type, item.name.encode("ascii", errors="replace")),
)
)
if callback is not None:
callback(item, messages)
def __call__(self, filename, options):
# FIXME find a better way to load settings
settings = load_full_settings("c2c://development.ini")
engine = engine_from_config(settings, "sqlalchemy.")
dbsession = get_session_factory(engine)()
messages = []
for rec in dbsession.query(HazardLevel):
messages.append((rec.title, type(rec).__name__))
for rec in dbsession.query(HazardType):
messages.append((rec.title, type(rec).__name__))
return [
Message(None, text, None, [], class_name, "", (filename, 1))
for text, class_name in messages if text != "" and text is not None
]
def __call__(self, filename, options):
configuration.init(filename)
settings = configuration.get_config()
for path in (
("getitfixed", "admin_new_issue_email", "email_subject"),
("getitfixed", "admin_new_issue_email", "email_body"),
("getitfixed", "new_issue_email", "email_subject"),
("getitfixed", "new_issue_email", "email_body"),
("getitfixed", "update_issue_email", "email_subject"),
("getitfixed", "update_issue_email", "email_body"),
("getitfixed", "resolved_issue_email", "email_subject"),
("getitfixed", "resolved_issue_email", "email_body"),
):
value = settings
for key in path:
value = value[key]
# yield Message(msgctxt msgid msgid_plural flags comment tcomment location)
yield Message(None, value, None, [], u"", u"",
(filename, "/".join(path)))
def __call__(self, filename, options):
# FIXME find a better way to load settings
settings = load_full_settings('development.ini')
engine = engine_from_config(settings, 'sqlalchemy.')
DBSession.configure(bind=engine)
messages = []
for rec in DBSession.query(HazardLevel):
messages.append((rec.title, type(rec).__name__))
for rec in DBSession.query(HazardType):
messages.append((rec.title, type(rec).__name__))
return [
Message(None, text, None, [], class_name, u'', (filename, 1))
for text, class_name in messages if text != '' and text is not None
]
def __call__(self, filename, options):
# FIXME find a better way to load settings
settings = load_full_settings("c2c://development.ini")
engine = engine_from_config(settings, "sqlalchemy.")
dbsession = get_session_factory(engine)()
messages = []
for rec in dbsession.query(ClimateChangeRecommendation):
messages.append((rec.text, type(rec).__name__))
for rec in dbsession.query(HazardCategory):
messages.append((rec.general_recommendation, type(rec).__name__))
for rec in dbsession.query(TechnicalRecommendation):
messages.append((rec.text, type(rec).__name__))
messages.append((rec.detail, type(rec).__name__))
return [
Message(None, text, None, [], class_name, "", (filename, 1))
for text, class_name in messages if text != "" and text is not None
]
def _import(
object_type: Type[Any],
messages: List[str],
callback: Optional[Callable[["main.Layer", List[str]], None]] = None,
has_interfaces: bool = True,
name_regex: str = ".*",
) -> None:
from c2cgeoportal_commons.models import DBSession # pylint: disable=import-outside-toplevel
from c2cgeoportal_commons.models.main import Interface # pylint: disable=import-outside-toplevel
filter_re = re.compile(name_regex)
query = DBSession.query(object_type)
interfaces = _get_config("INTERFACES")
if has_interfaces and interfaces is not None:
query.join(object_type.interface).filter(
Interface.name in interfaces.split("."))
for item in query.all():
assert item.name is not None
if filter_re.match(item.name):
messages.append(
Message(
None,
item.name,
None,
[],
"",
"",
(item.item_type,
item.name.encode("ascii", errors="replace")),
))
if callback is not None:
callback(item, messages)
def __call__(self, filename, options):
# FIXME find a better way to load settings
settings = load_full_settings('development.ini')
engine = engine_from_config(settings, 'sqlalchemy.')
DBSession.configure(bind=engine)
messages = []
for rec in DBSession.query(ClimateChangeRecommendation):
messages.append((rec.text, type(rec).__name__))
for rec in DBSession.query(HazardCategory):
messages.append((rec.general_recommendation, type(rec).__name__))
for rec in DBSession.query(TechnicalRecommendation):
messages.append((rec.text, type(rec).__name__))
messages.append((rec.detail, type(rec).__name__))
return [
Message(None, text, None, [], class_name, u'', (filename, 1))
for text, class_name in messages if text != '' and text is not None
]
def _import_layer_wms(self, layer, messages):
server = layer.ogc_server
url = server.url_wfs or server.url
if url is None:
return
for wms_layer in layer.layer.split(","):
self._import_layer_attributes(url, wms_layer, layer.item_type,
layer.name, layer.id, messages)
if layer.geo_table is not None and layer.geo_table != "":
exclude = [] if layer.exclude_properties is None else layer.exclude_properties.split(
",")
last_update_date = layer.get_metadatas("lastUpdateDateColumn")
if len(last_update_date) == 1:
exclude.append(last_update_date[0].value)
last_update_user = layer.get_metadatas("lastUpdateUserColumn")
if len(last_update_user) == 1:
exclude.append(last_update_user[0].value)
try:
cls = get_class(layer.geo_table, exclude_properties=exclude)
for column_property in class_mapper(cls).iterate_properties:
if isinstance(column_property, ColumnProperty) and \
len(column_property.columns) == 1 and \
not column_property.columns[0].primary_key and \
not column_property.columns[0].foreign_keys and \
not isinstance(column_property.columns[0].type, Geometry):
messages.append(
Message(
None, column_property.key, None, [], "", "",
(".".join(
["edit", layer.item_type,
str(layer.id)]), layer.name)))
except NoSuchTableError:
exit(
colorize(
"No such table '{}' for layer '{}'.".format(
layer.geo_table, layer.name), RED))
def __call__(self, filename, options):
config = get_config("config.yaml")
class Registry:
settings = config
class Request:
registry = Registry()
params = {}
GET = {}
user_agent = ""
@staticmethod
def static_url(*args, **kwargs):
return ""
@staticmethod
def static_path(*args, **kwargs):
return ""
@staticmethod
def route_url(*args, **kwargs):
return ""
@staticmethod
def current_route_url(*args, **kwargs):
return ""
init_region({"backend": "dogpile.cache.memory"})
int_filename = filename
if re.match(
"^" + re.escape("./{}/templates".format(config["package"])),
filename):
try:
empty_template = Template("")
class Lookup(TemplateLookup):
@staticmethod
def get_template(uri):
return empty_template
class MyTemplate(MakoTemplate):
def prepare(self, **options):
options.update({"input_encoding": self.encoding})
lookup = Lookup(**options)
if self.source:
self.tpl = Template(self.source,
lookup=lookup,
**options)
else:
self.tpl = Template(uri=self.name,
filename=self.filename,
lookup=lookup,
**options)
try:
processed = template(filename, {
"request": Request(),
"lang": "fr",
"debug": False,
"extra_params": {},
"permalink_themes": "",
"fulltextsearch_groups": [],
"wfs_types": [],
"_": lambda x: x,
},
template_adapter=MyTemplate)
int_filename = os.path.join(
os.path.dirname(filename),
"_" + os.path.basename(filename))
with open(int_filename, "wb") as file_open:
file_open.write(processed.encode("utf-8"))
except:
print(
colorize(
"ERROR! Occurred during the '{}' template generation"
.format(filename), RED))
print(colorize(traceback.format_exc(), RED))
if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
# Continue with the original one
int_filename = filename
else:
raise
except:
print(traceback.format_exc())
message_str = subprocess.check_output(
["node", "tools/extract-messages.js",
int_filename]).decode("utf-8")
if int_filename != filename:
os.unlink(int_filename)
try:
messages = []
for contexts, message in loads(message_str):
for context in contexts.split(", "):
messages.append(
Message(None, message, None, [], "", "",
context.split(":")))
return messages
except:
print(colorize("An error occurred", RED))
print(colorize(message_str, RED))
print("------")
raise