def _import_layer_attributes(self, url, layer, item_type, name, messages): attributes, layers = self._layer_attributes(url, layer) for sub_layer in layers: messages.append( Message( None, sub_layer, None, [], "", "", (".".join([item_type, name]), sub_layer.encode("ascii", "replace")), ) ) for attribute in attributes: messages.append( Message( None, attribute, None, [], "", "", (".".join([item_type, name]), layer.encode("ascii", "replace")), ) )
def _import_layer_attributes(self, url: str, layer: "main.Layer", item_type: str, name: str, messages: List[str]) -> None: attributes, layers = self._layer_attributes(url, layer) for sub_layer in layers: messages.append( Message( None, sub_layer, None, [], "", "", (".".join([item_type, name ]), sub_layer.encode("ascii", "replace")), )) for attribute in attributes: messages.append( Message( None, attribute, None, [], "", "", (".".join([item_type, name ]), layer.encode("ascii", "replace")), ))
def __call__(self, filename, options): with open(filename) as config_file: from c2cgeoportal.lib.print_ import * # noqa config = yaml.load(config_file) # for application config (.build/config.yaml) if "vars" in config: return [ Message(None, raster_layer, None, [], u"", u"", (filename, u"raster/%s" % raster_layer)) for raster_layer in config["vars"].get("raster", {}).keys() ] # for the print config elif "templates" in config: result = [] for template in config.get("templates").keys(): result.append( Message(None, template, None, [], u"", u"", (filename, u"template/%s" % template))) result += [ Message(None, attribute, None, [], u"", u"", (filename, u"template/%s/%s" % (template, attribute))) for attribute in config.get("templates")[template].attributes.keys() ] return result else: raise Exception("Not a known config file")
def _collect_app_config(cls, config, filename): # Collect raster layers names raster = [ Message(None, raster_layer, None, [], "", "", (filename, "raster/{}".format(raster_layer))) for raster_layer in list(config["vars"].get("raster", {}).keys()) ] # Collect layers enum values (for filters) settings = get_config("config.yaml") from c2cgeoportal.pyramid_ import init_dbsessions init_dbsessions(settings) from c2cgeoportal.models import DBSessions from c2cgeoportal.views.layers import Layers enums = [] enum_layers = config["vars"].get("layers", {}).get("enum", {}) for layername in list(enum_layers.keys()): layerinfos = enum_layers.get(layername, {}) attributes = layerinfos.get("attributes", {}) for fieldname in list(attributes.keys()): values = cls._enumerate_attributes_values( DBSessions, Layers, layerinfos, fieldname) for value, in values: if value != "": msgid = value if isinstance(value, str) else value location = "/layers/{}/values/{}/{}".format( layername, fieldname, value.encode("ascii", errors="replace") if isinstance(value, str) else value) enums.append( Message(None, msgid, None, [], "", "", (filename, location))) return raster + enums
def _collect_print_config(print_config: Dict[str, Any], filename: str) -> List[Message]: result = [] for template_ in list( cast(Dict[str, Any], print_config.get("templates")).keys()): assert template_ is not None result.append( Message(None, template_, None, [], "", "", (filename, f"template/{template_}"))) assert not [ attribute for attribute in list( print_config["templates"][template_]["attributes"].keys()) if attribute is None ] result += [ Message( None, attribute, None, [], "", "", (filename, f"template/{template_}/{attribute}"), ) for attribute in list(print_config["templates"][template_] ["attributes"].keys()) ] return result
def __call__(self, filename, options): messages = [] try: self.env = bootstrap(filename, options=os.environ) try: from c2cgeoportal_commons.models import DBSession from c2cgeoportal_commons.models.main import Theme, LayerGroup, \ LayerWMS, LayerWMTS, FullTextSearch self._import(Theme, messages) self._import(LayerGroup, messages) self._import(LayerWMS, messages, self._import_layer_wms) self._import(LayerWMTS, messages, self._import_layer_wmts) for ln, in DBSession.query( FullTextSearch.layer_name).distinct().all(): if ln is not None and ln != "": messages.append( Message( None, ln, None, [], "", "", ("fts", ln.encode("ascii", errors="replace")))) for ln, in DBSession.query( FullTextSearch.actions).distinct().all(): if ln is not None and ln != "": action = json.loads(ln) messages.append( Message(None, action['data'], None, [], "", "", ("fts", action['data'].encode( "ascii", errors="replace")))) except ProgrammingError as e: print( colorize( "ERROR! The database is probably not up to date " "(should be ignored when happen during the upgrade)", RED)) print(colorize(e, RED)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE": raise except NoSuchTableError as e: print( colorize( "ERROR! The schema didn't seem to exists " "(should be ignored when happen during the deploy)", RED)) print(colorize(e, RED)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE": raise except OperationalError as e: print( colorize( "ERROR! The database didn't seem to exists " "(should be ignored when happen during the deploy)", RED)) print(colorize(e, RED)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE": raise return messages
def _collect_print_config(cls, config, filename): result = [] for template_ in list(config.get("templates").keys()): result.append( Message(None, template_, None, [], "", "", (filename, "template/{}".format(template_)))) result += [ Message( None, attribute, None, [], "", "", (filename, "template/{}/{}".format(template_, attribute))) for attribute in list( config.get("templates")[template_].attributes.keys()) ] return result
def __call__(self, filename, options, fileobj=None, lineno=0): factories = list() trim = lambda x: x[1:-1] with open(filename, 'r+') as f: data = f.read() for match in REGEX_FACTORY.finditer(data): var, domain = match.groups() domain = trim(domain) if options.domain is None or options.domain == domain: factories.append(var) # pre calculation to now with lineno the string was found s = 0 linesum = list() for line in data.split('\n'): s += len(line) linesum.append(s) for factory in factories: regex = re.compile(REGEX_MSG % factory) for match in regex.finditer(data): msgid, default = match.groups() msgid, default = trim(msgid), trim(default) # calculate lineno start = match.start() for i, s in enumerate(linesum): lineno = i + 1 if start < s: break yield Message(None, msgid, None, [], 'Default: %s' % default, '', (filename, lineno))
def process_python(self, code, code_lineno, translator_strings): source = code.getvalue().strip() if source.endswith(compat.b(":")): if source in ( compat.b("try:"), compat.b("else:"), ) or source.startswith(compat.b("except")): source = compat.b("") # Ignore try/except and else elif source.startswith(compat.b("elif")): source = source[2:] # Replace "elif" with "if" source += compat.b("pass") code = io.BytesIO(source) for msg in self.python_extractor( self.filename, self.options, code, code_lineno - 1 ): if translator_strings: msg = Message( msg.msgctxt, msg.msgid, msg.msgid_plural, msg.flags, compat.u(" ").join(translator_strings + [msg.comment]), msg.tcomment, msg.location, ) yield msg
def _collect_app_config(self, filename): config.init(filename) settings = config.get_config() # Collect raster layers names raster = [ Message(None, raster_layer, None, [], "", "", (filename, "raster/{}".format(raster_layer))) for raster_layer in list(settings.get("raster", {}).keys()) ] # Collect layers enum values (for filters) class R: settings = None class C: registry = R() config_ = C() config_.registry.settings = settings init_dbsessions(settings, config_) from c2cgeoportal_commons import models c2cgeoportal_geoportal.init_dbsessions(settings, config_) from c2cgeoportal_geoportal.views.layers import Layers enums = [] enum_layers = settings.get("layers", {}).get("enum", {}) for layername in list(enum_layers.keys()): layerinfos = enum_layers.get(layername, {}) attributes = layerinfos.get("attributes", {}) for fieldname in list(attributes.keys()): values = self._enumerate_attributes_values( models.DBSessions, Layers, layerinfos, fieldname) for value, in values: if value != "": msgid = value if isinstance(value, str) else value location = "/layers/{}/values/{}/{}".format( layername, fieldname, value.encode("ascii", errors="replace") if isinstance(value, str) else value) enums.append( Message(None, msgid, None, [], "", "", (filename, location))) return raster + enums
def _import(self, object_type, messages, callback=None): from c2cgeoportal.models import DBSession items = DBSession.query(object_type) for item in items: messages.append( Message(None, item.name, None, [], u"", u"", (".".join([item.item_type, item.name]), item.id))) if callback is not None: callback(item, messages)
def _collect_print_config(print_config: Dict[str, Any], filename: str) -> List[Message]: result = [] for template_ in list( cast(Dict[str, Any], print_config.get("templates")).keys()): result.append( Message(None, template_, None, [], "", "", (filename, "template/{}".format(template_)))) result += [ Message( None, attribute, None, [], "", "", (filename, "template/{}/{}".format(template_, attribute)), ) for attribute in list(print_config["templates"][template_] ["attributes"].keys()) ] return result
def _import(object_type, messages, callback=None): from c2cgeoportal.models import DBSession items = DBSession.query(object_type) for item in items: messages.append( Message(None, item.name, None, [], "", "", (item.item_type, item.name.encode("ascii", errors="replace")))) if callback is not None: callback(item, messages)
def process_python(self, code, code_lineno, translator_strings): source = code.getvalue().strip() if source.endswith(compat.b(':')): source += compat.b(' pass') code = io.BytesIO(source) for msg in self.python_extractor(self.filename, self.options, code, code_lineno): if translator_strings: msg = Message( msg.msgctxt, msg.msgid, msg.msgid_plural, msg.flags, compat.u(' ').join(translator_strings + [msg.comment]), msg.tcomment, msg.location) yield msg
def _import_layer_wms(self, layer: "main.Layer", messages: List[str]) -> None: server = layer.ogc_server url = server.url_wfs or server.url if url is None: return if layer.ogc_server.wfs_support: for wms_layer in layer.layer.split(","): self._import_layer_attributes(url, wms_layer, layer.item_type, layer.name, messages) if layer.geo_table is not None and layer.geo_table != "": try: cls = get_layer_class(layer, with_last_update_columns=True) for column_property in class_mapper(cls).iterate_properties: if isinstance(column_property, ColumnProperty) and len( column_property.columns) == 1: column = column_property.columns[0] if not column.primary_key and not isinstance( column.type, Geometry): if column.foreign_keys: if column.name == "type_id": name = "type_" elif column.name.endswith("_id"): name = column.name[:-3] else: name = column.name + "_" else: name = column_property.key messages.append( Message( None, name, None, [], "", "", (".".join([ "edit", layer.item_type, str(layer.id) ]), layer.name), )) except NoSuchTableError: print( colorize( "ERROR! No such table '{}' for layer '{}'.".format( layer.geo_table, layer.name), Color.RED, )) print(colorize(traceback.format_exc(), Color.RED)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE": raise
def __call__( self, filename: str, options: Dict[str, Any], fileobj: Optional[Dict[str, Any]] = None, lineno: int = 0, ) -> List[Message]: del options, fileobj, lineno print(f"Running {self.__class__.__name__} on {filename}") with open(filename, encoding="utf8") as config_file: data = config_file.read() settings = yaml.load( data.replace("{{cookiecutter.geomapfish_main_version}}", os.environ["MAJOR_VERSION"]), Loader=yaml.SafeLoader, ) admin_interface = settings.get("vars", {}).get("admin_interface", {}) available_metadata = [] for elem in admin_interface.get("available_metadata", []): if "description" in elem: location = f"admin_interface/available_metadata/{elem.get('name', '')}" available_metadata.append( Message(None, elem["description"].strip(), None, [], "", "", (filename, location))) available_functionalities = [] for elem in admin_interface.get("available_functionalities", []): if "description" in elem: location = f"admin_interface/available_functionalities/{elem.get('name', '')}" available_functionalities.append( Message(None, elem["description"].strip(), None, [], "", "", (filename, location))) return available_metadata + available_functionalities
def _insert_attribute(self, attribute, location): if attribute not in self.fields: print(f'Adding attribute {attribute}') self.fields.add(attribute) self.messages.append( Message( None, u'%(name)s' % {'name': attribute}, None, [], "", "", location, ))
def __call__(self, filename, options): message_str = subprocess.check_output( ["node", "tools/extract-messages.js", filename]) try: messages = loads(message_str) return [ Message(None, message, None, [], u"", u"", context.split(":")) for context, message in messages ] except: print("An error occurred") print(message_str) print("------") raise
def __call__(self, filename, options): message_str = subprocess.check_output( ["node", "tools/extract-messages.js", filename]) try: messages = [] for contexts, message in loads(message_str): for context in contexts.split(", "): messages.append( Message(None, message, None, [], u"", u"", context.split(":"))) return messages except: print(colorize("An error occurred", RED)) print(colorize(message_str, RED)) print("------") raise
def _import_layer_wms(self, layer, messages): server = layer.ogc_server url = server.url_wfs or server.url if url is None: return for wms_layer in layer.layer.split(","): self._import_layer_attributes(url, wms_layer, layer.item_type, layer.name, messages) if layer.geo_table is not None and layer.geo_table != "": exclude = [] if layer.exclude_properties is None else layer.exclude_properties.split( ",") last_update_date = layer.get_metadatas("lastUpdateDateColumn") if len(last_update_date) == 1: exclude.append(last_update_date[0].value) last_update_user = layer.get_metadatas("lastUpdateUserColumn") if len(last_update_user) == 1: exclude.append(last_update_user[0].value) try: cls = get_class(layer.geo_table, exclude_properties=exclude) for column_property in class_mapper(cls).iterate_properties: if isinstance(column_property, ColumnProperty) and len( column_property.columns) == 1: column = column_property.columns[0] if not column.primary_key and not isinstance( column.type, Geometry): if column.foreign_keys: name = "type_" if column.name == "type_id" else \ column.name[0:column.name.rindex("_id")] else: name = column_property.key messages.append( Message(None, name, None, [], "", "", (".".join([ "edit", layer.item_type, str(layer.id) ]), layer.name))) except NoSuchTableError: print( colorize( "ERROR! No such table '{}' for layer '{}'.".format( layer.geo_table, layer.name), RED)) print(colorize(traceback.format_exc(), RED)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE": raise
def __call__(self, filename, options, fileobj=None, lineno=0): self.filename = filename if fileobj is None: fileobj = io.open(filename, encoding='utf-8') parser_func = parsers.get(self.syntax) source = fileobj.read() if isinstance(source, bytes): source = source.decode('UTF-8') tree = parser_func(filename, source, translatable=True) for node in tree: if isinstance(node, TranslatableText): yield Message(None, node.text, None, [], u'', u'', (filename, lineno + (node.position[0] or 1))) elif isinstance(node, Expression): for m in self.parse_python(node, filename, lineno, options): yield m
def _import(object_type, messages, callback=None): from c2cgeoportal_commons.models import DBSession # pylint: disable=import-outside-toplevel items = DBSession.query(object_type) for item in items: messages.append( Message( None, item.name, None, [], "", "", (item.item_type, item.name.encode("ascii", errors="replace")), ) ) if callback is not None: callback(item, messages)
def __call__(self, filename, options): # FIXME find a better way to load settings settings = load_full_settings("c2c://development.ini") engine = engine_from_config(settings, "sqlalchemy.") dbsession = get_session_factory(engine)() messages = [] for rec in dbsession.query(HazardLevel): messages.append((rec.title, type(rec).__name__)) for rec in dbsession.query(HazardType): messages.append((rec.title, type(rec).__name__)) return [ Message(None, text, None, [], class_name, "", (filename, 1)) for text, class_name in messages if text != "" and text is not None ]
def __call__(self, filename, options): configuration.init(filename) settings = configuration.get_config() for path in ( ("getitfixed", "admin_new_issue_email", "email_subject"), ("getitfixed", "admin_new_issue_email", "email_body"), ("getitfixed", "new_issue_email", "email_subject"), ("getitfixed", "new_issue_email", "email_body"), ("getitfixed", "update_issue_email", "email_subject"), ("getitfixed", "update_issue_email", "email_body"), ("getitfixed", "resolved_issue_email", "email_subject"), ("getitfixed", "resolved_issue_email", "email_body"), ): value = settings for key in path: value = value[key] # yield Message(msgctxt msgid msgid_plural flags comment tcomment location) yield Message(None, value, None, [], u"", u"", (filename, "/".join(path)))
def __call__(self, filename, options): # FIXME find a better way to load settings settings = load_full_settings('development.ini') engine = engine_from_config(settings, 'sqlalchemy.') DBSession.configure(bind=engine) messages = [] for rec in DBSession.query(HazardLevel): messages.append((rec.title, type(rec).__name__)) for rec in DBSession.query(HazardType): messages.append((rec.title, type(rec).__name__)) return [ Message(None, text, None, [], class_name, u'', (filename, 1)) for text, class_name in messages if text != '' and text is not None ]
def __call__(self, filename, options): # FIXME find a better way to load settings settings = load_full_settings("c2c://development.ini") engine = engine_from_config(settings, "sqlalchemy.") dbsession = get_session_factory(engine)() messages = [] for rec in dbsession.query(ClimateChangeRecommendation): messages.append((rec.text, type(rec).__name__)) for rec in dbsession.query(HazardCategory): messages.append((rec.general_recommendation, type(rec).__name__)) for rec in dbsession.query(TechnicalRecommendation): messages.append((rec.text, type(rec).__name__)) messages.append((rec.detail, type(rec).__name__)) return [ Message(None, text, None, [], class_name, "", (filename, 1)) for text, class_name in messages if text != "" and text is not None ]
def _import( object_type: Type[Any], messages: List[str], callback: Optional[Callable[["main.Layer", List[str]], None]] = None, has_interfaces: bool = True, name_regex: str = ".*", ) -> None: from c2cgeoportal_commons.models import DBSession # pylint: disable=import-outside-toplevel from c2cgeoportal_commons.models.main import Interface # pylint: disable=import-outside-toplevel filter_re = re.compile(name_regex) query = DBSession.query(object_type) interfaces = _get_config("INTERFACES") if has_interfaces and interfaces is not None: query.join(object_type.interface).filter( Interface.name in interfaces.split(".")) for item in query.all(): assert item.name is not None if filter_re.match(item.name): messages.append( Message( None, item.name, None, [], "", "", (item.item_type, item.name.encode("ascii", errors="replace")), )) if callback is not None: callback(item, messages)
def __call__(self, filename, options): # FIXME find a better way to load settings settings = load_full_settings('development.ini') engine = engine_from_config(settings, 'sqlalchemy.') DBSession.configure(bind=engine) messages = [] for rec in DBSession.query(ClimateChangeRecommendation): messages.append((rec.text, type(rec).__name__)) for rec in DBSession.query(HazardCategory): messages.append((rec.general_recommendation, type(rec).__name__)) for rec in DBSession.query(TechnicalRecommendation): messages.append((rec.text, type(rec).__name__)) messages.append((rec.detail, type(rec).__name__)) return [ Message(None, text, None, [], class_name, u'', (filename, 1)) for text, class_name in messages if text != '' and text is not None ]
def _import_layer_wms(self, layer, messages): server = layer.ogc_server url = server.url_wfs or server.url if url is None: return for wms_layer in layer.layer.split(","): self._import_layer_attributes(url, wms_layer, layer.item_type, layer.name, layer.id, messages) if layer.geo_table is not None and layer.geo_table != "": exclude = [] if layer.exclude_properties is None else layer.exclude_properties.split( ",") last_update_date = layer.get_metadatas("lastUpdateDateColumn") if len(last_update_date) == 1: exclude.append(last_update_date[0].value) last_update_user = layer.get_metadatas("lastUpdateUserColumn") if len(last_update_user) == 1: exclude.append(last_update_user[0].value) try: cls = get_class(layer.geo_table, exclude_properties=exclude) for column_property in class_mapper(cls).iterate_properties: if isinstance(column_property, ColumnProperty) and \ len(column_property.columns) == 1 and \ not column_property.columns[0].primary_key and \ not column_property.columns[0].foreign_keys and \ not isinstance(column_property.columns[0].type, Geometry): messages.append( Message( None, column_property.key, None, [], "", "", (".".join( ["edit", layer.item_type, str(layer.id)]), layer.name))) except NoSuchTableError: exit( colorize( "No such table '{}' for layer '{}'.".format( layer.geo_table, layer.name), RED))
def __call__(self, filename, options): config = get_config("config.yaml") class Registry: settings = config class Request: registry = Registry() params = {} GET = {} user_agent = "" @staticmethod def static_url(*args, **kwargs): return "" @staticmethod def static_path(*args, **kwargs): return "" @staticmethod def route_url(*args, **kwargs): return "" @staticmethod def current_route_url(*args, **kwargs): return "" init_region({"backend": "dogpile.cache.memory"}) int_filename = filename if re.match( "^" + re.escape("./{}/templates".format(config["package"])), filename): try: empty_template = Template("") class Lookup(TemplateLookup): @staticmethod def get_template(uri): return empty_template class MyTemplate(MakoTemplate): def prepare(self, **options): options.update({"input_encoding": self.encoding}) lookup = Lookup(**options) if self.source: self.tpl = Template(self.source, lookup=lookup, **options) else: self.tpl = Template(uri=self.name, filename=self.filename, lookup=lookup, **options) try: processed = template(filename, { "request": Request(), "lang": "fr", "debug": False, "extra_params": {}, "permalink_themes": "", "fulltextsearch_groups": [], "wfs_types": [], "_": lambda x: x, }, template_adapter=MyTemplate) int_filename = os.path.join( os.path.dirname(filename), "_" + os.path.basename(filename)) with open(int_filename, "wb") as file_open: file_open.write(processed.encode("utf-8")) except: print( colorize( "ERROR! Occurred during the '{}' template generation" .format(filename), RED)) print(colorize(traceback.format_exc(), RED)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE": # Continue with the original one int_filename = filename else: raise except: print(traceback.format_exc()) message_str = subprocess.check_output( ["node", "tools/extract-messages.js", int_filename]).decode("utf-8") if int_filename != filename: os.unlink(int_filename) try: messages = [] for contexts, message in loads(message_str): for context in contexts.split(", "): messages.append( Message(None, message, None, [], "", "", context.split(":"))) return messages except: print(colorize("An error occurred", RED)) print(colorize(message_str, RED)) print("------") raise