def files_to_remove(self, pre=False): error = False for element in self.get_upgrade("files_to_remove"): file_ = element["file"] if os.path.exists(file_): managed = False for pattern in self.project["managed_files"]: if re.match(pattern + '$', file_): print(colorize( "The file '{}' is no more use but not delete " "because he is in the managed_files as '{}'.".format(file_, pattern), RED )) error = True managed = True if not managed and not pre: print(colorize("The file '{}' is removed.".format(file_), GREEN)) if "version" in element and "from" in element: print("Was used in version {}, to be removed from version {}.".format( element["from"], element["version"] )) if os.path.isdir(file_): shutil.rmtree(file_) else: os.remove(file_) return error
def test_checkers(self) -> Tuple[bool, Optional[str]]: run_curl = "Run `curl --insecure {} '{}'` for more information.".format( " ".join([ "--header {}={}".format(*i) for i in self.project.get("checker_headers", {}).items() ]), self.project["checker_url"], ) try: requests.packages.urllib3.disable_warnings() resp = requests.get(self.project["checker_url"], headers=self.project.get("checker_headers"), verify=False) except requests.exceptions.ConnectionError as exception: return False, "\n".join( ["Connection error: {}".format(exception), run_curl]) except ConnectionRefusedError as exception: return False, "\n".join( ["Connection refused: {}".format(exception), run_curl]) if resp.status_code < 200 or resp.status_code >= 300: print(colorize("=============", Color.RED)) print(colorize("Checker error", Color.RED)) for name, value in resp.json()["failures"].items(): print( colorize("Test '{}' failed with result:".format(name), Color.YELLOW)) del value["level"] del value["timing"] print(yaml.dump(value) if value != {} else "No result") return False, "\n".join(["Checker error:", run_curl]) return True, None
def files_to_remove(self, element, prefix=""): for path in element["paths"]: file_ = os.path.join( prefix, path.format(package=self.project["project_package"])) if os.path.exists(file_): managed = False for pattern in self.project["managed_files"]: if re.match(pattern + '$', file_): print( colorize( "The file '{}' is no longer used, but not deleted " "because it is in the managed_files as '{}'.". format(file_, pattern), RED)) managed = True if not managed: print( colorize("The file '{}' is removed.".format(file_), GREEN)) if "version" in element and "from" in element: print( "Was used in version {}, to be removed from version {}." .format(element["from"], element["version"])) if os.path.isdir(file_): shutil.rmtree(file_) else: os.remove(file_)
def print_step(self, step, error=False, message=None, prompt="To continue type:"): print("") print(self.color_bar) if message is not None: print(colorize(message, RED if error else YELLOW)) if step >= 0: print(colorize(prompt, GREEN)) args = " --makefile=" + self.options.new_makefile if self.options.makefile != "Makefile" else "" print(colorize("./docker-run make{} upgrade{}", GREEN).format( args, step if step != 0 else "", ))
def print_step(self, step, error=False, message=None, prompt="To continue, type:"): print("") print(self.color_bar) if message is not None: print(colorize(message, RED if error else YELLOW)) if step >= 0: print(colorize(prompt, GREEN)) cmd = [ "./upgrade", os.environ['VERSION'] ] if step != 0: cmd.append("{}".format(step)) print(colorize(" ".join(cmd), GREEN))
def _import_layer_wms(self, layer: "main.Layer", messages: List[str]) -> None: server = layer.ogc_server url = server.url_wfs or server.url if url is None: return if layer.ogc_server.wfs_support: for wms_layer in layer.layer.split(","): self._import_layer_attributes(url, wms_layer, layer.item_type, layer.name, messages) if layer.geo_table is not None and layer.geo_table != "": try: cls = get_layer_class(layer, with_last_update_columns=True) for column_property in class_mapper(cls).iterate_properties: if isinstance(column_property, ColumnProperty) and len( column_property.columns) == 1: column = column_property.columns[0] if not column.primary_key and not isinstance( column.type, Geometry): if column.foreign_keys: if column.name == "type_id": name = "type_" elif column.name.endswith("_id"): name = column.name[:-3] else: name = column.name + "_" else: name = column_property.key messages.append( Message( None, name, None, [], "", "", (".".join([ "edit", layer.item_type, str(layer.id) ]), layer.name), )) except NoSuchTableError: print( colorize( "ERROR! No such table '{}' for layer '{}'.".format( layer.geo_table, layer.name), Color.RED, )) print(colorize(traceback.format_exc(), Color.RED)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE": raise
def files_to_remove(self, element: Dict[str, Any], prefix: str = "", force: bool = False) -> bool: task_to_do = False for path in element["paths"]: file_ = os.path.join( prefix, path.format(package=self.project["project_package"])) if os.path.exists(file_): managed = False if not force: for files in self.project["managed_files"]: if isinstance(files, str): pattern = files no_touch = False else: pattern = files["pattern"] no_touch = files.get("no_touch", False) if re.match(pattern + "$", file_): if no_touch: managed = True else: print( colorize( f"The file '{file_}' has been removed but he is in the " f"`managed_files` as '{pattern}'.", Color.RED, )) task_to_do = True for pattern in self.project.get("unmanaged_files", []): if re.match(pattern + "$", file_): print( colorize( f"The file '{file_}' has been removed but he is in the " f"`unmanaged_files` as '{pattern}'.", Color.YELLOW, )) task_to_do = True if not managed: print(f"The file '{file_}' is removed.") if "version" in element and "from" in element: print( f"Was used in version {element['from']}, to be removed from version " f"{element['version']}.") if os.path.isdir(file_): shutil.rmtree(file_) else: os.remove(file_) return task_to_do
def files_to_move(self, element, prefix="", force=False): task_to_do = False src = os.path.join( prefix, element["from"].format(package=self.project["project_package"])) dst = os.path.join( prefix, element["to"].format(package=self.project["project_package"])) if os.path.exists(src): managed = False type_ = "directory" if os.path.isdir(src) else "file" if not force: for pattern in self.project["managed_files"]: if re.match(pattern + '$', src): print( colorize( "The {} '{}' is present in the managed_files as '{}', but it will move." .format(type_, src, pattern), RED)) task_to_do = True managed = True break if re.match(pattern + '$', dst): print( colorize( "The {} '{}' is present in the managed_files as '{}', but it will move." .format(type_, dst, pattern), RED)) task_to_do = True managed = True break if not managed and os.path.exists(dst): print( colorize( "The destination '{}' already exists, ignoring.". format(dst), YELLOW)) if not managed: print( colorize( "Move the {} '{}' to '{}'.".format(type_, src, dst), GREEN)) if "version" in element: print("Needed from version {}.".format(element["version"])) if os.path.dirname(dst) != "": os.makedirs(os.path.dirname(dst), exist_ok=True) try: check_call(["git", "mv", src, dst]) except Exception as e: print("[Warning] Git move error: {}.".format(e)) os.rename(src, dst) return task_to_do
def _import_layer_wmts(self, layer, messages): from c2cgeoportal_commons.models import DBSession from c2cgeoportal_commons.models.main import OGCServer layers = [d.value for d in layer.metadatas if d.name == "queryLayers"] if len(layers) == 0: layers = [d.value for d in layer.metadatas if d.name == "wmsLayer"] server = [d.value for d in layer.metadatas if d.name == "ogcServer"] if len(server) >= 1 and len(layers) >= 1: for wms_layer in layers: try: db_server = DBSession.query(OGCServer).filter(OGCServer.name == server[0]).one() self._import_layer_attributes( db_server.url_wfs or db_server.url, wms_layer, layer.item_type, layer.name, messages ) except NoResultFound: print(colorize( "ERROR! the OGC server '{}' from the WMTS layer '{}' does not exist.".format( server[0], layer.name ), RED )) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE": raise
def files_to_get(self, pre=False): error = False for root, _, files in os.walk("CONST_create_template"): root = root[len("CONST_create_template/"):] for file_ in files: destination = os.path.join(root, file_) managed = False for pattern in self.project["managed_files"] + self.get_upgrade("default_project_file"): if re.match(pattern + '$', destination): managed = True for unpattern in self.project.get("unmanaged_files", []): if re.match(unpattern + '$', destination): managed = False break break source = os.path.join("CONST_create_template", destination) if not managed and (not os.path.exists(destination) or not filecmp.cmp(source, destination)): if not pre: print(colorize( "Get the file '{}' from the create template.".format(destination), GREEN )) if os.path.dirname(destination) != "": os.makedirs(os.path.dirname(destination), exist_ok=True) shutil.copyfile(source, destination) shutil.copymode(source, destination) return error
def get_project(): if not os.path.isfile("project.yaml"): print(colorize("Unable to find the required 'project.yaml' file.", RED)) exit(1) with open("project.yaml", "r") as f: return yaml.safe_load(f)
def _import_layer_wmts(self, layer: "main.Layer", messages: List[str]) -> None: from c2cgeoportal_commons.models import DBSession # pylint: disable=import-outside-toplevel from c2cgeoportal_commons.models.main import OGCServer # pylint: disable=import-outside-toplevel layers = [d.value for d in layer.metadatas if d.name == "queryLayers"] if not layers: layers = [d.value for d in layer.metadatas if d.name == "wmsLayer"] server = [d.value for d in layer.metadatas if d.name == "ogcServer"] if server and layers: layers = [layer for ls in layers for layer in ls.split(",")] for wms_layer in layers: try: db_server = DBSession.query(OGCServer).filter( OGCServer.name == server[0]).one() if db_server.wfs_support: self._import_layer_attributes( db_server.url_wfs or db_server.url, wms_layer, layer.item_type, layer.name, messages, ) except NoResultFound: print( colorize( "ERROR! the OGC server '{}' from the WMTS layer '{}' does not exist." .format(server[0], layer.name), Color.RED, )) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE": raise
def _enumerate_attributes_values(dbsessions: Dict[str, Session], layerinfos: Dict[str, Any], fieldname: str) -> List[Message]: dbname = layerinfos.get("dbsession", "dbsession") translate = cast(Dict[str, Any], layerinfos["attributes"]).get( fieldname, {}).get("translate", True) if not translate: return [] try: dbsession = dbsessions.get(dbname) return Layers.query_enumerate_attribute_values( dbsession, layerinfos, fieldname) except Exception as e: table = cast(Dict[str, Any], layerinfos["attributes"]).get(fieldname, {}).get("table") print( colorize( "ERROR! Unable to collect enumerate attributes for " "db: {}, table: {}, column: {} ({})".format( dbname, table, fieldname, e), Color.RED, )) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE": return [] raise
def files_to_get(self, step, pre=False): error = False for root, _, files in os.walk("CONST_create_template"): root = root[len("CONST_create_template/"):] for file_ in files: destination = os.path.join(root, file_) managed = self.is_managed(destination) source = os.path.join("CONST_create_template", destination) if not managed and (not os.path.exists(destination) or not filecmp.cmp(source, destination)): print(colorize( "Get the file '{}' from the create template.".format(destination), GREEN )) if not pre: if os.path.dirname(destination) != "": os.makedirs(os.path.dirname(destination), exist_ok=True) try: shutil.copyfile(source, destination) shutil.copymode(source, destination) except PermissionError as e: self.print_step( step, error=True, message=( "All your project files should be own by your user, " "current error:\n" + str(e) ), prompt="Fix it and run again the upgrade:") exit(1) elif os.path.exists(destination) and not filecmp.cmp(source, destination): print("The file '{}' is managed by the project".format(destination)) else: print("The file '{}' does not change".format(destination)) return error
def get_upgrade(section): if not os.path.isfile(".upgrade.yaml"): print(colorize("Unable to find the required '.upgrade.yaml' file.", RED)) exit(1) with open(".upgrade.yaml", "r") as f: return yaml.safe_load(f)[section]
def _enumerate_attributes_values(dbsessions: Dict[str, Session], layerinfos: Dict[str, Any], fieldname: str) -> Set[Tuple[str, ...]]: dbname = layerinfos.get("dbsession", "dbsession") translate = cast(Dict[str, Any], layerinfos["attributes"]).get( fieldname, {}).get("translate", True) if not translate: return set() try: dbsession = dbsessions.get(dbname) return Layers.query_enumerate_attribute_values( dbsession, layerinfos, fieldname) except Exception as e: table = cast(Dict[str, Any], layerinfos["attributes"]).get(fieldname, {}).get("table") print( colorize( "ERROR! Unable to collect enumerate attributes for " f"db: {dbname}, table: {table}, column: {fieldname} ({e!s})", Color.RED, )) if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") == "TRUE": return set() raise
def _import_layer_wmts(self, layer, messages): from c2cgeoportal_commons.models import DBSession from c2cgeoportal_commons.models.main import OGCServer layers = [d.value for d in layer.metadatas if d.name == "queryLayers"] if len(layers) == 0: layers = [d.value for d in layer.metadatas if d.name == "wmsLayer"] server = [d.value for d in layer.metadatas if d.name == "ogcServer"] if len(server) >= 1 and len(layers) >= 1: for wms_layer in layers: try: db_server = DBSession.query(OGCServer).filter(OGCServer.name == server[0]).one() self._import_layer_attributes( db_server.url_wfs or db_server.url, wms_layer, layer.item_type, layer.name, messages ) except NoResultFound: print(colorize( "ERROR! the OGC server '{}' from the WMTS layer '{}' does not exist.".format( server[0], layer.name ), RED )) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE": raise
def print_step(self, step, error=False, message=None, prompt="To continue, type:"): print("") print(self.color_bar) if message is not None: print(colorize(message, RED if error else YELLOW)) if step >= 0: print(colorize(prompt, GREEN)) if self.options.use_makefile: args = " --makefile={}".format(self.options.makefile) \ if self.options.makefile != "Makefile" else "" print( colorize( "./docker-run --home make{} upgrade{}".format( args, step if step != 0 else "", ), GREEN)) else: cmd = [ "./docker-run", "--home", "--image=camptocamp/geomapfish-build", "--version=" + pkg_resources.get_distribution( "c2cgeoportal_commons").version, "c2cupgrade", ] if self.options.windows: cmd.append("--windows") if self.options.nondocker: cmd.append("--nondocker") if self.options.force_docker: cmd.append("--force-docker") if self.options.git_remote != "origin": cmd.append("--git-remote={}".format( self.options.git_remote)) if self.options.makefile != "Makefile": cmd.append("--makefile={}".format(self.options.makefile)) if self.options.new_makefile != self.options.makefile: cmd.append("--new-makefile={}".format( self.options.new_makefile)) if step != 0: cmd.append("--step={}".format(step)) print(colorize(" ".join(cmd), GREEN))
def files_to_remove(self, element, prefix="", force=False): task_to_do = False for path in element["paths"]: file_ = os.path.join( prefix, path.format(package=self.project["project_package"])) if os.path.exists(file_): managed = False if not force: for files in self.project["managed_files"]: if isinstance(files, str): pattern = files no_touch = False else: pattern = files["pattern"] no_touch = files.get("no_touch", False) if re.match(pattern + "$", file_): if no_touch: managed = True else: # fmt: off print( colorize( "The file '{}' has been removed but he is in the `managed_files` as '{}'." .format(file_, pattern), RED)) # fmt: on task_to_do = True for pattern in self.project.get("unmanaged_files", []): if re.match(pattern + "$", file_): # fmt: off print( colorize( "The file '{}' has been removed but he is in the `unmanaged_files` as '{}'." .format(file_, pattern), YELLOW)) # fmt: on task_to_do = True if not managed: print("The file '{}' is removed.".format(file_)) if "version" in element and "from" in element: print( "Was used in version {}, to be removed from version {}." .format(element["from"], element["version"])) if os.path.isdir(file_): shutil.rmtree(file_) else: os.remove(file_) return task_to_do
def files_to_move(self, pre=False, prefix="", force=False): error = False for element in self.get_upgrade("files_to_move"): src = os.path.join(prefix, element["from"]) dst = os.path.join(prefix, element["to"]) if os.path.exists(src): managed = False type_ = "directory" if os.path.isdir(src) else "file" if not force: for pattern in self.project["managed_files"]: if re.match(pattern + '$', src): print(colorize( "The {} '{}' is present in the managed_files as '{}', but it will move." .format( type_, src, pattern ), RED )) error = True managed = True break if re.match(pattern + '$', dst): print(colorize( "The {} '{}' is present in the managed_files as '{}', but it will move." .format( type_, dst, pattern ), RED )) error = True managed = True break if not managed and os.path.exists(dst): print(colorize( "The destination '{}' already exists, ignoring.".format(dst), YELLOW )) if not managed and not pre: print(colorize("Move the {} '{}' to '{}'.".format(type_, src, dst), GREEN)) if "version" in element: print("Needed from version {}.".format(element["version"])) if os.path.dirname(dst) != "": os.makedirs(os.path.dirname(dst), exist_ok=True) os.rename(src, dst) return error
def get_project() -> Dict[str, Any]: if not os.path.isfile("project.yaml"): print( colorize("Unable to find the required 'project.yaml' file.", Color.RED)) sys.exit(1) with open("project.yaml", encoding="utf8") as project_file: return cast(Dict[str, Any], yaml.safe_load(project_file))
def _import_layer_wms(self, layer, messages): server = layer.ogc_server url = server.url_wfs or server.url if url is None: return for wms_layer in layer.layer.split(","): self._import_layer_attributes(url, wms_layer, layer.item_type, layer.name, messages) if layer.geo_table is not None and layer.geo_table != "": exclude = [] if layer.exclude_properties is None else layer.exclude_properties.split( ",") last_update_date = layer.get_metadatas("lastUpdateDateColumn") if len(last_update_date) == 1: exclude.append(last_update_date[0].value) last_update_user = layer.get_metadatas("lastUpdateUserColumn") if len(last_update_user) == 1: exclude.append(last_update_user[0].value) try: cls = get_class(layer.geo_table, exclude_properties=exclude) for column_property in class_mapper(cls).iterate_properties: if isinstance(column_property, ColumnProperty) and len( column_property.columns) == 1: column = column_property.columns[0] if not column.primary_key and not isinstance( column.type, Geometry): if column.foreign_keys: name = "type_" if column.name == "type_id" else \ column.name[0:column.name.rindex("_id")] else: name = column_property.key messages.append( Message(None, name, None, [], "", "", (".".join([ "edit", layer.item_type, str(layer.id) ]), layer.name))) except NoSuchTableError: print( colorize( "ERROR! No such table '{}' for layer '{}'.".format( layer.geo_table, layer.name), RED)) print(colorize(traceback.format_exc(), RED)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE": raise
def print_step(self, step: int, error: bool = False, message: str = None, prompt: str = "To continue, type:") -> None: with open(".UPGRADE_INSTRUCTIONS", "w") as instructions: print("") print(self.color_bar) if message is not None: print(colorize(message, Color.RED if error else Color.YELLOW)) instructions.write("{}\n".format(message)) if step >= 0: print(colorize(prompt, Color.GREEN)) instructions.write("{}\n".format(prompt)) cmd = ["./upgrade", os.environ["VERSION"]] if step != 0: cmd.append("{}".format(step)) print(colorize(" ".join(cmd), Color.GREEN)) instructions.write("{}\n".format(" ".join(cmd)))
def get_upgrade(section: str) -> Union[List[Any], Dict[str, Any]]: if not os.path.isfile(".upgrade.yaml"): print( colorize("Unable to find the required '.upgrade.yaml' file.", Color.RED)) sys.exit(1) with open(".upgrade.yaml", encoding="utf8") as project_file: return cast(Union[List[Any], Dict[str, Any]], yaml.safe_load(project_file)[section])
def files_to_move(self, pre=False): error = False for element in self.get_upgrade("files_to_move"): src = element["from"] dst = element["to"] if os.path.exists(src): type_ = "directory" if os.path.isdir(src) else "file" managed = False for pattern in self.project["managed_files"]: if re.match(pattern + '$', src): print(colorize( "The {} '{}' is present in the managed_files as '{}' but he will move.".format( type_, src, pattern), RED )) error = True managed = True break if re.match(pattern + '$', dst): print(colorize( "The {} '{}' is present in the managed_files as '{}' but he will move.".format( type_, dst, pattern), RED )) error = True managed = True break if not managed and os.path.exists(dst): print(colorize( "The destination '{}' already exists.".format(dst), RED )) error = True if not pre: raise InteruptedException("The destination '{}' already exists.".format(dst)) if not managed and not pre: print(colorize("Move the {} '{}' to '{}'.".format(type_, src, dst), GREEN)) if "version" in element: print("Needed from version {}.".format(element["version"])) if os.path.dirname(dst) != "": os.makedirs(os.path.dirname(dst), exist_ok=True) os.rename(src, dst) return error
def test_checkers(self) -> Tuple[bool, Optional[str]]: headers = " ".join([ f"--header {i[0]}={i[1]}" for i in self.project.get("checker_headers", {}).items() ]) run_curl = f"Run `curl --insecure {headers} '{self.project['checker_url']}'` for more information." try: requests.packages.urllib3.disable_warnings() # type: ignore resp = requests.get( self.project["checker_url"], headers=self.project.get("checker_headers"), verify=False, # nosec ) except requests.exceptions.ConnectionError as exception: return False, "\n".join( [f"Connection error: {exception}", run_curl]) except ConnectionRefusedError as exception: return False, "\n".join( [f"Connection refused: {exception}", run_curl]) if resp.status_code < 200 or resp.status_code >= 300: print(colorize("=============", Color.RED)) print(colorize("Checker error", Color.RED)) try: for name, value in resp.json()["failures"].items(): print( colorize(f"Test '{name}' failed with result:", Color.YELLOW)) del value["level"] del value["timing"] print(yaml.dump(value) if value != {} else "No result") except JSONDecodeError: print( colorize( f"Response is not a JSON '{resp.text}', {resp.reason} {resp.status_code}", Color.RED, )) return False, "\n".join(["Checker error:", run_curl]) return True, None
def __call__(self, filename, options): messages = [] try: self.env = bootstrap(filename, options=os.environ) try: from c2cgeoportal_commons.models import DBSession from c2cgeoportal_commons.models.main import Theme, LayerGroup, \ LayerWMS, LayerWMTS, FullTextSearch self._import(Theme, messages) self._import(LayerGroup, messages) self._import(LayerWMS, messages, self._import_layer_wms) self._import(LayerWMTS, messages, self._import_layer_wmts) for ln, in DBSession.query( FullTextSearch.layer_name).distinct().all(): if ln is not None and ln != "": messages.append( Message( None, ln, None, [], "", "", ("fts", ln.encode("ascii", errors="replace")))) for ln, in DBSession.query( FullTextSearch.actions).distinct().all(): if ln is not None and ln != "": action = json.loads(ln) messages.append( Message(None, action['data'], None, [], "", "", ("fts", action['data'].encode( "ascii", errors="replace")))) except ProgrammingError as e: print( colorize( "ERROR! The database is probably not up to date " "(should be ignored when happen during the upgrade)", RED)) print(colorize(e, RED)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE": raise except NoSuchTableError as e: print( colorize( "ERROR! The schema didn't seem to exists " "(should be ignored when happen during the deploy)", RED)) print(colorize(e, RED)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE": raise except OperationalError as e: print( colorize( "ERROR! The database didn't seem to exists " "(should be ignored when happen during the deploy)", RED)) print(colorize(e, RED)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE": raise return messages
def post(self, command, output_dir, vars_): """ Overrides the base template class to print "Welcome to c2cgeoportal!" after a successful scaffolding rendering. """ if os.name == 'posix': for file_ in ("docker-run", "docker-compose-run"): dest = os.path.join(output_dir, "CONST_create_template", file_) subprocess.check_call(["chmod", "+x", dest]) self.out(colorize("\nWelcome to c2cgeoportal!", GREEN)) return BaseTemplate.post(self, command, output_dir, vars_)
def print_step( self, step: int, error: bool = False, message: Optional[str] = None, prompt: str = "To continue, type:", ) -> None: with open(".UPGRADE_INSTRUCTIONS", "w", encoding="utf8") as instructions: print("") print(self.color_bar) if message is not None: print(colorize(message, Color.RED if error else Color.YELLOW)) instructions.write(f"{message}\n") if step >= 0: print(colorize(prompt, Color.GREEN)) instructions.write(f"{prompt}\n") cmd = ["./upgrade", os.environ["VERSION"]] if step != 0: cmd.append(f"{step}") print(colorize(" ".join(cmd), Color.GREEN)) instructions.write(f"{' '.join(cmd)}\n")
def __call__(self, filename, options, fileobj=None, lineno=0): print('Entering into %s extractor' % self.__class__) del fileobj, lineno try: # initialize DB connections in a way similar to c2cgeoportal_geoportal.lib.lingua_extractor settings = config.get_config() class R: settings = None class C: registry = R() config_ = C() config_.registry.settings = settings init_dbsessions(settings, config_) try: self._extract_messages() file = "geoportailv3_geoportal-legends.pot" if str(self.__class__).find('LuxembourgTooltipsExtractor') > 0: file = "geoportailv3_geoportal-tooltips.pot" for attribute in self._get_missing_keys(file): self._insert_attribute(attribute.msgid, (attribute.comment, "")) except ProgrammingError as e: print( colorize( "ERROR! The database is probably not up to date " "(should be ignored when happen during the upgrade)", RED, )) print(colorize(e, RED)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE": raise except NoSuchTableError as e: print( colorize( "ERROR! The schema didn't seem to exists " "(should be ignored when happen during the deploy)", RED, )) print(colorize(e, RED)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE": raise except OperationalError as e: print( colorize( "ERROR! The database didn't seem to exists " "(should be ignored when happen during the deploy)", RED, )) print(colorize(e, RED)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE": raise return self.messages
def post(self, command, output_dir, vars_): """ Overrides the base template class to print "Welcome to c2cgeoportal!" after a successful scaffolding rendering. """ if os.name == 'posix': for file_ in ("docker-run", "docker-compose-run"): dest = os.path.join(output_dir, "CONST_create_template", file_) subprocess.check_call(["chmod", "+x", dest]) self.out(colorize("\nWelcome to c2cgeoportal!", GREEN)) return BaseTemplate.post(self, command, output_dir, vars_)
def _import_layer_wms(self, layer, messages): server = layer.ogc_server url = server.url_wfs or server.url if url is None: return for wms_layer in layer.layer.split(","): self._import_layer_attributes(url, wms_layer, layer.item_type, layer.name, messages) if layer.geo_table is not None and layer.geo_table != "": exclude = [] if layer.exclude_properties is None else layer.exclude_properties.split(",") last_update_date = layer.get_metadatas("lastUpdateDateColumn") if len(last_update_date) == 1: exclude.append(last_update_date[0].value) last_update_user = layer.get_metadatas("lastUpdateUserColumn") if len(last_update_user) == 1: exclude.append(last_update_user[0].value) try: cls = get_class(layer.geo_table, exclude_properties=exclude) for column_property in class_mapper(cls).iterate_properties: if isinstance(column_property, ColumnProperty) and len(column_property.columns) == 1: column = column_property.columns[0] if not column.primary_key and not isinstance(column.type, Geometry): if column.foreign_keys: name = "type_" if column.name == "type_id" else \ column.name[0:column.name.rindex("_id")] else: name = column_property.key messages.append(Message( None, name, None, [], "", "", (".".join(["edit", layer.item_type, str(layer.id)]), layer.name) )) except NoSuchTableError: print(colorize( "ERROR! No such table '{}' for layer '{}'.".format(layer.geo_table, layer.name), RED )) print(colorize(traceback.format_exc(), RED)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE": raise
def step12(self, _): check_call(["git", "commit", "-m", "Upgrade to GeoMapFish {}".format( pkg_resources.get_distribution("c2cgeoportal_commons").version )]) print("") print(self.color_bar) print("") print(colorize("Congratulations your upgrade is a success.", GREEN)) print("") branch = check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).decode("utf-8").strip() print("Now all your files will be committed, you should do a git push:") print("git push {0!s} {1!s}.".format( self.options.git_remote, branch ))
def _enumerate_attributes_values(dbsessions, layers, layerinfos, fieldname): dbname = layerinfos.get("dbsession", "dbsession") try: dbsession = dbsessions.get(dbname) return layers.query_enumerate_attribute_values(dbsession, layerinfos, fieldname) except Exception as e: table = layerinfos.get("attributes").get(fieldname, {}).get("table") print(colorize( "ERROR! Unable to collect enumerate attributes for " "db: {}, table: {}, column: {} ({})".format(dbname, table, fieldname, e), RED )) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE": return [] else: raise
def post(self, command, output_dir, vars_): """ Overrides the base template class to print the next step. """ if os.name == 'posix': for file_ in ("docker-run", "docker-compose-run"): dest = os.path.join(output_dir, file_) subprocess.check_call(["chmod", "+x", dest]) self.out("\nContinue with:") self.out(colorize( "SRID={vars[srid]} EXTENT={vars[extent]} ./docker-run pcreate --scaffold c2cgeoportal_update " "--ignore-conflicting-name --package-name {vars[package]} ../{vars[project]}".format(vars=vars_), GREEN )) return BaseTemplate.post(self, command, output_dir, vars_)
def __call__(self, filename, options): messages = [] try: self.env = bootstrap(filename) try: from c2cgeoportal_commons.models import DBSession from c2cgeoportal_commons.models.main import Theme, LayerGroup, \ LayerWMS, LayerWMTS, FullTextSearch self._import(Theme, messages) self._import(LayerGroup, messages) self._import(LayerWMS, messages, self._import_layer_wms) self._import(LayerWMTS, messages, self._import_layer_wmts) for ln, in DBSession.query(FullTextSearch.layer_name).distinct().all(): if ln is not None and ln != "": messages.append(Message( None, ln, None, [], "", "", ("fts", ln.encode("ascii", errors="replace")) )) except ProgrammingError as e: print(colorize( "ERROR! The database is probably not up to date " "(should be ignored when happen during the upgrade)", RED )) print(colorize(e, RED)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE": raise except NoSuchTableError as e: print(colorize( "ERROR! The schema didn't seem to exists " "(should be ignored when happen during the deploy)", RED )) print(colorize(e, RED)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE": raise except OperationalError as e: print(colorize( "ERROR! The database didn't seem to exists " "(should be ignored when happen during the deploy)", RED )) print(colorize(e, RED)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE": raise return messages
def __call__(self, filename, options): init_region({"backend": "dogpile.cache.memory"}) int_filename = filename if re.match("^" + re.escape("./{}/templates".format(self.config["package"])), filename): try: empty_template = Template("") class Lookup(TemplateLookup): @staticmethod def get_template(uri): return empty_template class MyTemplate(MakoTemplate): def prepare(self, **options): options.update({"input_encoding": self.encoding}) lookup = Lookup(**options) if self.source: self.tpl = Template(self.source, lookup=lookup, **options) else: self.tpl = Template( uri=self.name, filename=self.filename, lookup=lookup, **options) try: processed = template( filename, { "request": _Request(self.config), "lang": "fr", "debug": False, "extra_params": {}, "permalink_themes": "", "fulltextsearch_groups": [], "wfs_types": [], "_": lambda x: x, }, template_adapter=MyTemplate ) int_filename = os.path.join(os.path.dirname(filename), "_" + os.path.basename(filename)) with open(int_filename, "wb") as file_open: file_open.write(processed.encode("utf-8")) except Exception: print(colorize( "ERROR! Occurred during the '{}' template generation".format(filename), RED )) print(colorize(traceback.format_exc(), RED)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE": # Continue with the original one int_filename = filename else: raise except Exception: print(traceback.format_exc()) message_str = subprocess.check_output([ "node", "tools/extract-messages.js", int_filename ]).decode("utf-8") if int_filename != filename: os.unlink(int_filename) try: messages = [] for contexts, message in loads(message_str): for context in contexts.split(", "): messages.append(Message( None, message, None, [], "", "", context.split(":") )) return messages except Exception: print(colorize("An error occurred", RED)) print(colorize(message_str, RED)) print("------") raise
def _layer_attributes(self, url, layer): errors = set() request = _Request() request.registry.settings = self.config # static schema will not be supported url = get_url2("Layer", url, request, errors) if len(errors) > 0: print("\n".join(errors)) return [] wms_getcap_url = add_url_params(url, { "SERVICE": "WMS", "VERSION": "1.1.1", "REQUEST": "GetCapabilities", }) hostname = urlsplit(url).hostname if url not in self.wmscap_cache: print("Get WMS GetCapabilities for URL: {}".format(url)) self.wmscap_cache[url] = None # forward request to target (without Host Header) http = httplib2.Http() h = {} if hostname == "localhost": # pragma: no cover h["Host"] = self.package["host"] try: resp, content = http.request(wms_getcap_url, method="GET", headers=h) try: self.wmscap_cache[url] = WebMapService(None, xml=content) except Exception as e: print(colorize( "ERROR! an error occurred while trying to " "parse the GetCapabilities document.", RED)) print(colorize(str(e), RED)) print("URL: {}\nxml:\n{}".format(wms_getcap_url, content)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE": raise except Exception as e: # pragma: no cover print(colorize(str(e), RED)) print(colorize( "ERROR! Unable to GetCapabilities from URL: {}".format(wms_getcap_url), RED, )) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE": raise wmscap = self.wmscap_cache[url] wfs_descrfeat_url = add_url_params(url, { "SERVICE": "WFS", "VERSION": "1.1.0", "REQUEST": "DescribeFeatureType", }) if url not in self.featuretype_cache: print("Get WFS DescribeFeatureType for URL: {}".format(wfs_descrfeat_url)) self.featuretype_cache[url] = None # forward request to target (without Host Header) http = httplib2.Http() h = {} if hostname == "localhost": # pragma: no cover h["Host"] = self.package["host"] try: resp, content = http.request(wfs_descrfeat_url, method="GET", headers=h) except Exception as e: # pragma: no cover print(colorize(str(e), RED)) print(colorize( "ERROR! Unable to DescribeFeatureType from URL: {}".format(wfs_descrfeat_url), RED, )) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE": return [] else: raise if resp.status < 200 or resp.status >= 300: # pragma: no cover print(colorize( "ERROR! DescribeFeatureType from URL {} return the error: {1:d} {}".format( wfs_descrfeat_url, resp.status, resp.reason ), RED, )) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE": return [] else: raise Exception("Aborted") try: describe = parseString(content) featurestype = {} self.featuretype_cache[url] = featurestype for type_element in describe.getElementsByTagNameNS( "http://www.w3.org/2001/XMLSchema", "complexType" ): featurestype[type_element.getAttribute("name")] = type_element except ExpatError as e: print(colorize( "ERROR! an error occurred while trying to " "parse the DescribeFeatureType document.", RED )) print(colorize(str(e), RED)) print("URL: {}\nxml:\n{}".format(wfs_descrfeat_url, content)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE": return [] else: raise except AttributeError: print(colorize( "ERROR! an error occurred while trying to " "read the Mapfile and recover the themes.", RED )) print("URL: {}\nxml:\n{}".format(wfs_descrfeat_url, content)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE": return [] else: raise else: featurestype = self.featuretype_cache[url] if featurestype is None: return [] layers = [layer] if wmscap is not None and layer in list(wmscap.contents): layer_obj = wmscap[layer] if len(layer_obj.layers) > 0: layers = [l.name for l in layer_obj.layers] attributes = [] for sub_layer in layers: # Should probably be adapted for other king of servers type_element = featurestype.get("{}Type".format(sub_layer)) if type_element is not None: for element in type_element.getElementsByTagNameNS( "http://www.w3.org/2001/XMLSchema", "element" ): if not element.getAttribute("type").startswith("gml:"): attributes.append(element.getAttribute("name")) return attributes, layers