def files_to_remove(self, pre=False):
error = False
for element in self.get_upgrade("files_to_remove"):
file_ = element["file"]
if os.path.exists(file_):
managed = False
for pattern in self.project["managed_files"]:
if re.match(pattern + '$', file_):
print(colorize(
"The file '{}' is no more use but not delete "
"because he is in the managed_files as '{}'.".format(file_, pattern),
RED
))
error = True
managed = True
if not managed and not pre:
print(colorize("The file '{}' is removed.".format(file_), GREEN))
if "version" in element and "from" in element:
print("Was used in version {}, to be removed from version {}.".format(
element["from"], element["version"]
))
if os.path.isdir(file_):
shutil.rmtree(file_)
else:
os.remove(file_)
return error
def test_checkers(self) -> Tuple[bool, Optional[str]]:
run_curl = "Run `curl --insecure {} '{}'` for more information.".format(
" ".join([
"--header {}={}".format(*i)
for i in self.project.get("checker_headers", {}).items()
]),
self.project["checker_url"],
)
try:
requests.packages.urllib3.disable_warnings()
resp = requests.get(self.project["checker_url"],
headers=self.project.get("checker_headers"),
verify=False)
except requests.exceptions.ConnectionError as exception:
return False, "\n".join(
["Connection error: {}".format(exception), run_curl])
except ConnectionRefusedError as exception:
return False, "\n".join(
["Connection refused: {}".format(exception), run_curl])
if resp.status_code < 200 or resp.status_code >= 300:
print(colorize("=============", Color.RED))
print(colorize("Checker error", Color.RED))
for name, value in resp.json()["failures"].items():
print(
colorize("Test '{}' failed with result:".format(name),
Color.YELLOW))
del value["level"]
del value["timing"]
print(yaml.dump(value) if value != {} else "No result")
return False, "\n".join(["Checker error:", run_curl])
return True, None
def files_to_remove(self, element, prefix=""):
for path in element["paths"]:
file_ = os.path.join(
prefix, path.format(package=self.project["project_package"]))
if os.path.exists(file_):
managed = False
for pattern in self.project["managed_files"]:
if re.match(pattern + '$', file_):
print(
colorize(
"The file '{}' is no longer used, but not deleted "
"because it is in the managed_files as '{}'.".
format(file_, pattern), RED))
managed = True
if not managed:
print(
colorize("The file '{}' is removed.".format(file_),
GREEN))
if "version" in element and "from" in element:
print(
"Was used in version {}, to be removed from version {}."
.format(element["from"], element["version"]))
if os.path.isdir(file_):
shutil.rmtree(file_)
else:
os.remove(file_)
def print_step(self, step, error=False, message=None, prompt="To continue type:"):
print("")
print(self.color_bar)
if message is not None:
print(colorize(message, RED if error else YELLOW))
if step >= 0:
print(colorize(prompt, GREEN))
args = " --makefile=" + self.options.new_makefile if self.options.makefile != "Makefile" else ""
print(colorize("./docker-run make{} upgrade{}", GREEN).format(
args, step if step != 0 else "",
))
def print_step(self, step, error=False, message=None, prompt="To continue, type:"):
print("")
print(self.color_bar)
if message is not None:
print(colorize(message, RED if error else YELLOW))
if step >= 0:
print(colorize(prompt, GREEN))
cmd = [
"./upgrade", os.environ['VERSION']
]
if step != 0:
cmd.append("{}".format(step))
print(colorize(" ".join(cmd), GREEN))
def _import_layer_wms(self, layer: "main.Layer",
messages: List[str]) -> None:
server = layer.ogc_server
url = server.url_wfs or server.url
if url is None:
return
if layer.ogc_server.wfs_support:
for wms_layer in layer.layer.split(","):
self._import_layer_attributes(url, wms_layer, layer.item_type,
layer.name, messages)
if layer.geo_table is not None and layer.geo_table != "":
try:
cls = get_layer_class(layer, with_last_update_columns=True)
for column_property in class_mapper(cls).iterate_properties:
if isinstance(column_property, ColumnProperty) and len(
column_property.columns) == 1:
column = column_property.columns[0]
if not column.primary_key and not isinstance(
column.type, Geometry):
if column.foreign_keys:
if column.name == "type_id":
name = "type_"
elif column.name.endswith("_id"):
name = column.name[:-3]
else:
name = column.name + "_"
else:
name = column_property.key
messages.append(
Message(
None,
name,
None,
[],
"",
"",
(".".join([
"edit", layer.item_type,
str(layer.id)
]), layer.name),
))
except NoSuchTableError:
print(
colorize(
"ERROR! No such table '{}' for layer '{}'.".format(
layer.geo_table, layer.name),
Color.RED,
))
print(colorize(traceback.format_exc(), Color.RED))
if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
raise
def files_to_remove(self,
element: Dict[str, Any],
prefix: str = "",
force: bool = False) -> bool:
task_to_do = False
for path in element["paths"]:
file_ = os.path.join(
prefix, path.format(package=self.project["project_package"]))
if os.path.exists(file_):
managed = False
if not force:
for files in self.project["managed_files"]:
if isinstance(files, str):
pattern = files
no_touch = False
else:
pattern = files["pattern"]
no_touch = files.get("no_touch", False)
if re.match(pattern + "$", file_):
if no_touch:
managed = True
else:
print(
colorize(
f"The file '{file_}' has been removed but he is in the "
f"`managed_files` as '{pattern}'.",
Color.RED,
))
task_to_do = True
for pattern in self.project.get("unmanaged_files", []):
if re.match(pattern + "$", file_):
print(
colorize(
f"The file '{file_}' has been removed but he is in the "
f"`unmanaged_files` as '{pattern}'.",
Color.YELLOW,
))
task_to_do = True
if not managed:
print(f"The file '{file_}' is removed.")
if "version" in element and "from" in element:
print(
f"Was used in version {element['from']}, to be removed from version "
f"{element['version']}.")
if os.path.isdir(file_):
shutil.rmtree(file_)
else:
os.remove(file_)
return task_to_do
def files_to_move(self, element, prefix="", force=False):
task_to_do = False
src = os.path.join(
prefix,
element["from"].format(package=self.project["project_package"]))
dst = os.path.join(
prefix,
element["to"].format(package=self.project["project_package"]))
if os.path.exists(src):
managed = False
type_ = "directory" if os.path.isdir(src) else "file"
if not force:
for pattern in self.project["managed_files"]:
if re.match(pattern + '$', src):
print(
colorize(
"The {} '{}' is present in the managed_files as '{}', but it will move."
.format(type_, src, pattern), RED))
task_to_do = True
managed = True
break
if re.match(pattern + '$', dst):
print(
colorize(
"The {} '{}' is present in the managed_files as '{}', but it will move."
.format(type_, dst, pattern), RED))
task_to_do = True
managed = True
break
if not managed and os.path.exists(dst):
print(
colorize(
"The destination '{}' already exists, ignoring.".
format(dst), YELLOW))
if not managed:
print(
colorize(
"Move the {} '{}' to '{}'.".format(type_, src, dst),
GREEN))
if "version" in element:
print("Needed from version {}.".format(element["version"]))
if os.path.dirname(dst) != "":
os.makedirs(os.path.dirname(dst), exist_ok=True)
try:
check_call(["git", "mv", src, dst])
except Exception as e:
print("[Warning] Git move error: {}.".format(e))
os.rename(src, dst)
return task_to_do
def _import_layer_wmts(self, layer, messages):
from c2cgeoportal_commons.models import DBSession
from c2cgeoportal_commons.models.main import OGCServer
layers = [d.value for d in layer.metadatas if d.name == "queryLayers"]
if len(layers) == 0:
layers = [d.value for d in layer.metadatas if d.name == "wmsLayer"]
server = [d.value for d in layer.metadatas if d.name == "ogcServer"]
if len(server) >= 1 and len(layers) >= 1:
for wms_layer in layers:
try:
db_server = DBSession.query(OGCServer).filter(OGCServer.name == server[0]).one()
self._import_layer_attributes(
db_server.url_wfs or db_server.url, wms_layer,
layer.item_type, layer.name, messages
)
except NoResultFound:
print(colorize(
"ERROR! the OGC server '{}' from the WMTS layer '{}' does not exist.".format(
server[0], layer.name
),
RED
))
if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
raise
def files_to_get(self, pre=False):
error = False
for root, _, files in os.walk("CONST_create_template"):
root = root[len("CONST_create_template/"):]
for file_ in files:
destination = os.path.join(root, file_)
managed = False
for pattern in self.project["managed_files"] + self.get_upgrade("default_project_file"):
if re.match(pattern + '$', destination):
managed = True
for unpattern in self.project.get("unmanaged_files", []):
if re.match(unpattern + '$', destination):
managed = False
break
break
source = os.path.join("CONST_create_template", destination)
if not managed and (not os.path.exists(destination) or not filecmp.cmp(source, destination)):
if not pre:
print(colorize(
"Get the file '{}' from the create template.".format(destination), GREEN
))
if os.path.dirname(destination) != "":
os.makedirs(os.path.dirname(destination), exist_ok=True)
shutil.copyfile(source, destination)
shutil.copymode(source, destination)
return error
def get_project():
if not os.path.isfile("project.yaml"):
print(colorize("Unable to find the required 'project.yaml' file.", RED))
exit(1)
with open("project.yaml", "r") as f:
return yaml.safe_load(f)
def _import_layer_wmts(self, layer: "main.Layer",
messages: List[str]) -> None:
from c2cgeoportal_commons.models import DBSession # pylint: disable=import-outside-toplevel
from c2cgeoportal_commons.models.main import OGCServer # pylint: disable=import-outside-toplevel
layers = [d.value for d in layer.metadatas if d.name == "queryLayers"]
if not layers:
layers = [d.value for d in layer.metadatas if d.name == "wmsLayer"]
server = [d.value for d in layer.metadatas if d.name == "ogcServer"]
if server and layers:
layers = [layer for ls in layers for layer in ls.split(",")]
for wms_layer in layers:
try:
db_server = DBSession.query(OGCServer).filter(
OGCServer.name == server[0]).one()
if db_server.wfs_support:
self._import_layer_attributes(
db_server.url_wfs or db_server.url,
wms_layer,
layer.item_type,
layer.name,
messages,
)
except NoResultFound:
print(
colorize(
"ERROR! the OGC server '{}' from the WMTS layer '{}' does not exist."
.format(server[0], layer.name),
Color.RED,
))
if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
raise
def _enumerate_attributes_values(dbsessions: Dict[str, Session],
layerinfos: Dict[str, Any],
fieldname: str) -> List[Message]:
dbname = layerinfos.get("dbsession", "dbsession")
translate = cast(Dict[str, Any], layerinfos["attributes"]).get(
fieldname, {}).get("translate", True)
if not translate:
return []
try:
dbsession = dbsessions.get(dbname)
return Layers.query_enumerate_attribute_values(
dbsession, layerinfos, fieldname)
except Exception as e:
table = cast(Dict[str, Any],
layerinfos["attributes"]).get(fieldname,
{}).get("table")
print(
colorize(
"ERROR! Unable to collect enumerate attributes for "
"db: {}, table: {}, column: {} ({})".format(
dbname, table, fieldname, e),
Color.RED,
))
if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
return []
raise
def files_to_get(self, step, pre=False):
error = False
for root, _, files in os.walk("CONST_create_template"):
root = root[len("CONST_create_template/"):]
for file_ in files:
destination = os.path.join(root, file_)
managed = self.is_managed(destination)
source = os.path.join("CONST_create_template", destination)
if not managed and (not os.path.exists(destination) or not filecmp.cmp(source, destination)):
print(colorize(
"Get the file '{}' from the create template.".format(destination), GREEN
))
if not pre:
if os.path.dirname(destination) != "":
os.makedirs(os.path.dirname(destination), exist_ok=True)
try:
shutil.copyfile(source, destination)
shutil.copymode(source, destination)
except PermissionError as e:
self.print_step(
step, error=True, message=(
"All your project files should be own by your user, "
"current error:\n" + str(e)
),
prompt="Fix it and run again the upgrade:")
exit(1)
elif os.path.exists(destination) and not filecmp.cmp(source, destination):
print("The file '{}' is managed by the project".format(destination))
else:
print("The file '{}' does not change".format(destination))
return error
def get_upgrade(section):
if not os.path.isfile(".upgrade.yaml"):
print(colorize("Unable to find the required '.upgrade.yaml' file.", RED))
exit(1)
with open(".upgrade.yaml", "r") as f:
return yaml.safe_load(f)[section]
def _enumerate_attributes_values(dbsessions: Dict[str, Session],
layerinfos: Dict[str, Any],
fieldname: str) -> Set[Tuple[str, ...]]:
dbname = layerinfos.get("dbsession", "dbsession")
translate = cast(Dict[str, Any], layerinfos["attributes"]).get(
fieldname, {}).get("translate", True)
if not translate:
return set()
try:
dbsession = dbsessions.get(dbname)
return Layers.query_enumerate_attribute_values(
dbsession, layerinfos, fieldname)
except Exception as e:
table = cast(Dict[str, Any],
layerinfos["attributes"]).get(fieldname,
{}).get("table")
print(
colorize(
"ERROR! Unable to collect enumerate attributes for "
f"db: {dbname}, table: {table}, column: {fieldname} ({e!s})",
Color.RED,
))
if _get_config_str("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
return set()
raise
def _import_layer_wmts(self, layer, messages):
from c2cgeoportal_commons.models import DBSession
from c2cgeoportal_commons.models.main import OGCServer
layers = [d.value for d in layer.metadatas if d.name == "queryLayers"]
if len(layers) == 0:
layers = [d.value for d in layer.metadatas if d.name == "wmsLayer"]
server = [d.value for d in layer.metadatas if d.name == "ogcServer"]
if len(server) >= 1 and len(layers) >= 1:
for wms_layer in layers:
try:
db_server = DBSession.query(OGCServer).filter(OGCServer.name == server[0]).one()
self._import_layer_attributes(
db_server.url_wfs or db_server.url, wms_layer,
layer.item_type, layer.name, messages
)
except NoResultFound:
print(colorize(
"ERROR! the OGC server '{}' from the WMTS layer '{}' does not exist.".format(
server[0], layer.name
),
RED
))
if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
raise
def print_step(self,
step,
error=False,
message=None,
prompt="To continue, type:"):
print("")
print(self.color_bar)
if message is not None:
print(colorize(message, RED if error else YELLOW))
if step >= 0:
print(colorize(prompt, GREEN))
if self.options.use_makefile:
args = " --makefile={}".format(self.options.makefile) \
if self.options.makefile != "Makefile" else ""
print(
colorize(
"./docker-run --home make{} upgrade{}".format(
args,
step if step != 0 else "",
), GREEN))
else:
cmd = [
"./docker-run",
"--home",
"--image=camptocamp/geomapfish-build",
"--version=" + pkg_resources.get_distribution(
"c2cgeoportal_commons").version,
"c2cupgrade",
]
if self.options.windows:
cmd.append("--windows")
if self.options.nondocker:
cmd.append("--nondocker")
if self.options.force_docker:
cmd.append("--force-docker")
if self.options.git_remote != "origin":
cmd.append("--git-remote={}".format(
self.options.git_remote))
if self.options.makefile != "Makefile":
cmd.append("--makefile={}".format(self.options.makefile))
if self.options.new_makefile != self.options.makefile:
cmd.append("--new-makefile={}".format(
self.options.new_makefile))
if step != 0:
cmd.append("--step={}".format(step))
print(colorize(" ".join(cmd), GREEN))
def files_to_remove(self, element, prefix="", force=False):
task_to_do = False
for path in element["paths"]:
file_ = os.path.join(
prefix, path.format(package=self.project["project_package"]))
if os.path.exists(file_):
managed = False
if not force:
for files in self.project["managed_files"]:
if isinstance(files, str):
pattern = files
no_touch = False
else:
pattern = files["pattern"]
no_touch = files.get("no_touch", False)
if re.match(pattern + "$", file_):
if no_touch:
managed = True
else:
# fmt: off
print(
colorize(
"The file '{}' has been removed but he is in the `managed_files` as '{}'."
.format(file_, pattern), RED))
# fmt: on
task_to_do = True
for pattern in self.project.get("unmanaged_files", []):
if re.match(pattern + "$", file_):
# fmt: off
print(
colorize(
"The file '{}' has been removed but he is in the `unmanaged_files` as '{}'."
.format(file_, pattern), YELLOW))
# fmt: on
task_to_do = True
if not managed:
print("The file '{}' is removed.".format(file_))
if "version" in element and "from" in element:
print(
"Was used in version {}, to be removed from version {}."
.format(element["from"], element["version"]))
if os.path.isdir(file_):
shutil.rmtree(file_)
else:
os.remove(file_)
return task_to_do
def files_to_move(self, pre=False, prefix="", force=False):
error = False
for element in self.get_upgrade("files_to_move"):
src = os.path.join(prefix, element["from"])
dst = os.path.join(prefix, element["to"])
if os.path.exists(src):
managed = False
type_ = "directory" if os.path.isdir(src) else "file"
if not force:
for pattern in self.project["managed_files"]:
if re.match(pattern + '$', src):
print(colorize(
"The {} '{}' is present in the managed_files as '{}', but it will move."
.format(
type_, src, pattern
),
RED
))
error = True
managed = True
break
if re.match(pattern + '$', dst):
print(colorize(
"The {} '{}' is present in the managed_files as '{}', but it will move."
.format(
type_, dst, pattern
),
RED
))
error = True
managed = True
break
if not managed and os.path.exists(dst):
print(colorize(
"The destination '{}' already exists, ignoring.".format(dst),
YELLOW
))
if not managed and not pre:
print(colorize("Move the {} '{}' to '{}'.".format(type_, src, dst), GREEN))
if "version" in element:
print("Needed from version {}.".format(element["version"]))
if os.path.dirname(dst) != "":
os.makedirs(os.path.dirname(dst), exist_ok=True)
os.rename(src, dst)
return error
def get_project() -> Dict[str, Any]:
if not os.path.isfile("project.yaml"):
print(
colorize("Unable to find the required 'project.yaml' file.",
Color.RED))
sys.exit(1)
with open("project.yaml", encoding="utf8") as project_file:
return cast(Dict[str, Any], yaml.safe_load(project_file))
def _import_layer_wms(self, layer, messages):
server = layer.ogc_server
url = server.url_wfs or server.url
if url is None:
return
for wms_layer in layer.layer.split(","):
self._import_layer_attributes(url, wms_layer, layer.item_type,
layer.name, messages)
if layer.geo_table is not None and layer.geo_table != "":
exclude = [] if layer.exclude_properties is None else layer.exclude_properties.split(
",")
last_update_date = layer.get_metadatas("lastUpdateDateColumn")
if len(last_update_date) == 1:
exclude.append(last_update_date[0].value)
last_update_user = layer.get_metadatas("lastUpdateUserColumn")
if len(last_update_user) == 1:
exclude.append(last_update_user[0].value)
try:
cls = get_class(layer.geo_table, exclude_properties=exclude)
for column_property in class_mapper(cls).iterate_properties:
if isinstance(column_property, ColumnProperty) and len(
column_property.columns) == 1:
column = column_property.columns[0]
if not column.primary_key and not isinstance(
column.type, Geometry):
if column.foreign_keys:
name = "type_" if column.name == "type_id" else \
column.name[0:column.name.rindex("_id")]
else:
name = column_property.key
messages.append(
Message(None, name, None, [], "", "",
(".".join([
"edit", layer.item_type,
str(layer.id)
]), layer.name)))
except NoSuchTableError:
print(
colorize(
"ERROR! No such table '{}' for layer '{}'.".format(
layer.geo_table, layer.name), RED))
print(colorize(traceback.format_exc(), RED))
if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
raise
def print_step(self,
step: int,
error: bool = False,
message: str = None,
prompt: str = "To continue, type:") -> None:
with open(".UPGRADE_INSTRUCTIONS", "w") as instructions:
print("")
print(self.color_bar)
if message is not None:
print(colorize(message, Color.RED if error else Color.YELLOW))
instructions.write("{}\n".format(message))
if step >= 0:
print(colorize(prompt, Color.GREEN))
instructions.write("{}\n".format(prompt))
cmd = ["./upgrade", os.environ["VERSION"]]
if step != 0:
cmd.append("{}".format(step))
print(colorize(" ".join(cmd), Color.GREEN))
instructions.write("{}\n".format(" ".join(cmd)))
def get_upgrade(section: str) -> Union[List[Any], Dict[str, Any]]:
if not os.path.isfile(".upgrade.yaml"):
print(
colorize("Unable to find the required '.upgrade.yaml' file.",
Color.RED))
sys.exit(1)
with open(".upgrade.yaml", encoding="utf8") as project_file:
return cast(Union[List[Any], Dict[str, Any]],
yaml.safe_load(project_file)[section])
def files_to_move(self, pre=False):
error = False
for element in self.get_upgrade("files_to_move"):
src = element["from"]
dst = element["to"]
if os.path.exists(src):
type_ = "directory" if os.path.isdir(src) else "file"
managed = False
for pattern in self.project["managed_files"]:
if re.match(pattern + '$', src):
print(colorize(
"The {} '{}' is present in the managed_files as '{}' but he will move.".format(
type_, src, pattern),
RED
))
error = True
managed = True
break
if re.match(pattern + '$', dst):
print(colorize(
"The {} '{}' is present in the managed_files as '{}' but he will move.".format(
type_, dst, pattern),
RED
))
error = True
managed = True
break
if not managed and os.path.exists(dst):
print(colorize(
"The destination '{}' already exists.".format(dst),
RED
))
error = True
if not pre:
raise InteruptedException("The destination '{}' already exists.".format(dst))
if not managed and not pre:
print(colorize("Move the {} '{}' to '{}'.".format(type_, src, dst), GREEN))
if "version" in element:
print("Needed from version {}.".format(element["version"]))
if os.path.dirname(dst) != "":
os.makedirs(os.path.dirname(dst), exist_ok=True)
os.rename(src, dst)
return error
def test_checkers(self) -> Tuple[bool, Optional[str]]:
headers = " ".join([
f"--header {i[0]}={i[1]}"
for i in self.project.get("checker_headers", {}).items()
])
run_curl = f"Run `curl --insecure {headers} '{self.project['checker_url']}'` for more information."
try:
requests.packages.urllib3.disable_warnings() # type: ignore
resp = requests.get(
self.project["checker_url"],
headers=self.project.get("checker_headers"),
verify=False, # nosec
)
except requests.exceptions.ConnectionError as exception:
return False, "\n".join(
[f"Connection error: {exception}", run_curl])
except ConnectionRefusedError as exception:
return False, "\n".join(
[f"Connection refused: {exception}", run_curl])
if resp.status_code < 200 or resp.status_code >= 300:
print(colorize("=============", Color.RED))
print(colorize("Checker error", Color.RED))
try:
for name, value in resp.json()["failures"].items():
print(
colorize(f"Test '{name}' failed with result:",
Color.YELLOW))
del value["level"]
del value["timing"]
print(yaml.dump(value) if value != {} else "No result")
except JSONDecodeError:
print(
colorize(
f"Response is not a JSON '{resp.text}', {resp.reason} {resp.status_code}",
Color.RED,
))
return False, "\n".join(["Checker error:", run_curl])
return True, None
def __call__(self, filename, options):
messages = []
try:
self.env = bootstrap(filename, options=os.environ)
try:
from c2cgeoportal_commons.models import DBSession
from c2cgeoportal_commons.models.main import Theme, LayerGroup, \
LayerWMS, LayerWMTS, FullTextSearch
self._import(Theme, messages)
self._import(LayerGroup, messages)
self._import(LayerWMS, messages, self._import_layer_wms)
self._import(LayerWMTS, messages, self._import_layer_wmts)
for ln, in DBSession.query(
FullTextSearch.layer_name).distinct().all():
if ln is not None and ln != "":
messages.append(
Message(
None, ln, None, [], "", "",
("fts", ln.encode("ascii", errors="replace"))))
for ln, in DBSession.query(
FullTextSearch.actions).distinct().all():
if ln is not None and ln != "":
action = json.loads(ln)
messages.append(
Message(None, action['data'], None, [], "", "",
("fts", action['data'].encode(
"ascii", errors="replace"))))
except ProgrammingError as e:
print(
colorize(
"ERROR! The database is probably not up to date "
"(should be ignored when happen during the upgrade)",
RED))
print(colorize(e, RED))
if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
raise
except NoSuchTableError as e:
print(
colorize(
"ERROR! The schema didn't seem to exists "
"(should be ignored when happen during the deploy)", RED))
print(colorize(e, RED))
if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
raise
except OperationalError as e:
print(
colorize(
"ERROR! The database didn't seem to exists "
"(should be ignored when happen during the deploy)", RED))
print(colorize(e, RED))
if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
raise
return messages
def post(self, command, output_dir, vars_):
"""
Overrides the base template class to print "Welcome to c2cgeoportal!"
after a successful scaffolding rendering.
"""
if os.name == 'posix':
for file_ in ("docker-run", "docker-compose-run"):
dest = os.path.join(output_dir, "CONST_create_template", file_)
subprocess.check_call(["chmod", "+x", dest])
self.out(colorize("\nWelcome to c2cgeoportal!", GREEN))
return BaseTemplate.post(self, command, output_dir, vars_)
def print_step(
self,
step: int,
error: bool = False,
message: Optional[str] = None,
prompt: str = "To continue, type:",
) -> None:
with open(".UPGRADE_INSTRUCTIONS", "w",
encoding="utf8") as instructions:
print("")
print(self.color_bar)
if message is not None:
print(colorize(message, Color.RED if error else Color.YELLOW))
instructions.write(f"{message}\n")
if step >= 0:
print(colorize(prompt, Color.GREEN))
instructions.write(f"{prompt}\n")
cmd = ["./upgrade", os.environ["VERSION"]]
if step != 0:
cmd.append(f"{step}")
print(colorize(" ".join(cmd), Color.GREEN))
instructions.write(f"{' '.join(cmd)}\n")
def __call__(self, filename, options, fileobj=None, lineno=0):
print('Entering into %s extractor' % self.__class__)
del fileobj, lineno
try:
# initialize DB connections in a way similar to c2cgeoportal_geoportal.lib.lingua_extractor
settings = config.get_config()
class R:
settings = None
class C:
registry = R()
config_ = C()
config_.registry.settings = settings
init_dbsessions(settings, config_)
try:
self._extract_messages()
file = "geoportailv3_geoportal-legends.pot"
if str(self.__class__).find('LuxembourgTooltipsExtractor') > 0:
file = "geoportailv3_geoportal-tooltips.pot"
for attribute in self._get_missing_keys(file):
self._insert_attribute(attribute.msgid,
(attribute.comment, ""))
except ProgrammingError as e:
print(
colorize(
"ERROR! The database is probably not up to date "
"(should be ignored when happen during the upgrade)",
RED,
))
print(colorize(e, RED))
if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
raise
except NoSuchTableError as e:
print(
colorize(
"ERROR! The schema didn't seem to exists "
"(should be ignored when happen during the deploy)",
RED,
))
print(colorize(e, RED))
if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
raise
except OperationalError as e:
print(
colorize(
"ERROR! The database didn't seem to exists "
"(should be ignored when happen during the deploy)",
RED,
))
print(colorize(e, RED))
if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
raise
return self.messages
def post(self, command, output_dir, vars_):
"""
Overrides the base template class to print "Welcome to c2cgeoportal!"
after a successful scaffolding rendering.
"""
if os.name == 'posix':
for file_ in ("docker-run", "docker-compose-run"):
dest = os.path.join(output_dir, "CONST_create_template", file_)
subprocess.check_call(["chmod", "+x", dest])
self.out(colorize("\nWelcome to c2cgeoportal!", GREEN))
return BaseTemplate.post(self, command, output_dir, vars_)
def _import_layer_wms(self, layer, messages):
server = layer.ogc_server
url = server.url_wfs or server.url
if url is None:
return
for wms_layer in layer.layer.split(","):
self._import_layer_attributes(url, wms_layer, layer.item_type, layer.name, messages)
if layer.geo_table is not None and layer.geo_table != "":
exclude = [] if layer.exclude_properties is None else layer.exclude_properties.split(",")
last_update_date = layer.get_metadatas("lastUpdateDateColumn")
if len(last_update_date) == 1:
exclude.append(last_update_date[0].value)
last_update_user = layer.get_metadatas("lastUpdateUserColumn")
if len(last_update_user) == 1:
exclude.append(last_update_user[0].value)
try:
cls = get_class(layer.geo_table, exclude_properties=exclude)
for column_property in class_mapper(cls).iterate_properties:
if isinstance(column_property, ColumnProperty) and len(column_property.columns) == 1:
column = column_property.columns[0]
if not column.primary_key and not isinstance(column.type, Geometry):
if column.foreign_keys:
name = "type_" if column.name == "type_id" else \
column.name[0:column.name.rindex("_id")]
else:
name = column_property.key
messages.append(Message(
None, name, None, [], "", "",
(".".join(["edit", layer.item_type, str(layer.id)]), layer.name)
))
except NoSuchTableError:
print(colorize(
"ERROR! No such table '{}' for layer '{}'.".format(layer.geo_table, layer.name),
RED
))
print(colorize(traceback.format_exc(), RED))
if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
raise
def step12(self, _):
check_call(["git", "commit", "-m", "Upgrade to GeoMapFish {}".format(
pkg_resources.get_distribution("c2cgeoportal_commons").version
)])
print("")
print(self.color_bar)
print("")
print(colorize("Congratulations your upgrade is a success.", GREEN))
print("")
branch = check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).decode("utf-8").strip()
print("Now all your files will be committed, you should do a git push:")
print("git push {0!s} {1!s}.".format(
self.options.git_remote, branch
))
def _enumerate_attributes_values(dbsessions, layers, layerinfos, fieldname):
dbname = layerinfos.get("dbsession", "dbsession")
try:
dbsession = dbsessions.get(dbname)
return layers.query_enumerate_attribute_values(dbsession, layerinfos, fieldname)
except Exception as e:
table = layerinfos.get("attributes").get(fieldname, {}).get("table")
print(colorize(
"ERROR! Unable to collect enumerate attributes for "
"db: {}, table: {}, column: {} ({})".format(dbname, table, fieldname, e),
RED
))
if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
return []
else:
raise
def post(self, command, output_dir, vars_):
"""
Overrides the base template class to print the next step.
"""
if os.name == 'posix':
for file_ in ("docker-run", "docker-compose-run"):
dest = os.path.join(output_dir, file_)
subprocess.check_call(["chmod", "+x", dest])
self.out("\nContinue with:")
self.out(colorize(
"SRID={vars[srid]} EXTENT={vars[extent]} ./docker-run pcreate --scaffold c2cgeoportal_update "
"--ignore-conflicting-name --package-name {vars[package]} ../{vars[project]}".format(vars=vars_),
GREEN
))
return BaseTemplate.post(self, command, output_dir, vars_)
def __call__(self, filename, options):
messages = []
try:
self.env = bootstrap(filename)
try:
from c2cgeoportal_commons.models import DBSession
from c2cgeoportal_commons.models.main import Theme, LayerGroup, \
LayerWMS, LayerWMTS, FullTextSearch
self._import(Theme, messages)
self._import(LayerGroup, messages)
self._import(LayerWMS, messages, self._import_layer_wms)
self._import(LayerWMTS, messages, self._import_layer_wmts)
for ln, in DBSession.query(FullTextSearch.layer_name).distinct().all():
if ln is not None and ln != "":
messages.append(Message(
None, ln, None, [], "", "",
("fts", ln.encode("ascii", errors="replace"))
))
except ProgrammingError as e:
print(colorize(
"ERROR! The database is probably not up to date "
"(should be ignored when happen during the upgrade)",
RED
))
print(colorize(e, RED))
if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
raise
except NoSuchTableError as e:
print(colorize(
"ERROR! The schema didn't seem to exists "
"(should be ignored when happen during the deploy)",
RED
))
print(colorize(e, RED))
if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
raise
except OperationalError as e:
print(colorize(
"ERROR! The database didn't seem to exists "
"(should be ignored when happen during the deploy)",
RED
))
print(colorize(e, RED))
if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
raise
return messages
def __call__(self, filename, options):
init_region({"backend": "dogpile.cache.memory"})
int_filename = filename
if re.match("^" + re.escape("./{}/templates".format(self.config["package"])), filename):
try:
empty_template = Template("")
class Lookup(TemplateLookup):
@staticmethod
def get_template(uri):
return empty_template
class MyTemplate(MakoTemplate):
def prepare(self, **options):
options.update({"input_encoding": self.encoding})
lookup = Lookup(**options)
if self.source:
self.tpl = Template(self.source, lookup=lookup, **options)
else:
self.tpl = Template(
uri=self.name,
filename=self.filename,
lookup=lookup, **options)
try:
processed = template(
filename,
{
"request": _Request(self.config),
"lang": "fr",
"debug": False,
"extra_params": {},
"permalink_themes": "",
"fulltextsearch_groups": [],
"wfs_types": [],
"_": lambda x: x,
},
template_adapter=MyTemplate
)
int_filename = os.path.join(os.path.dirname(filename), "_" + os.path.basename(filename))
with open(int_filename, "wb") as file_open:
file_open.write(processed.encode("utf-8"))
except Exception:
print(colorize(
"ERROR! Occurred during the '{}' template generation".format(filename),
RED
))
print(colorize(traceback.format_exc(), RED))
if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
# Continue with the original one
int_filename = filename
else:
raise
except Exception:
print(traceback.format_exc())
message_str = subprocess.check_output([
"node", "tools/extract-messages.js", int_filename
]).decode("utf-8")
if int_filename != filename:
os.unlink(int_filename)
try:
messages = []
for contexts, message in loads(message_str):
for context in contexts.split(", "):
messages.append(Message(
None, message, None, [], "", "", context.split(":")
))
return messages
except Exception:
print(colorize("An error occurred", RED))
print(colorize(message_str, RED))
print("------")
raise
def _layer_attributes(self, url, layer):
errors = set()
request = _Request()
request.registry.settings = self.config
# static schema will not be supported
url = get_url2("Layer", url, request, errors)
if len(errors) > 0:
print("\n".join(errors))
return []
wms_getcap_url = add_url_params(url, {
"SERVICE": "WMS",
"VERSION": "1.1.1",
"REQUEST": "GetCapabilities",
})
hostname = urlsplit(url).hostname
if url not in self.wmscap_cache:
print("Get WMS GetCapabilities for URL: {}".format(url))
self.wmscap_cache[url] = None
# forward request to target (without Host Header)
http = httplib2.Http()
h = {}
if hostname == "localhost": # pragma: no cover
h["Host"] = self.package["host"]
try:
resp, content = http.request(wms_getcap_url, method="GET", headers=h)
try:
self.wmscap_cache[url] = WebMapService(None, xml=content)
except Exception as e:
print(colorize(
"ERROR! an error occurred while trying to "
"parse the GetCapabilities document.",
RED))
print(colorize(str(e), RED))
print("URL: {}\nxml:\n{}".format(wms_getcap_url, content))
if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
raise
except Exception as e: # pragma: no cover
print(colorize(str(e), RED))
print(colorize(
"ERROR! Unable to GetCapabilities from URL: {}".format(wms_getcap_url),
RED,
))
if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
raise
wmscap = self.wmscap_cache[url]
wfs_descrfeat_url = add_url_params(url, {
"SERVICE": "WFS",
"VERSION": "1.1.0",
"REQUEST": "DescribeFeatureType",
})
if url not in self.featuretype_cache:
print("Get WFS DescribeFeatureType for URL: {}".format(wfs_descrfeat_url))
self.featuretype_cache[url] = None
# forward request to target (without Host Header)
http = httplib2.Http()
h = {}
if hostname == "localhost": # pragma: no cover
h["Host"] = self.package["host"]
try:
resp, content = http.request(wfs_descrfeat_url, method="GET", headers=h)
except Exception as e: # pragma: no cover
print(colorize(str(e), RED))
print(colorize(
"ERROR! Unable to DescribeFeatureType from URL: {}".format(wfs_descrfeat_url),
RED,
))
if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
return []
else:
raise
if resp.status < 200 or resp.status >= 300: # pragma: no cover
print(colorize(
"ERROR! DescribeFeatureType from URL {} return the error: {1:d} {}".format(
wfs_descrfeat_url, resp.status, resp.reason
),
RED,
))
if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
return []
else:
raise Exception("Aborted")
try:
describe = parseString(content)
featurestype = {}
self.featuretype_cache[url] = featurestype
for type_element in describe.getElementsByTagNameNS(
"http://www.w3.org/2001/XMLSchema", "complexType"
):
featurestype[type_element.getAttribute("name")] = type_element
except ExpatError as e:
print(colorize(
"ERROR! an error occurred while trying to "
"parse the DescribeFeatureType document.",
RED
))
print(colorize(str(e), RED))
print("URL: {}\nxml:\n{}".format(wfs_descrfeat_url, content))
if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
return []
else:
raise
except AttributeError:
print(colorize(
"ERROR! an error occurred while trying to "
"read the Mapfile and recover the themes.",
RED
))
print("URL: {}\nxml:\n{}".format(wfs_descrfeat_url, content))
if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
return []
else:
raise
else:
featurestype = self.featuretype_cache[url]
if featurestype is None:
return []
layers = [layer]
if wmscap is not None and layer in list(wmscap.contents):
layer_obj = wmscap[layer]
if len(layer_obj.layers) > 0:
layers = [l.name for l in layer_obj.layers]
attributes = []
for sub_layer in layers:
# Should probably be adapted for other king of servers
type_element = featurestype.get("{}Type".format(sub_layer))
if type_element is not None:
for element in type_element.getElementsByTagNameNS(
"http://www.w3.org/2001/XMLSchema", "element"
):
if not element.getAttribute("type").startswith("gml:"):
attributes.append(element.getAttribute("name"))
return attributes, layers
