Esempio n. 1
0
 def files_to_remove(self, pre=False):
     error = False
     for element in self.get_upgrade("files_to_remove"):
         file_ = element["file"]
         if os.path.exists(file_):
             managed = False
             for pattern in self.project["managed_files"]:
                 if re.match(pattern + '$', file_):
                     print(
                         colorize(
                             "The file '{}' is no more use but not delete "
                             "because he is in the managed_files as '{}'.".
                             format(file_, pattern), RED))
                     error = True
                     managed = True
             if not managed and not pre:
                 print(
                     colorize("The file '{}' is removed.".format(file_),
                              GREEN))
                 if "version" in element and "from" in element:
                     print(
                         "Was used in version {}, to be removed from version {}."
                         .format(element["from"], element["version"]))
                 if os.path.isdir(file_):
                     shutil.rmtree(file_)
                 else:
                     os.remove(file_)
     return error
    def __call__(self, filename, options):
        messages = []

        self.env = bootstrap(filename)
        with open("project.yaml") as f:
            self.package = yaml.load(f)
        self.config = get_config(".build/config.yaml")

        try:
            from c2cgeoportal.models import DBSession, Theme, LayerGroup, \
                LayerWMS, LayerWMTS, FullTextSearch

            self._import(Theme, messages)
            self._import(LayerGroup, messages)
            self._import(LayerWMS, messages, self._import_layer_wms)
            self._import(LayerWMTS, messages, self._import_layer_wmts)
        except ProgrammingError as e:
            print(colorize(
                "ERROR: The database is probably not up to date "
                "(should be ignored when happen during the upgrade)",
                RED
            ))
            print(colorize(e, RED))

        for ln, in DBSession.query(FullTextSearch.layer_name).distinct().all():
            if ln is not None and ln != "":
                messages.append(Message(
                    None, ln, None, [], u"", u"",
                    ("fts", ln.encode("ascii", errors="replace"))
                ))

        return messages
Esempio n. 3
0
 def __call__(self, filename, options):
     message_str = subprocess.check_output(["node", "tools/extract-messages.js", filename])
     try:
         messages = loads(message_str)
         return [Message(
             None, message, None, [], u"", u"", context.split(":")
         ) for context, message in messages]
     except:
         print(colorize("An error occurred", RED))
         print(colorize(message_str, RED))
         print("------")
         raise
Esempio n. 4
0
 def files_to_get(self, pre=False):
     error = False
     for root, _, files in os.walk("CONST_create_template"):
         root = root[len("CONST_create_template/"):]
         for file_ in files:
             destination = os.path.join(root, file_)
             managed = False
             for pattern in self.project["managed_files"] + self.get_upgrade(
                     "default_project_file"):
                 if re.match(pattern + '$', destination):
                     managed = True
                     for unpattern in self.project.get(
                             "unmanaged_files", []):
                         if re.match(unpattern + '$', destination):
                             managed = False
                             break
                     break
             source = os.path.join("CONST_create_template", destination)
             if not managed and (not os.path.exists(destination)
                                 or not filecmp.cmp(source, destination)):
                 if not pre:
                     print(
                         colorize(
                             "Get the file '{}' from the create template.".
                             format(destination), GREEN))
                     if os.path.dirname(destination) != "":
                         os.makedirs(os.path.dirname(destination),
                                     exist_ok=True)
                     shutil.copyfile(source, destination)
                     shutil.copymode(source, destination)
     return error
Esempio n. 5
0
 def _import_layer_wms(self, layer, messages):
     server = layer.ogc_server
     url = server.url_wfs or server.url
     if url is None:
         return
     for wms_layer in layer.layer.split(","):
         self._import_layer_attributes(
             url, wms_layer, layer.item_type, layer.name, layer.id, messages
         )
     if layer.geo_table is not None:
         exclude = [] if layer.exclude_properties is None else layer.exclude_properties.split(",")
         last_update_date = layer.get_metadatas("lastUpdateDateColumn")
         if len(last_update_date) == 1:
             exclude.append(last_update_date[0].value)
         last_update_user = layer.get_metadatas("lastUpdateUserColumn")
         if len(last_update_user) == 1:
             exclude.append(last_update_user[0].value)
         try:
             cls = get_class(layer.geo_table, exclude_properties=exclude)
             for column_property in class_mapper(cls).iterate_properties:
                 if isinstance(column_property, ColumnProperty) and \
                         len(column_property.columns) == 1 and \
                         not column_property.columns[0].primary_key and \
                         not column_property.columns[0].foreign_keys and \
                         not isinstance(column_property.columns[0].type, Geometry):
                     messages.append(Message(
                         None, column_property.key, None, [], "", "",
                         (".".join(["edit", layer.item_type, str(layer.id)]), layer.name)
                     ))
         except NoSuchTableError:
             exit(colorize("No such table '{}' for layer '{}'.".format(layer.geo_table, layer.name), RED))
Esempio n. 6
0
 def __call__(self, filename, options):
     message_str = subprocess.check_output(
         ["node", "tools/extract-messages.js", filename])
     try:
         messages = []
         for contexts, message in loads(message_str):
             for context in contexts.split(", "):
                 messages.append(
                     Message(None, message, None, [], u"", u"",
                             context.split(":")))
         return messages
     except:
         print(colorize("An error occurred", RED))
         print(colorize(message_str, RED))
         print("------")
         raise
Esempio n. 7
0
    def deploy(self):
        if not self.test_checkers():
            print(colorize("Correct them and run again", RED))
            exit(1)

        check_call(["sudo", "-u", "deploy", "deploy", "-r", "deploy/deploy.cfg", self.options.host])
        check_call(["make", "-f", self.options.file, "build"])
Esempio n. 8
0
    def deploy(self):
        if not self.test_checkers():
            print(colorize("Correct them and run again", RED))
            exit(1)

        check_call(["sudo", "-u", "deploy", "deploy", "-r", "deploy/deploy.cfg", self.options.host])
        check_call(["make", "-f", self.options.file, "build"])
Esempio n. 9
0
 def print_step(self,
                step,
                error=False,
                message=None,
                prompt="To continue type:"):
     print("")
     print(self.color_bar)
     if message is not None:
         print(colorize(message, RED if error else YELLOW))
     if step >= 0:
         print(colorize(prompt, GREEN))
         args = " --makefile=" + self.options.new_makefile if self.options.makefile != "Makefile" else ""
         print(
             colorize("./docker-run make{} upgrade{}", GREEN).format(
                 args,
                 step if step != 0 else "",
             ))
Esempio n. 10
0
    def get_upgrade(section):
        if not os.path.isfile(".upgrade.yaml"):
            print(
                colorize("Unable to find the required '.upgrade.yaml' file.",
                         RED))
            exit(1)

        with open(".upgrade.yaml", "r") as f:
            return yaml.safe_load(f)[section]
Esempio n. 11
0
    def get_project():
        if not os.path.isfile("project.yaml"):
            print(
                colorize("Unable to find the required 'project.yaml' file.",
                         RED))
            exit(1)

        with open("project.yaml", "r") as f:
            return yaml.safe_load(f)
Esempio n. 12
0
    def step0(self):
        project_template_keys = self.project.get("template_vars").keys()
        messages = []
        for required in REQUIRED_TEMPLATE_KEYS:
            if required not in project_template_keys:
                messages.append(
                    "The element '{}' is missing in the 'template_vars' of "
                    "the file 'project.yaml.mako', you should for example: {}: {}.".format(
                        required, required, TEMPLATE_EXAMPLE.get('required', '')
                    )
                )
        if len(messages) > 0:
            print("")
            print(self.color_bar)
            print(colorize("\n".join(messages), RED))
            print("")
            self.print_step(0, intro="Fix it and run again the upgrade:")
            exit(1)

        if re.match(VERSION_RE, self.options.version) is not None:
            http = httplib2.Http()
            url = (
                "http://raw.github.com/camptocamp/c2cgeoportal/%s/"
                "c2cgeoportal/scaffolds/update/CONST_versions_requirements.txt" %
                self.options.version
            )
            headers, _ = http.request(url)
            if headers.status != 200:
                print("")
                print(self.color_bar)
                print(
                    "Failed downloading the c2cgeoportal "
                    "CONST_versions_requirements.txt file from URL:"
                )
                print(url)
                print("The upgrade is impossible")
                exit(1)

        if os.path.split(os.path.realpath("."))[1] != self.project["project_folder"]:
            print("")
            print(self.color_bar)
            print("Your project isn't in the right folder!")
            print("It should be in folder '%s' instead of folder '%s'." % (
                self.project["project_folder"], os.path.split(os.path.realpath("."))[1]
            ))
            print("")
            self.print_step(0, intro="Fix it and lunch again the upgrade:")
            exit(1)

        check_call(["git", "status"])
        print("")
        print(self.color_bar)
        print(
            "Here is the output of 'git status'. Please make sure to commit all your changes "
            "before going further. All uncommited changes will be lost."
        )
        self.print_step(1)
Esempio n. 13
0
    def __call__(self, filename, options):
        messages = []
        self.capabilities_cache = {}

        self.env = bootstrap(filename)

        try:
            from c2cgeoportal.models import Theme, LayerGroup, \
                LayerWMS, LayerWMTS

            self._import(Theme, messages)
            self._import(LayerGroup, messages)
            self._import(LayerWMS, messages)
            self._import(LayerWMTS, messages)
        except ProgrammingError as e:
            print(colorize("ERROR: The database is probably not up to date", RED))
            print(colorize(e, RED))

        return messages
    def __call__(self, filename, options):
        messages = []

        self.env = bootstrap(filename)
        with open("project.yaml") as f:
            self.package = yaml.load(f)

        try:
            from c2cgeoportal.models import Theme, LayerGroup, LayerWMS, LayerWMTS

            self._import(Theme, messages)
            self._import(LayerGroup, messages)
            self._import(LayerWMS, messages, self._import_layer_wms)
            self._import(LayerWMTS, messages, self._import_layer_wmts)
        except ProgrammingError as e:
            print(colorize("ERROR: The database is probably not up to date", RED))
            print(colorize(e, RED))

        return messages
Esempio n. 15
0
    def post(self, command, output_dir, vars):
        if os.name == 'posix':
            dest = os.path.join(output_dir, ".whiskey/action_hooks/pre-build.mako")
            subprocess.check_call(["chmod", "+x", dest])
        """
        Overrides the base template class to print "Welcome to c2cgeoportal!"
        after a successful scaffolding rendering.
        """

        self.out(colorize("\nWelcome to c2cgeoportal!", GREEN))

        return BaseTemplate.post(self, command, output_dir, vars)
Esempio n. 16
0
    def post(self, command, output_dir, vars):
        if os.name == 'posix':
            dest = os.path.join(output_dir, ".whiskey/action_hooks/pre-build.mako")
            subprocess.check_call(["chmod", "+x", dest])
        """
        Overrides the base template class to print "Welcome to c2cgeoportal!"
        after a successful scaffolding rendering.
        """

        self.out(colorize("\nWelcome to c2cgeoportal!", GREEN))

        return BaseTemplate.post(self, command, output_dir, vars)
Esempio n. 17
0
    def step4(self):
        check_call(["git", "commit", "-m", "Upgrade to GeoMapFish %s" % self.options.version])

        print("")
        print(self.color_bar)
        print("")
        print(colorize("Congratulations your upgrade is a success.", GREEN))
        print("")
        branch = check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).strip()
        print("Now all your files will be commited, you should do a git push %s %s." % (
            self.options.git_remote, branch
        ))
Esempio n. 18
0
 def _import_layer_wms(self, layer, messages):
     server = layer.ogc_server
     url = server.url_wfs or server.url
     if url is None:
         return
     for wms_layer in layer.layer.split(","):
         self._import_layer_attributes(url, wms_layer, layer.item_type,
                                       layer.name, messages)
     if layer.geo_table is not None and layer.geo_table != "":
         exclude = [] if layer.exclude_properties is None else layer.exclude_properties.split(
             ",")
         last_update_date = layer.get_metadatas("lastUpdateDateColumn")
         if len(last_update_date) == 1:
             exclude.append(last_update_date[0].value)
         last_update_user = layer.get_metadatas("lastUpdateUserColumn")
         if len(last_update_user) == 1:
             exclude.append(last_update_user[0].value)
         try:
             cls = get_class(layer.geo_table, exclude_properties=exclude)
             for column_property in class_mapper(cls).iterate_properties:
                 if isinstance(column_property, ColumnProperty) and \
                         len(column_property.columns) == 1 and \
                         not column_property.columns[0].primary_key and \
                         not column_property.columns[0].foreign_keys and \
                         not isinstance(column_property.columns[0].type, Geometry):
                     messages.append(
                         Message(
                             None, column_property.key, None, [], "", "",
                             (".".join(
                                 ["edit", layer.item_type,
                                  str(layer.id)]), layer.name)))
         except NoSuchTableError:
             print(
                 colorize(
                     "ERROR! No such table '{}' for layer '{}'.".format(
                         layer.geo_table, layer.name), RED))
             print(colorize(traceback.format_exc(), RED))
             if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
                 raise
Esempio n. 19
0
def main():
    """
    tool used to help th use in the user tash
    """

    usage = """usage: {prog} [command] [options]

Available commands:

""" + colorize("help", GREEN) + """: show this page
""" + colorize("upgrade", GREEN) + """: upgrade the application to a new version
""" + colorize("deploy", GREEN) + """: deploy the application to a server

To have some help on a command type:
{prog} help [command]""".format(prog=sys.argv[0])

    if len(sys.argv) <= 1:
        print(usage)
        exit()

    if sys.argv[1] == "help":
        if len(sys.argv) > 2:
            parser = _fill_arguments(sys.argv[2])
            parser.print_help()
        else:
            print(usage)
        exit()

    parser = _fill_arguments(sys.argv[1])
    options = parser.parse_args(sys.argv[2:])

    c2ctool = C2cTool(options)
    if sys.argv[1] == "upgrade":
        c2ctool.upgrade()
    elif sys.argv[1] == "deploy":
        c2ctool.deploy()
    else:
        print("Unknown command")
Esempio n. 20
0
def main():  # pragma: no cover
    """
    tool used to help th use in the user tash
    """

    usage = """usage: {prog} [command] [options]

Available commands:

""" + colorize("help", GREEN) + """: show this page
""" + colorize("upgrade", GREEN) + """: upgrade the application to a new version
""" + colorize("deploy", GREEN) + """: deploy the application to a server

To have some help on a command type:
{prog} help [command]""".format(prog=sys.argv[0])

    if len(sys.argv) <= 1:
        print(usage)
        exit()

    if sys.argv[1] == "help":
        if len(sys.argv) > 2:
            parser = _fill_arguments(sys.argv[2])
            parser.print_help()
        else:
            print(usage)
        exit()

    parser = _fill_arguments(sys.argv[1])
    options = parser.parse_args(sys.argv[2:])

    c2ctool = C2cTool(options)
    if sys.argv[1] == "upgrade":
        c2ctool.upgrade()
    elif sys.argv[1] == "deploy":
        c2ctool.deploy()
    else:
        print("Unknown command")
Esempio n. 21
0
    def step6(self):
        check_call(["git", "commit", "-m", "Upgrade to GeoMapFish {}".format(
            pkg_resources.get_distribution("c2cgeoportal").version
        )])

        print("")
        print(self.color_bar)
        print("")
        print(colorize("Congratulations your upgrade is a success.", GREEN))
        print("")
        branch = check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).strip()
        print("Now all your files will be commited, you should do a git push {0!s} {1!s}.".format(
            self.options.git_remote, branch
        ))
Esempio n. 22
0
    def step6(self):
        check_call(["git", "commit", "-m", "Upgrade to GeoMapFish {}".format(
            pkg_resources.get_distribution("c2cgeoportal").version
        )])

        print("")
        print(self.color_bar)
        print("")
        print(colorize("Congratulations your upgrade is a success.", GREEN))
        print("")
        branch = check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).strip()
        print("Now all your files will be commited, you should do a git push %s %s." % (
            self.options.git_remote, branch
        ))
Esempio n. 23
0
 def files_to_move(self, pre=False):
     error = False
     for element in self.get_upgrade("files_to_move"):
         src = element["from"]
         dst = element["to"]
         if os.path.exists(src):
             type_ = "directory" if os.path.isdir(src) else "file"
             managed = False
             for pattern in self.project["managed_files"]:
                 if re.match(pattern + '$', src):
                     print(
                         colorize(
                             "The {} '{}' is present in the managed_files as '{}' but he will move."
                             .format(type_, src, pattern), RED))
                     error = True
                     managed = True
                     break
             if not managed and os.path.exists(dst):
                 print(
                     colorize(
                         "The destination '{}' already exists.".format(dst),
                         RED))
                 error = True
                 if not pre:
                     raise InteruptedException(
                         "The destination '{}' already exists.".format(dst))
             if not managed and not pre:
                 print(
                     colorize(
                         "Move the {} '{}' to '{}'.".format(
                             type_, src, dst), GREEN))
                 if "version" in element:
                     print("Needed from version {}.".format(
                         element["version"]))
                 os.rename(src, dst)
     return error
Esempio n. 24
0
    def __call__(self, filename, options):
        messages = []

        self.env = bootstrap(filename)
        with open("project.yaml") as f:
            self.package = yaml.load(f)
        self.config = get_config(".build/config.yaml")

        try:
            from c2cgeoportal.models import Theme, LayerGroup, \
                LayerWMS, LayerWMTS

            self._import(Theme, messages)
            self._import(LayerGroup, messages)
            self._import(LayerWMS, messages, self._import_layer_wms)
            self._import(LayerWMTS, messages, self._import_layer_wmts)
        except ProgrammingError as e:
            print(
                colorize(
                    "ERROR: The database is probably not up to date "
                    "(should be ignored when happen during the upgrade)", RED))
            print(colorize(e, RED))

        return messages
Esempio n. 25
0
    def post(self, command, output_dir, vars):
        """
        Overrides the base template class to print the next step.
        """

        if os.name == 'posix':
            for file in ("post-restore-code", "pre-restore-database.mako"):
                dest = os.path.join(output_dir, "deploy/hooks", file)
                subprocess.check_call(["chmod", "+x", dest])

        self.out("\nContinue with:")
        self.out(colorize(
            ".build/venv/bin/pcreate -s c2cgeoportal_update ../{vars[project]} "
            "package={vars[package]} srid={vars[srid]} extent={vars[extent]}".format(vars=vars),
            GREEN
        ))

        return BaseTemplate.post(self, command, output_dir, vars)
Esempio n. 26
0
    def __call__(self, filename, options):
        messages = []

        try:
            self.env = bootstrap(filename)
            with open("project.yaml") as f:
                self.package = yaml.safe_load(f)
            self.config = get_config("config.yaml")

            try:
                from c2cgeoportal.models import DBSession, Theme, LayerGroup, \
                    LayerWMS, LayerWMTS, FullTextSearch

                self._import(Theme, messages)
                self._import(LayerGroup, messages)
                self._import(LayerWMS, messages, self._import_layer_wms)
                self._import(LayerWMTS, messages, self._import_layer_wmts)

                for ln, in DBSession.query(
                        FullTextSearch.layer_name).distinct().all():
                    if ln is not None and ln != "":
                        messages.append(
                            Message(
                                None, ln, None, [], "", "",
                                ("fts", ln.encode("ascii", errors="replace"))))
            except ProgrammingError as e:
                print(
                    colorize(
                        "ERROR! The database is probably not up to date "
                        "(should be ignored when happen during the upgrade)",
                        RED))
                print(colorize(e, RED))
                if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
                    raise
        except NoSuchTableError as e:
            print(
                colorize(
                    "ERROR! The schema didn't seem to exists "
                    "(should be ignored when happen during the deploy)", RED))
            print(colorize(e, RED))
            if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
                raise
        except OperationalError as e:
            print(
                colorize(
                    "ERROR! The database didn't seem to exists "
                    "(should be ignored when happen during the deploy)", RED))
            print(colorize(e, RED))
            if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
                raise

        return messages
Esempio n. 27
0
    def post(self, command, output_dir, vars):
        """
        Overrides the base template class to print the next step.
        """

        if os.name == 'posix':
            for file in ("post-restore-code", "pre-restore-database.mako"):
                dest = os.path.join(output_dir, "deploy/hooks", file)
                subprocess.check_call(["chmod", "+x", dest])

        self.out("\nContinue with:")
        self.out(colorize(
            ".build/venv/bin/pcreate -s c2cgeoportal_update ../{vars[project]} "
            "package={vars[package]} srid={vars[srid]} extent={vars[extent]}".format(vars=vars),
            GREEN
        ))

        return BaseTemplate.post(self, command, output_dir, vars)
Esempio n. 28
0
 def _enumerate_attributes_values(cls, dbsessions, layers, layerinfos,
                                  fieldname):
     dbname = layerinfos.get("dbsession", "dbsession")
     try:
         dbsession = dbsessions.get(dbname)
         return layers.query_enumerate_attribute_values(
             dbsession, layerinfos, fieldname)
     except Exception as e:
         table = layerinfos.get("attributes").get(fieldname,
                                                  {}).get("table")
         print(
             colorize(
                 "ERROR! Unable to collect enumerate attributes for "
                 "db: {}, table: {}, column: {} ({})".format(
                     dbname, table, fieldname, e), RED))
         if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
             return []
         else:
             raise
Esempio n. 29
0
    def _import_layer_wmts(self, layer, messages):
        from c2cgeoportal.models import DBSession, OGCServer

        layers = [d.value for d in layer.metadatas if d.name == "queryLayers"]
        if len(layers) == 0:
            layers = [d.value for d in layer.metadatas if d.name == "wmsLayer"]
        server = [d.value for d in layer.metadatas if d.name == "ogcServer"]
        if len(server) >= 1 and len(layers) >= 1:
            for wms_layer in layers:
                try:
                    db_server = DBSession.query(OGCServer).filter(
                        OGCServer.name == server[0]).one()
                    self._import_layer_attributes(
                        db_server.url_wfs or db_server.url, wms_layer,
                        layer.item_type, layer.name, messages)
                except NoResultFound:
                    print(
                        colorize(
                            "ERROR! the OGC server '{}' from the WMTS layer '{}' does not exist."
                            .format(server[0], layer.name), RED))
                    if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
                        raise
Esempio n. 30
0
 def print_step(self, step, intro="To continue type:"):
     print(intro)
     print(colorize("make -f %s upgrade%i", YELLOW) % (
         self.options.file if self.options.file is not None else "<user.mk>",
         step
     ))
Esempio n. 31
0
    def __call__(self, filename, options):
        config = get_config("config.yaml")

        class Registry:
            settings = config

        class Request:
            registry = Registry()
            params = {}
            GET = {}
            user_agent = ""

            @staticmethod
            def static_url(*args, **kwargs):
                return ""

            @staticmethod
            def static_path(*args, **kwargs):
                return ""

            @staticmethod
            def route_url(*args, **kwargs):
                return ""

            @staticmethod
            def current_route_url(*args, **kwargs):
                return ""

        init_region({"backend": "dogpile.cache.memory"})

        int_filename = filename
        if re.match(
                "^" + re.escape("./{}/templates".format(config["package"])),
                filename):
            try:
                empty_template = Template("")

                class Lookup(TemplateLookup):
                    @staticmethod
                    def get_template(uri):
                        return empty_template

                class MyTemplate(MakoTemplate):
                    def prepare(self, **options):
                        options.update({"input_encoding": self.encoding})
                        lookup = Lookup(**options)
                        if self.source:
                            self.tpl = Template(self.source,
                                                lookup=lookup,
                                                **options)
                        else:
                            self.tpl = Template(uri=self.name,
                                                filename=self.filename,
                                                lookup=lookup,
                                                **options)

                try:
                    processed = template(filename, {
                        "request": Request(),
                        "lang": "fr",
                        "debug": False,
                        "extra_params": {},
                        "permalink_themes": "",
                        "fulltextsearch_groups": [],
                        "wfs_types": [],
                        "_": lambda x: x,
                    },
                                         template_adapter=MyTemplate)
                    int_filename = os.path.join(
                        os.path.dirname(filename),
                        "_" + os.path.basename(filename))
                    with open(int_filename, "wb") as file_open:
                        file_open.write(processed.encode("utf-8"))
                except:
                    print(
                        colorize(
                            "ERROR! Occurred during the '{}' template generation"
                            .format(filename), RED))
                    print(colorize(traceback.format_exc(), RED))
                    if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
                        # Continue with the original one
                        int_filename = filename
                    else:
                        raise
            except:
                print(traceback.format_exc())

        message_str = subprocess.check_output(
            ["node", "tools/extract-messages.js",
             int_filename]).decode("utf-8")
        if int_filename != filename:
            os.unlink(int_filename)
        try:
            messages = []
            for contexts, message in loads(message_str):
                for context in contexts.split(", "):
                    messages.append(
                        Message(None, message, None, [], "", "",
                                context.split(":")))
            return messages
        except:
            print(colorize("An error occurred", RED))
            print(colorize(message_str, RED))
            print("------")
            raise
Esempio n. 32
0
    def _layer_attributes(self, url, layer):
        errors = set()

        class Registry:
            setting = None

        class Request:
            registry = Registry()

        request = Request()
        request.registry.settings = self.config
        # static schema will not be supported
        url = get_url2("Layer", url, request, errors)
        if len(errors) > 0:
            print("\n".join(errors))
            return []

        wms_getcap_url = add_url_params(url, {
            "SERVICE": "WMS",
            "VERSION": "1.1.1",
            "REQUEST": "GetCapabilities",
        })

        hostname = urlsplit(url).hostname
        if url not in self.wmscap_cache:
            print("Get WMS GetCapabilities for URL: {}".format(url))
            self.wmscap_cache[url] = None

            # forward request to target (without Host Header)
            http = httplib2.Http()
            h = {}
            if hostname == "localhost":  # pragma: no cover
                h["Host"] = self.package["host"]
            try:
                resp, content = http.request(wms_getcap_url,
                                             method="GET",
                                             headers=h)

                try:
                    self.wmscap_cache[url] = WebMapService(None, xml=content)
                except Exception as e:
                    print(
                        colorize(
                            "ERROR! an error occurred while trying to "
                            "parse the GetCapabilities document.", RED))
                    print(colorize(str(e), RED))
                    print("URL: {}\nxml:\n{}".format(wms_getcap_url, content))
                    if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
                        raise
            except Exception as e:  # pragma: no cover
                print(colorize(str(e), RED))
                print(
                    colorize(
                        "ERROR! Unable to GetCapabilities from URL: {}".format(
                            wms_getcap_url),
                        RED,
                    ))
                if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE":
                    raise

        wmscap = self.wmscap_cache[url]

        wfs_descrfeat_url = add_url_params(
            url, {
                "SERVICE": "WFS",
                "VERSION": "1.1.0",
                "REQUEST": "DescribeFeatureType",
            })

        if url not in self.featuretype_cache:
            print("Get WFS DescribeFeatureType for URL: {}".format(
                wfs_descrfeat_url))
            self.featuretype_cache[url] = None

            # forward request to target (without Host Header)
            http = httplib2.Http()
            h = {}
            if hostname == "localhost":  # pragma: no cover
                h["Host"] = self.package["host"]
            try:
                resp, content = http.request(wfs_descrfeat_url,
                                             method="GET",
                                             headers=h)
            except Exception as e:  # pragma: no cover
                print(colorize(str(e), RED))
                print(
                    colorize(
                        "ERROR! Unable to DescribeFeatureType from URL: {}".
                        format(wfs_descrfeat_url),
                        RED,
                    ))
                if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
                    return []
                else:
                    raise

            if resp.status < 200 or resp.status >= 300:  # pragma: no cover
                print(
                    colorize(
                        "ERROR! DescribeFeatureType from URL {} return the error: {1:d} {}"
                        .format(wfs_descrfeat_url, resp.status, resp.reason),
                        RED,
                    ))
                if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
                    return []
                else:
                    raise Exception("Aborted")

            try:
                describe = parseString(content)
                featurestype = {}
                self.featuretype_cache[url] = featurestype
                for type_element in describe.getElementsByTagNameNS(
                        "http://www.w3.org/2001/XMLSchema", "complexType"):
                    featurestype[type_element.getAttribute(
                        "name")] = type_element
            except ExpatError as e:
                print(
                    colorize(
                        "ERROR! an error occurred while trying to "
                        "parse the DescribeFeatureType document.", RED))
                print(colorize(str(e), RED))
                print("URL: {}\nxml:\n{}".format(wfs_descrfeat_url, content))
                if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
                    return []
                else:
                    raise
            except AttributeError:
                print(
                    colorize(
                        "ERROR! an error occurred while trying to "
                        "read the Mapfile and recover the themes.", RED))
                print("URL: {}\nxml:\n{}".format(wfs_descrfeat_url, content))
                if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE":
                    return []
                else:
                    raise
        else:
            featurestype = self.featuretype_cache[url]

        if featurestype is None:
            return []

        layers = [layer]
        if wmscap is not None and layer in list(wmscap.contents):
            layer_obj = wmscap[layer]
            if len(layer_obj.layers) > 0:
                layers = [l.name for l in layer_obj.layers]

        attributes = []
        for sub_layer in layers:
            # Should probably be adapted for other king of servers
            type_element = featurestype.get("{}Type".format(sub_layer))
            if type_element is not None:
                for element in type_element.getElementsByTagNameNS(
                        "http://www.w3.org/2001/XMLSchema", "element"):
                    if not element.getAttribute("type").startswith("gml:"):
                        attributes.append(element.getAttribute("name"))

        return attributes
Esempio n. 33
0
    def step1(self):
        notes = []

        check_call(["git", "reset", "--hard"])
        check_call(["git", "clean", "--force", "-d"])
        check_call(["git", "submodule", "foreach", "--recursive", "git", "reset", "--hard"])
        check_call(["git", "submodule", "foreach", "--recursive", "git", "clean", "--force", "-d"])

        branch = check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).strip()
        # remove all no more existing branches
        check_call(["git", "fetch", "origin", "--prune"])
        branches = check_output(["git", "branch", "--all"]).split("\n")
        if "  remotes/origin/%s" % branch in branches:
            try:
                check_call(["git", "pull", "--rebase", self.options.git_remote, branch])
            except CalledProcessError:
                print(self.color_bar)
                print(colorize("The pull (rebase) failed.", RED))
                print(colorize("Please solve the rebase and run the step again.", YELLOW))
                exit(1)

        check_call(["git", "submodule", "sync"])
        check_call(["git", "submodule", "update", "--init"])
        check_call(["git", "submodule", "foreach", "git", "submodule", "sync"])
        check_call(["git", "submodule", "foreach", "git", "submodule", "update", "--init"])

        if len(check_output(["git", "status", "-z"]).strip()) != 0:
            print(self.color_bar)
            print(colorize("The pull isn't fast forward.", RED))
            print(colorize("Please solve the rebase and run the step again.", YELLOW))
            exit(1)

        check_call(["git", "submodule", "foreach", "git", "submodule", "sync"])
        check_call(["git", "submodule", "foreach", "git", "submodule", "update", "--init"])

        if not self.options.windows:
            check_call(["make", "-f", self.options.file, ".build/requirements.timestamp"])
            pip_cmd = ["%s/pip" % self.venv_bin, "install"]
            if self.options.version == "master":
                check_call(["%s/pip" % self.venv_bin, "uninstall", "--yes", "c2cgeoportal"])
                pip_cmd += ["--pre", "c2cgeoportal"]
            else:
                pip_cmd += ["c2cgeoportal==%s" % (self.options.version)]
            check_call(pip_cmd)

        check_call([
            "%s/pcreate" % self.venv_bin, "--ignore-conflicting-name", "--overwrite",
            "--scaffold=c2cgeoportal_update", "../%s" % self.project["project_folder"]
        ])
        pcreate_cmd = [
            "%s/pcreate" % self.venv_bin, "--ignore-conflicting-name",
            "--scaffold=c2cgeoportal_create", "/tmp/%s" % self.project["project_folder"],
        ]
        for name, value in self.project["template_vars"].items():
            if isinstance(value, basestring):
                value = value.encode('utf-8')
            pcreate_cmd.append("{}={}".format(name, value))
        check_call(pcreate_cmd)
        check_call(["make", "-f", self.options.file, self.options.clean])

        diff_file = open("changelog.diff", "w")
        check_call(["git", "diff", "CONST_CHANGELOG.txt"], stdout=diff_file)
        diff_file.close()

        check_call(["make", "-f", self.options.file, "update-node-modules"])
        check_call(["make", "-f", self.options.file, ".build/requirements.timestamp"])

        print("")
        print(self.color_bar)
        print("\n".join(notes))
        print(
            "Apply the manual migration steps based on what is in the CONST_CHANGELOG.txt file"
            " (listed in the `changelog.diff` file)."
        )
        self.print_step(2)
Esempio n. 34
0
    def step1(self):
        notes = []

        check_call(["git", "reset", "--hard"])
        check_call(["git", "clean", "--force", "-d"])
        check_call(["git", "submodule", "foreach", "--recursive", "git", "reset", "--hard"])
        check_call(["git", "submodule", "foreach", "--recursive", "git", "clean", "--force", "-d"])

        branch = check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).strip()
        # remove all no more existing branches
        check_call(["git", "update", "origin", "--prune"])
        branches = check_output(["git", "branch", "--all"]).split("\n")
        if "  remotes/origin/%s" % branch in branches:
            try:
                check_call(["git", "pull", "--rebase", self.options.git_remote, branch])
            except CalledProcessError:
                print(self.color_bar)
                print(colorize("The pull (rebase) failed.", RED))
                print(colorize("Please solve the rebase and run the step again.", YELLOW))
                exit(1)

        check_call(["git", "submodule", "sync"])
        check_call(["git", "submodule", "update", "--init"])
        check_call(["git", "submodule", "foreach", "git", "submodule", "sync"])
        check_call(["git", "submodule", "foreach", "git", "submodule", "update", "--init"])

        if len(check_output(["git", "status", "-z"]).strip()) != 0:
            print(self.color_bar)
            print(colorize("The pull isn't fast forward.", RED))
            print(colorize("Please solve the rebase and run the step again.", YELLOW))
            exit(1)

        check_call(["git", "submodule", "foreach", "git", "fetch", "origin"])
        if self.options.version == "master":
            check_call([
                "git", "submodule", "foreach", "git", "reset", "--hard",
                "origin/%s" % self.options.version, "--"
            ])
        elif re.match(VERSION_RE, self.options.version) is not None:
            check_call([
                "git", "submodule", "foreach", "git", "reset", "--hard",
                self.options.version, "--"
            ])
        else:
            notes.append(
                "We can't define the cgxp revision, than you should manually do:\n"
                "git submodule foreach git reset --hard <revision>"
            )

        check_call(["git", "submodule", "foreach", "git", "submodule", "sync"])
        check_call(["git", "submodule", "foreach", "git", "submodule", "update", "--init"])

        if not self.options.windows:
            check_call(["make", "-f", self.options.file, ".build/requirements.timestamp"])
            pip_cmd = [
                "%s/pip" % self.venv_bin, "install",
                "--find-links", self.options.index_url,
            ]
            if self.options.version == "master":
                check_call(["%s/pip" % self.venv_bin, "uninstall", "--yes", self.package])
                pip_cmd += ["--pre", self.package]
            else:
                pip_cmd += ["%s==%s" % (self.package, self.options.version)]
            check_call(pip_cmd)

        check_call([
            "%s/pcreate" % self.venv_bin, "--overwrite", "--scaffold=c2cgeoportal_update",
            "../%s" % self.project["project_folder"], "package=%s" % self.project["project_package"]
        ])
        check_call([
            "%s/pcreate" % self.venv_bin, "-s", "c2cgeoportal_create",
            "/tmp/%s" % self.project["project_folder"],
            "package=%s" % self.project["project_package"],
            "mobile_application_title=%s" %
            self.project["template_vars"]["mobile_application_title"],
            "srid=%s" % self.project["template_vars"].get("srid", 21781),
        ])
        check_call(["make", "-f", self.options.file, self.options.clean])

        diff_file = open("changelog.diff", "w")
        check_call(["git", "diff", "CONST_CHANGELOG.txt"], stdout=diff_file)
        diff_file.close()

        check_call(["make", "-f", self.options.file, ".build/requirements.timestamp"])

        print("")
        print(self.color_bar)
        print("\n".join(notes))
        print(
            "Apply the manual migration steps based on what is in the CONST_CHANGELOG.txt file"
            " (listed in the `changelog.diff` file)."
        )
        self.print_step(2)
Esempio n. 35
0
    def __call__(self, filename, options):
        config = get_config(".build/config.yaml")

        class Registry:
            settings = config

        class Request:
            registry = Registry()
            params = {}
            GET = {}
            user_agent = ""

            def static_url(*args, **kwargs):
                return ""

            def static_path(*args, **kwargs):
                return ""

            def route_url(*args, **kwargs):
                return ""

            def current_route_url(*args, **kwargs):
                return ""

        init_region({"backend": "dogpile.cache.memory"})

        int_filename = filename
        if re.match("^" + re.escape("./{}/templates".format(config["package"])), filename):
            try:
                empty_template = Template("")

                class Lookup(TemplateLookup):
                    @staticmethod
                    def get_template(uri):
                        return empty_template

                class MyTemplate(MakoTemplate):
                    def prepare(self, **options):
                        options.update({"input_encoding": self.encoding})
                        lookup = Lookup(**options)
                        if self.source:
                            self.tpl = Template(self.source, lookup=lookup, **options)
                        else:
                            self.tpl = Template(
                                uri=self.name,
                                filename=self.filename,
                                lookup=lookup, **options)

                processed = template(
                    filename,
                    {
                        "request": Request(),
                        "lang": "fr",
                        "debug": False,
                        "extra_params": {},
                        "permalink_themes": "",
                        "fulltextsearch_groups": [],
                        "wfs_types": [],
                        "_": lambda x: x,
                    },
                    template_adapter=MyTemplate
                )
                int_filename = os.path.join(os.path.dirname(filename), "_" + os.path.basename(filename))
                with open(int_filename, "wb") as file_open:
                    file_open.write(processed.encode("utf-8"))
            except:
                print(traceback.format_exc())

        message_str = subprocess.check_output(["node", "tools/extract-messages.js", int_filename])
        if int_filename != filename:
            os.unlink(int_filename)
        try:
            messages = []
            for contexts, message in loads(message_str):
                for context in contexts.split(", "):
                    messages.append(Message(
                        None, message, None, [], u"", u"", context.split(":")
                    ))
            return messages
        except:
            print(colorize("An error occurred", RED))
            print(colorize(message_str, RED))
            print("------")
            raise
Esempio n. 36
0
class C2cTool:

    color_bar = colorize("================================================================", GREEN)

    def print_step(self, step, intro="To continue type:"):
        print(colorize(intro, YELLOW))
        print(colorize("make -f {} upgrade{}", GREEN).format(
            self.options.file if self.options.file is not None else "<user.mk>",
            step if step != 0 else "",
        ))

    def get_project(self):
        if not os.path.isfile("project.yaml"):
            print("Unable to find the required 'project.yaml' file.")
            exit(1)

        with open("project.yaml", "r") as f:
            return yaml.load(f)

    def test_checkers(self):
        http = httplib2.Http()
        for check_type in ("", "type=all"):
            resp, content = http.request(
                "http://localhost%s%s" % (self.project["checker_path"], check_type),
                method="GET",
                headers={
                    "Host": self.project["host"]
                }
            )
            if resp.status < 200 or resp.status >= 300:
                print(self.color_bar)
                print("Checker error:")
                print("Open `http://%s%s%s` for more informations." % (
                    self.project["host"], self.project["checker_path"], check_type
                ))
                return False
        return True

    def __init__(self, options):
        self.options = options
        self.project = self.get_project()

    def upgrade(self):
        self.venv_bin = ".build/venv/bin"
        if self.options.windows:
            self.options.clean = "clean"
            self.venv_bin = ".build/venv/Scripts"

        if self.options.step == 0:
            self.step0()
        elif self.options.step == 1:
            self.step1()
        elif self.options.step == 2:
            self.step2()
        elif self.options.step == 3:
            self.step3()
        elif self.options.step == 4:
            self.step4()
        elif self.options.step == 5:
            self.step5()
        elif self.options.step == 6:
            self.step6()

    def step0(self):
        project_template_keys = self.project.get("template_vars").keys()
        messages = []
        for required in REQUIRED_TEMPLATE_KEYS:
            if required not in project_template_keys:
                messages.append(
                    "The element '{}' is missing in the 'template_vars' of "
                    "the file 'project.yaml.mako', you should for example: {}: {}.".format(
                        required, required, TEMPLATE_EXAMPLE.get('required', '')
                    )
                )
        if len(messages) > 0:
            print("")
            print(self.color_bar)
            print(colorize("\n".join(messages), RED))
            print("")
            self.print_step(0, intro="Fix it and run again the upgrade:")
            exit(1)

        if self.options.version is None:
            print("")
            print(self.color_bar)
            print("The VERSION environment variable is required for this upgrade step")
            exit(1)

        if re.match(VERSION_RE, self.options.version) is not None:
            http = httplib2.Http()
            url = (
                "http://raw.github.com/camptocamp/c2cgeoportal/%s/"
                "c2cgeoportal/scaffolds/update/CONST_versions_requirements.txt" %
                self.options.version
            )
            headers, _ = http.request(url)
            if headers.status != 200:
                print("")
                print(self.color_bar)
                print(
                    "Failed downloading the c2cgeoportal "
                    "CONST_versions_requirements.txt file from URL:"
                )
                print(url)
                print("The upgrade is impossible")
                exit(1)

        if os.path.split(os.path.realpath("."))[1] != self.project["project_folder"]:
            print("")
            print(self.color_bar)
            print("Your project isn't in the right folder!")
            print("It should be in folder '%s' instead of folder '%s'." % (
                self.project["project_folder"], os.path.split(os.path.realpath("."))[1]
            ))
            print("")
            self.print_step(0, intro="Fix it and lunch again the upgrade:")
            exit(1)

        if check_output(["git", "status", "--short"]) == "":
            self.step1()
        else:
            check_call(["git", "status"])
            print("")
            print(self.color_bar)
            print(
                "Here is the output of 'git status'. Please make sure to commit all your changes "
                "before going further. All uncommited changes will be lost."
            )
            self.print_step(1)

    def step1(self):
        if self.options.version is None:
            print("")
            print(self.color_bar)
            print("The VERSION environment variable is required for this upgrade step")
            exit(1)

        check_call(["git", "reset", "--hard"])
        check_call(["git", "clean", "--force", "-d"])
        check_call(["git", "submodule", "foreach", "--recursive", "git", "reset", "--hard"])
        check_call(["git", "submodule", "foreach", "--recursive", "git", "clean", "--force", "-d"])

        branch = check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).strip()
        # remove all no more existing branches
        check_call(["git", "fetch", "origin", "--prune"])
        branches = check_output(["git", "branch", "--all"]).split("\n")
        if "  remotes/origin/%s" % branch in branches:
            try:
                check_call(["git", "pull", "--rebase", self.options.git_remote, branch])
            except CalledProcessError:
                print(self.color_bar)
                print("")
                print(colorize("The pull (rebase) failed.", RED))
                print("")
                self.print_step(1, intro="Please solve the rebase and run the step 1 again:")
                exit(1)

        check_call(["git", "submodule", "sync"])
        check_call(["git", "submodule", "update", "--init"])
        check_call(["git", "submodule", "foreach", "git", "submodule", "sync"])
        check_call(["git", "submodule", "foreach", "git", "submodule", "update", "--init"])

        if len(check_output(["git", "status", "-z"]).strip()) != 0:
            print(self.color_bar)
            print("")
            print(colorize("The pull isn't fast forward.", RED))
            print("")
            self.print_step(1, intro="Please solve the rebase and run the step 1 again:")
            exit(1)

        check_call(["git", "submodule", "foreach", "git", "submodule", "sync"])
        check_call(["git", "submodule", "foreach", "git", "submodule", "update", "--init"])

        if not self.options.windows:
            check_call(["make", "-f", self.options.file, ".build/requirements.timestamp"])
            pip_cmd = ["%s/pip" % self.venv_bin, "install"]
            if self.options.version == "master":
                check_call(["%s/pip" % self.venv_bin, "uninstall", "--yes", "c2cgeoportal"])
                pip_cmd += ["--pre", "c2cgeoportal"]
            else:
                pip_cmd += ["c2cgeoportal==%s" % (self.options.version)]
            check_call(pip_cmd)

        check_call([
            "%s/pcreate" % self.venv_bin, "--ignore-conflicting-name", "--overwrite",
            "--scaffold=c2cgeoportal_update", "../%s" % self.project["project_folder"]
        ])
        check_call(["make", "-f", self.options.file, self.options.clean])

        with open("changelog.diff", "w") as diff_file:
            check_call(["git", "diff", "--", "CONST_CHANGELOG.txt"], stdout=diff_file)

        check_call(["make", "-f", self.options.file, "update-node-modules"])
        check_call(["make", "-f", self.options.file, ".build/requirements.timestamp"])

        if os.path.getsize("changelog.diff") == 0:
            self.step2()
        else:
            print("")
            print(self.color_bar)
            print(
                "Apply the manual migration steps based on what is in the CONST_CHANGELOG.txt file"
                " (listed in the `changelog.diff` file)."
            )
            self.print_step(2)

    def step2(self):
        if os.path.isfile("changelog.diff"):
            os.unlink("changelog.diff")

        with open("ngeo.diff", "w") as diff_file:
            check_call([
                "git", "diff", "--",
                "CONST_create_template/{}/templates".format(self.project["project_package"]),
                "CONST_create_template/{}/static-ngeo".format(self.project["project_package"]),
            ], stdout=diff_file)

        if os.path.getsize("ngeo.diff") == 0:
            self.step3()
        else:
            print("")
            print(self.color_bar)
            print(
                "Apply the ngeo application diff available in the `ngeo.diff` file."
            )
            self.print_step(3)

    def step3(self):
        if os.path.isfile("ngeo.diff"):
            os.unlink("ngeo.diff")

        status = check_output(["git", "status", "--short", "CONST_create_template"])
        status = [s for s in status.split("\n") if len(s) > 3]
        status = [s[3:] for s in status if not s.startswith("?? ")]
        status = [s for s in status if not s.startswith(
            "CONST_create_template/{}/templates/".format(self.project["project_package"]),
        )]
        status = [s for s in status if not s.startswith(
            "CONST_create_template/{}/static-ngeo/".format(self.project["project_package"]),
        )]
        matcher = re.compile(r"CONST_create_tremplate.*/CONST_.+")
        status = [s for s in status if not matcher.match(s)]
        status = [s for s in status if s != "CONST_create_template/package.json"]

        with open("create.diff", "w") as diff_file:
            check_call(["git", "diff", "--"] + status, stdout=diff_file)

        if os.path.getsize("create.diff") == 0:
            self.step4()
        else:
            print("")
            print(self.color_bar)
            print(
                "This is an optional step but it helps to have a standard project.\n"
                "In the `create.diff` file, there are all the changes made in the create template "
                "that are recommended to be reported in your project.\n"
                "An advise to be more effective: In most of the casees it concerned a file that "
                "you never customise, or a file that you customised in depth, then respectively "
                "copy the new file from CONST_create_template, respectively ignore the changes."
            )
            self.print_step(4)

    def step4(self):
        if self.options.file is None:
            print("")
            print(self.color_bar)
            print("The makefile is missing")
            print("")
            self.print_step(4, intro="Fix it and run again the step 4:")
            exit(1)

        if os.path.isfile("create.diff"):
            os.unlink("create.diff")

        check_call(["make", "-f", self.options.file, "build"])

        command.upgrade(Config("alembic.ini"), "head")
        command.upgrade(Config("alembic_static.ini"), "head")

        if not self.options.windows:
            check_call(self.project.get("cmds", {}).get(
                "apache_graceful",
                ["sudo", "/usr/sbin/apache2ctl", "graceful"]
            ))

        print("")
        print(self.color_bar)
        print("The upgrade is nearly done, now you should:")
        print("- Test your application.")

        if self.options.windows:
            print("You are running on Windows, please restart your Apache server,")
            print("because we can not do that automatically.")

        self.print_step(5)

    def step5(self):
        if not self.test_checkers():
            print("")
            self.print_step(5, intro="Correct the checker, the step 5 again:")
            exit(1)

        # Required to remove from the Git stage the ignored file when we lunch the step again
        check_call(["git", "reset", "--mixed"])

        check_call(["git", "add", "-A"])
        check_call(["git", "status"])

        print("")
        print(self.color_bar)
        print("We will commit all the above files!")
        print(
            "If there are some files which should not be commited, then you should "
            "add them into the `.gitignore` file and launch step 5 again."
        )

        self.print_step(6, intro="Then to commit your changes type:")

    def step6(self):
        check_call(["git", "commit", "-m", "Upgrade to GeoMapFish {}".format(
            pkg_resources.get_distribution("c2cgeoportal").version
        )])

        print("")
        print(self.color_bar)
        print("")
        print(colorize("Congratulations your upgrade is a success.", GREEN))
        print("")
        branch = check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).strip()
        print("Now all your files will be commited, you should do a git push %s %s." % (
            self.options.git_remote, branch
        ))

    def deploy(self):
        if not self.test_checkers():
            print(colorize("Correct them and run again", RED))
            exit(1)

        check_call(["sudo", "-u", "deploy", "deploy", "-r", "deploy/deploy.cfg", self.options.host])
        check_call(["make", "-f", self.options.file, "build"])
Esempio n. 37
0
    def step1(self):
        if self.options.version is None:
            print("")
            print(self.color_bar)
            print("The VERSION environment variable is required for this upgrade step")
            exit(1)

        check_call(["git", "reset", "--hard"])
        check_call(["git", "clean", "--force", "-d"])
        check_call(["git", "submodule", "foreach", "--recursive", "git", "reset", "--hard"])
        check_call(["git", "submodule", "foreach", "--recursive", "git", "clean", "--force", "-d"])

        branch = check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).strip()
        # remove all no more existing branches
        check_call(["git", "fetch", "origin", "--prune"])
        branches = check_output(["git", "branch", "--all"]).split("\n")
        if "  remotes/origin/%s" % branch in branches:
            try:
                check_call(["git", "pull", "--rebase", self.options.git_remote, branch])
            except CalledProcessError:
                print(self.color_bar)
                print("")
                print(colorize("The pull (rebase) failed.", RED))
                print("")
                self.print_step(1, intro="Please solve the rebase and run the step 1 again:")
                exit(1)

        check_call(["git", "submodule", "sync"])
        check_call(["git", "submodule", "update", "--init"])
        check_call(["git", "submodule", "foreach", "git", "submodule", "sync"])
        check_call(["git", "submodule", "foreach", "git", "submodule", "update", "--init"])

        if len(check_output(["git", "status", "-z"]).strip()) != 0:
            print(self.color_bar)
            print("")
            print(colorize("The pull isn't fast forward.", RED))
            print("")
            self.print_step(1, intro="Please solve the rebase and run the step 1 again:")
            exit(1)

        check_call(["git", "submodule", "foreach", "git", "submodule", "sync"])
        check_call(["git", "submodule", "foreach", "git", "submodule", "update", "--init"])

        if not self.options.windows:
            check_call(["make", "-f", self.options.file, ".build/requirements.timestamp"])
            pip_cmd = ["%s/pip" % self.venv_bin, "install"]
            if self.options.version == "master":
                check_call(["%s/pip" % self.venv_bin, "uninstall", "--yes", "c2cgeoportal"])
                pip_cmd += ["--pre", "c2cgeoportal"]
            else:
                pip_cmd += ["c2cgeoportal==%s" % (self.options.version)]
            check_call(pip_cmd)

        check_call([
            "%s/pcreate" % self.venv_bin, "--ignore-conflicting-name", "--overwrite",
            "--scaffold=c2cgeoportal_update", "../%s" % self.project["project_folder"]
        ])
        check_call(["make", "-f", self.options.file, self.options.clean])

        with open("changelog.diff", "w") as diff_file:
            check_call(["git", "diff", "--", "CONST_CHANGELOG.txt"], stdout=diff_file)

        check_call(["make", "-f", self.options.file, "update-node-modules"])
        check_call(["make", "-f", self.options.file, ".build/requirements.timestamp"])

        if os.path.getsize("changelog.diff") == 0:
            self.step2()
        else:
            print("")
            print(self.color_bar)
            print(
                "Apply the manual migration steps based on what is in the CONST_CHANGELOG.txt file"
                " (listed in the `changelog.diff` file)."
            )
            self.print_step(2)
Esempio n. 38
0
    def step0(self):
        project_template_keys = self.project.get("template_vars").keys()
        messages = []
        for required in REQUIRED_TEMPLATE_KEYS:
            if required not in project_template_keys:
                messages.append(
                    "The element '{}' is missing in the 'template_vars' of "
                    "the file 'project.yaml.mako', you should for example: {}: {}.".format(
                        required, required, TEMPLATE_EXAMPLE.get('required', '')
                    )
                )
        if len(messages) > 0:
            print("")
            print(self.color_bar)
            print(colorize("\n".join(messages), RED))
            print("")
            self.print_step(0, intro="Fix it and run again the upgrade:")
            exit(1)

        if self.options.version is None:
            print("")
            print(self.color_bar)
            print("The VERSION environment variable is required for this upgrade step")
            exit(1)

        if re.match(VERSION_RE, self.options.version) is not None:
            http = httplib2.Http()
            url = (
                "http://raw.github.com/camptocamp/c2cgeoportal/%s/"
                "c2cgeoportal/scaffolds/update/CONST_versions_requirements.txt" %
                self.options.version
            )
            headers, _ = http.request(url)
            if headers.status != 200:
                print("")
                print(self.color_bar)
                print(
                    "Failed downloading the c2cgeoportal "
                    "CONST_versions_requirements.txt file from URL:"
                )
                print(url)
                print("The upgrade is impossible")
                exit(1)

        if os.path.split(os.path.realpath("."))[1] != self.project["project_folder"]:
            print("")
            print(self.color_bar)
            print("Your project isn't in the right folder!")
            print("It should be in folder '%s' instead of folder '%s'." % (
                self.project["project_folder"], os.path.split(os.path.realpath("."))[1]
            ))
            print("")
            self.print_step(0, intro="Fix it and lunch again the upgrade:")
            exit(1)

        if check_output(["git", "status", "--short"]) == "":
            self.step1()
        else:
            check_call(["git", "status"])
            print("")
            print(self.color_bar)
            print(
                "Here is the output of 'git status'. Please make sure to commit all your changes "
                "before going further. All uncommited changes will be lost."
            )
            self.print_step(1)
Esempio n. 39
0
 def print_step(self, step, intro="To continue type:"):
     print(colorize(intro, YELLOW))
     print(colorize("make -f {} upgrade{}", GREEN).format(
         self.options.file if self.options.file is not None else "<user.mk>",
         step if step != 0 else "",
     ))
Esempio n. 40
0
    def step1(self):
        if self.options.version is None:
            print("")
            print(self.color_bar)
            print("The VERSION environment variable is required for this upgrade step")
            exit(1)

        check_call(["git", "reset", "--hard"])
        check_call(["git", "clean", "--force", "-d"])
        check_call(["git", "submodule", "foreach", "--recursive", "git", "reset", "--hard"])
        check_call(["git", "submodule", "foreach", "--recursive", "git", "clean", "--force", "-d"])

        branch = check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).strip()
        # remove all no more existing branches
        check_call(["git", "fetch", "origin", "--prune"])
        branches = check_output(["git", "branch", "--all"]).split("\n")
        if "  remotes/origin/{0!s}".format(branch) in branches:
            try:
                check_call(["git", "pull", "--rebase", self.options.git_remote, branch])
            except CalledProcessError:
                print(self.color_bar)
                print("")
                print(colorize("The pull (rebase) failed.", RED))
                print("")
                self.print_step(1, intro="Please solve the rebase and run the step 1 again:")
                exit(1)

        check_call(["git", "submodule", "sync"])
        check_call(["git", "submodule", "update", "--init"])
        check_call(["git", "submodule", "foreach", "git", "submodule", "sync"])
        check_call(["git", "submodule", "foreach", "git", "submodule", "update", "--init"])

        if len(check_output(["git", "status", "-z"]).strip()) != 0:
            print(self.color_bar)
            print("")
            print(colorize("The pull is not fast forward.", RED))
            print("")
            self.print_step(1, intro="Please solve the rebase and run the step 1 again:")
            exit(1)

        check_call(["git", "submodule", "foreach", "git", "submodule", "sync"])
        check_call(["git", "submodule", "foreach", "git", "submodule", "update", "--init"])

        if not self.options.windows:
            check_call(["make", "-f", self.options.file, ".build/requirements.timestamp"])
            pip_cmd = ["{0!s}/pip".format(self.venv_bin), "install"]
            if self.options.version == "master":
                check_call(["{0!s}/pip".format(self.venv_bin), "uninstall", "--yes", "c2cgeoportal"])
                pip_cmd += ["--pre", "c2cgeoportal"]
            else:
                pip_cmd += ["c2cgeoportal=={0!s}".format((self.options.version))]
            check_call(pip_cmd)

        check_call([
            "{0!s}/pcreate".format(self.venv_bin), "--ignore-conflicting-name", "--overwrite",
            "--scaffold=c2cgeoportal_update", "../{0!s}".format(self.project["project_folder"])
        ])
        check_call(["make", "-f", self.options.file, self.options.clean])

        # Update the package.json file
        if os.path.exists("package.json") and os.path.getsize("package.json") > 0:
            with open("package.json", "r") as package_json_file:
                package_json = json.loads(package_json_file.read(), encoding="utf-8")
            with open("CONST_create_template/package.json", "r") as package_json_file:
                template_package_json = json.loads(package_json_file.read(), encoding="utf-8")
            if "devDependencies" not in package_json:
                package_json["devDependencies"] = {}
            for package, version in template_package_json.get("devDependencies", {}).items():
                package_json["devDependencies"][package] = version
            with open("package.json", "w") as package_json_file:
                json.dump(
                    package_json, package_json_file,
                    encoding="utf-8", sort_keys=True, separators=(',', ': '), indent=2
                )
                package_json_file.write("\n")
        else:
            shutil.copyfile("CONST_create_template/package.json", "package.json")

        with open("changelog.diff", "w") as diff_file:
            check_call(["git", "diff", "--", "CONST_CHANGELOG.txt"], stdout=diff_file)

        check_call(["make", "-f", self.options.file, "update-node-modules"])
        check_call(["make", "-f", self.options.file, ".build/requirements.timestamp"])

        if os.path.getsize("changelog.diff") == 0:
            self.step2()
        else:
            print("")
            print(self.color_bar)
            print(
                "Apply the manual migration steps based on what is in the CONST_CHANGELOG.txt file"
                " (listed in the `changelog.diff` file)."
            )
            self.print_step(2)
Esempio n. 41
0
class C2cUpgradeTool:

    color_bar = colorize(
        "================================================================",
        GREEN)
    project = None

    def __init__(self, options):
        self.options = options
        self.project = self.get_project()

    @staticmethod
    def get_project():
        if not os.path.isfile("project.yaml"):
            print(
                colorize("Unable to find the required 'project.yaml' file.",
                         RED))
            exit(1)

        with open("project.yaml", "r") as f:
            return yaml.safe_load(f)

    @staticmethod
    def get_upgrade(section):
        if not os.path.isfile(".upgrade.yaml"):
            print(
                colorize("Unable to find the required '.upgrade.yaml' file.",
                         RED))
            exit(1)

        with open(".upgrade.yaml", "r") as f:
            return yaml.safe_load(f)[section]

    def print_step(self,
                   step,
                   error=False,
                   message=None,
                   prompt="To continue type:"):
        print("")
        print(self.color_bar)
        if message is not None:
            print(colorize(message, RED if error else YELLOW))
        if step >= 0:
            print(colorize(prompt, GREEN))
            args = " --makefile=" + self.options.new_makefile if self.options.makefile != "Makefile" else ""
            print(
                colorize("./docker-run make{} upgrade{}", GREEN).format(
                    args,
                    step if step != 0 else "",
                ))

    def run_step(self, step):
        getattr(self, "step{}".format(step))()

    def test_checkers(self):
        http = httplib2.Http()
        for check_type in ("", "type=all"):
            resp, _ = http.request("http://localhost{}{}".format(
                self.project["checker_path"], check_type),
                                   method="GET",
                                   headers={"Host": self.project["host"]})
            if resp.status < 200 or resp.status >= 300:
                return False, "\n".join([
                    "Checker error:",
                    "Open `http://{}{}{}` for more informations."
                ]).format(self.project["host"], self.project["checker_path"],
                          check_type)

        return True, None

    def upgrade(self):
        self.run_step(self.options.step)

    @Step(0)
    def step0(self, step):
        project_template_keys = list(self.project.get("template_vars").keys())
        messages = []
        for required in REQUIRED_TEMPLATE_KEYS:
            if required not in project_template_keys:
                messages.append(
                    "The element '{}' is missing in the 'template_vars' of "
                    "the file 'project.yaml.mako', you should for example: {}: {}."
                    .format(required, required,
                            TEMPLATE_EXAMPLE.get('required', '')))
        if len(messages) > 0:
            self.print_step(step,
                            error=True,
                            message="\n".join(messages),
                            prompt="Fix it and run again the upgrade:")
            exit(1)

        if check_output(["git", "status", "--short"]).decode("utf-8") == "":
            self.run_step(step + 1)
        else:
            check_call(["git", "status"])
            self.print_step(
                step + 1,
                message=
                "Here is the output of 'git status'. Please make sure to commit all your "
                "changes before going further. All uncommited changes will be lost."
            )

    @Step(1)
    def step1(self, step):
        check_call(["git", "reset", "--hard"])
        check_call(["git", "clean", "--force", "-d"])
        check_call([
            "git", "submodule", "foreach", "--recursive", "git", "reset",
            "--hard"
        ])
        check_call([
            "git", "submodule", "foreach", "--recursive", "git", "clean",
            "--force", "-d"
        ])

        branch = check_output(["git", "rev-parse", "--abbrev-ref",
                               "HEAD"]).decode("utf-8").strip()
        # remove all no more existing branches
        check_call(["git", "fetch", "origin", "--prune"])
        branches = check_output(["git", "branch",
                                 "--all"]).decode("utf-8").split("\n")
        if "  remotes/origin/{0!s}".format(branch) in branches:
            try:
                check_call([
                    "git", "pull", "--rebase", self.options.git_remote, branch
                ])
            except subprocess.CalledProcessError:
                self.print_step(
                    step,
                    error=True,
                    message="The pull (rebase) failed.",
                    prompt="Please solve the rebase and run it again:")
                exit(1)

        check_call(["git", "submodule", "sync"])
        check_call(["git", "submodule", "update", "--init"])
        check_call(["git", "submodule", "foreach", "git", "submodule", "sync"])
        check_call([
            "git", "submodule", "foreach", "git", "submodule", "update",
            "--init"
        ])

        if len(check_output(["git", "status", "-z"
                             ]).decode("utf-8").strip()) != 0:
            self.print_step(step,
                            error=True,
                            message="The pull is not fast forward.",
                            prompt="Please solve the rebase and run it again:")
            exit(1)

        check_call(["git", "submodule", "foreach", "git", "submodule", "sync"])
        check_call([
            "git", "submodule", "foreach", "git", "submodule", "update",
            "--init"
        ])

        if os.path.exists("CONST_create_template"):
            shutil.rmtree("CONST_create_template")

        project_path = os.path.join("/tmp", self.project["project_folder"])
        check_call(["ln", "-s", "/src", project_path])
        check_call([
            "pcreate", "--ignore-conflicting-name", "--overwrite",
            "--scaffold=c2cgeoportal_update", project_path
        ])
        if self.options.nondocker:
            check_call([
                "pcreate", "--ignore-conflicting-name", "--overwrite",
                "--scaffold=c2cgeoportal_nondockerupdate", project_path
            ])
        os.remove(project_path)

        check_call(
            ["make", "--makefile=" + self.options.makefile, "clean-all"])
        self.options.makefile = self.options.new_makefile

        # Update the package.json file
        if os.path.exists(
                "package.json") and os.path.getsize("package.json") > 0:
            with open("package.json", "r") as package_json_file:
                package_json = json.loads(package_json_file.read(),
                                          encoding="utf-8")
            with open("CONST_create_template/package.json",
                      "r") as package_json_file:
                template_package_json = json.loads(package_json_file.read(),
                                                   encoding="utf-8")
            if "devDependencies" not in package_json:
                package_json["devDependencies"] = {}
            for package, version in list(
                    template_package_json.get("devDependencies", {}).items()):
                package_json["devDependencies"][package] = version
            with open("package.json", "w") as package_json_file:
                json.dump(package_json,
                          package_json_file,
                          sort_keys=True,
                          separators=(',', ': '),
                          indent=2)
                package_json_file.write("\n")
        else:
            shutil.copyfile("CONST_create_template/package.json",
                            "package.json")
        self.run_step(step + 1)

    @Step(2)
    def step2(self, step):
        error = False

        print("Files to remove")
        error |= self.files_to_remove(pre=True)
        print("Files to move")
        error |= self.files_to_move(pre=True)
        print("Files to get")
        error |= self.files_to_get(pre=True)

        if error:
            self.print_step(
                step,
                error=True,
                message=
                "There was some error on your project configuration, see above",
                prompt="Fix it and run it again:")
            exit(1)
        else:
            if "managed_files" not in self.project:
                self.print_step(
                    step,
                    message=
                    "In the new version we will also manage almost all the create "
                    "template files.\n"
                    "By default following regex pattern will not be replaced:\n{}"
                    "Than you should fill the 'managed_files' in you 'project.yaml' file with at least "
                    "`[]`.".format("\n".join([
                        "- " + e
                        for e in self.project.get('unmanaged_files', [])
                    ])),
                    prompt="Fill it and run it again:")
            else:
                self.run_step(step + 1)

    @Step(3)
    def step3(self, step):
        self.files_to_remove()
        self.run_step(step + 1)

    def files_to_remove(self, pre=False):
        error = False
        for element in self.get_upgrade("files_to_remove"):
            file_ = element["file"]
            if os.path.exists(file_):
                managed = False
                for pattern in self.project["managed_files"]:
                    if re.match(pattern + '$', file_):
                        print(
                            colorize(
                                "The file '{}' is no more use but not delete "
                                "because he is in the managed_files as '{}'.".
                                format(file_, pattern), RED))
                        error = True
                        managed = True
                if not managed and not pre:
                    print(
                        colorize("The file '{}' is removed.".format(file_),
                                 GREEN))
                    if "version" in element and "from" in element:
                        print(
                            "Was used in version {}, to be removed from version {}."
                            .format(element["from"], element["version"]))
                    if os.path.isdir(file_):
                        shutil.rmtree(file_)
                    else:
                        os.remove(file_)
        return error

    @Step(4)
    def step4(self, step):
        self.files_to_move()
        self.run_step(step + 1)

    def files_to_move(self, pre=False):
        error = False
        for element in self.get_upgrade("files_to_move"):
            src = element["from"]
            dst = element["to"]
            if os.path.exists(src):
                type_ = "directory" if os.path.isdir(src) else "file"
                managed = False
                for pattern in self.project["managed_files"]:
                    if re.match(pattern + '$', src):
                        print(
                            colorize(
                                "The {} '{}' is present in the managed_files as '{}' but he will move."
                                .format(type_, src, pattern), RED))
                        error = True
                        managed = True
                        break
                if not managed and os.path.exists(dst):
                    print(
                        colorize(
                            "The destination '{}' already exists.".format(dst),
                            RED))
                    error = True
                    if not pre:
                        raise InteruptedException(
                            "The destination '{}' already exists.".format(dst))
                if not managed and not pre:
                    print(
                        colorize(
                            "Move the {} '{}' to '{}'.".format(
                                type_, src, dst), GREEN))
                    if "version" in element:
                        print("Needed from version {}.".format(
                            element["version"]))
                    os.rename(src, dst)
        return error

    @Step(5)
    def step5(self, step):
        self.files_to_get()
        self.run_step(step + 1)

    def files_to_get(self, pre=False):
        error = False
        for root, _, files in os.walk("CONST_create_template"):
            root = root[len("CONST_create_template/"):]
            for file_ in files:
                destination = os.path.join(root, file_)
                managed = False
                for pattern in self.project["managed_files"] + self.get_upgrade(
                        "default_project_file"):
                    if re.match(pattern + '$', destination):
                        managed = True
                        for unpattern in self.project.get(
                                "unmanaged_files", []):
                            if re.match(unpattern + '$', destination):
                                managed = False
                                break
                        break
                source = os.path.join("CONST_create_template", destination)
                if not managed and (not os.path.exists(destination)
                                    or not filecmp.cmp(source, destination)):
                    if not pre:
                        print(
                            colorize(
                                "Get the file '{}' from the create template.".
                                format(destination), GREEN))
                        if os.path.dirname(destination) != "":
                            os.makedirs(os.path.dirname(destination),
                                        exist_ok=True)
                        shutil.copyfile(source, destination)
                        shutil.copymode(source, destination)
        return error

    @Step(6)
    def step6(self, step):
        with open("changelog.diff", "w") as diff_file:
            check_call(["git", "diff", "--", "CONST_CHANGELOG.txt"],
                       stdout=diff_file)

        from210 = False
        try:
            check_call(["grep", "--", "-Version 2.1.0", "changelog.diff"])
            from210 = True
        except subprocess.CalledProcessError:
            pass
        if from210:
            check_call(["cp", "CONST_CHANGELOG.txt", "changelog.diff"])

        if os.path.getsize("changelog.diff") == 0:
            self.run_step(step + 1)
        else:
            self.print_step(
                step + 1,
                message=
                "Apply the manual migration steps based on what is in the CONST_CHANGELOG.txt "
                "file (listed in the `changelog.diff` file).")

    @Step(7)
    def step7(self, step):
        if os.path.isfile("changelog.diff"):
            os.unlink("changelog.diff")

        status = check_output(
            ["git", "status", "--short",
             "CONST_nondocker_CHANGELOG.txt"]).decode("utf-8")
        if status.strip() == "?? CONST_nondocker_CHANGELOG.txt":
            check_call([
                "cp", "CONST_nondocker_CHANGELOG.txt",
                "nondocker-changelog.diff"
            ])
        else:
            with open("nondocker-changelog.diff", "w") as diff_file:
                check_call(
                    ["git", "diff", "--", "CONST_nondocker_CHANGELOG.txt"],
                    stdout=diff_file)

        if os.path.getsize("nondocker-changelog.diff") == 0:
            self.run_step(step + 1)
        else:
            self.print_step(
                step + 1,
                message=
                "Apply the manual migration steps based on what is in the "
                "CONST_nondocker_CHANGELOG.txt file (listed in the `nondocker-changelog.diff` file)."
            )

    @Step(8)
    def step8(self, step):
        if os.path.isfile("nondocker-changelog.diff"):
            os.unlink("nondocker-changelog.diff")

        with open("ngeo.diff", "w") as diff_file:
            check_call([
                "git",
                "diff",
                "--",
                "CONST_create_template/{}/templates".format(
                    self.project["project_package"]),
                "CONST_create_template/{}/static-ngeo".format(
                    self.project["project_package"]),
            ],
                       stdout=diff_file)

        if os.path.getsize("ngeo.diff") == 0:
            self.run_step(step + 1)
        else:
            self.print_step(
                step + 1,
                message=
                "Manually apply the ngeo application changes as shown in the `ngeo.diff` file.\n"
                + DIFF_NOTICE)

    @Step(9)
    def step9(self, step):
        if os.path.isfile("ngeo.diff"):
            os.unlink("ngeo.diff")

        status = check_output(
            ["git", "status", "--short",
             "CONST_create_template"]).decode("utf-8")
        status = [s for s in status.split("\n") if len(s) > 3]
        status = [s[3:] for s in status if s.startswith(" M ")]
        status = [
            s for s in status if not s.startswith(
                "CONST_create_template/{}/templates/".format(
                    self.project["project_package"]), )
        ]
        status = [
            s for s in status if not s.startswith(
                "CONST_create_template/{}/static-ngeo/".format(
                    self.project["project_package"]), )
        ]
        matcher = re.compile(r"CONST_create_tremplate.*/CONST_.+")
        status = [s for s in status if not matcher.match(s)]
        status = [
            s for s in status if s != "CONST_create_template/package.json"
        ]
        status = [
            s for s in status
            if not filecmp.cmp(s, s[len("CONST_create_template/"):])
        ]

        if len(status) > 0:
            with open("create.diff", "w") as diff_file:
                check_call(["git", "diff", "--"] + status, stdout=diff_file)

            if os.path.getsize("create.diff") == 0:
                self.run_step(step + 1)
            else:
                self.print_step(
                    step + 1,
                    message=
                    "This is an optional step but it helps to have a standard project.\n"
                    "The `create.diff` file is a recommendation of the changes that you should apply "
                    "to your project.\n" + DIFF_NOTICE)
        else:
            self.run_step(step + 1)

    @Step(10)
    def step10(self, step):
        if os.path.isfile("create.diff"):
            os.unlink("create.diff")

        os.environ["IGNORE_I18N_ERRORS"] = "TRUE"
        check_call(
            ["make", "--makefile=" + self.options.new_makefile, "build"])
        del os.environ["IGNORE_I18N_ERRORS"]
        check_call([
            "git", "checkout", "{0}/locale/*/LC_MESSAGES/{0}-client.po".format(
                self.project["project_package"])
        ])

        if self.options.nondocker:
            command.upgrade(Config("alembic.ini"), "head")
            command.upgrade(Config("alembic_static.ini"), "head")

            message = [
                "The upgrade is nearly done, now you should:",
                "- Test your application."
            ]
        else:
            message = [
                "The upgrade is nearly done, now you should:",
                "- run `docker-compose up`",
                "- Test your application on 'http://localhost:8480/desktop'."
            ]

        if self.options.windows:
            message.append(
                "You are running on Windows, please restart your Apache server,"
                "because we can not do that automatically.")

        if os.path.isfile(".upgrade.yaml"):
            os.unlink(".upgrade.yaml")
        with open(".UPGRADE_SUCCESS", "w"):
            pass
        self.print_step(step + 1, message="\n".join(message))

    @Step(11)
    def step11(self, step):
        if os.path.isfile(".UPGRADE_SUCCESS"):
            os.unlink(".UPGRADE_SUCCESS")
        ok, message = self.test_checkers()
        if not ok:
            self.print_step(step,
                            error=True,
                            message=message,
                            prompt="Correct the checker, the it again:")
            exit(1)

        # Required to remove from the Git stage the ignored file when we lunch the step again
        check_call(["git", "reset", "--mixed"])

        check_call(["git", "add", "-A"])
        check_call(["git", "status"])

        self.print_step(
            step + 1,
            message="We will commit all the above files!\n"
            "If there are some files which should not be committed, then you should "
            "add them into the `.gitignore` file and launch upgrade5 again.",
            prompt="Then to commit your changes type:")

    @Step(12)
    def step12(self, _):
        check_call([
            "git", "commit", "-m", "Upgrade to GeoMapFish {}".format(
                pkg_resources.get_distribution("c2cgeoportal").version)
        ])

        print("")
        print(self.color_bar)
        print("")
        print(colorize("Congratulations your upgrade is a success.", GREEN))
        print("")
        branch = check_output(["git", "rev-parse", "--abbrev-ref",
                               "HEAD"]).decode("utf-8").strip()
        print(
            "Now all your files will be committed, you should do a git push:")
        print("git push {0!s} {1!s}.".format(self.options.git_remote, branch))
Esempio n. 42
0
 def print_step(self, step, intro="To continue type:"):
     print(colorize(intro, YELLOW))
     print(colorize("make -f {} upgrade{}", GREEN).format(
         self.options.file if self.options.file is not None else "<user.mk>",
         step if step != 0 else "",
     ))
Esempio n. 43
0
    def step1(self):
        if self.options.version is None:
            print("")
            print(self.color_bar)
            print("The VERSION environment variable is required for this upgrade step")
            exit(1)

        check_call(["git", "reset", "--hard"])
        check_call(["git", "clean", "--force", "-d"])
        check_call(["git", "submodule", "foreach", "--recursive", "git", "reset", "--hard"])
        check_call(["git", "submodule", "foreach", "--recursive", "git", "clean", "--force", "-d"])

        branch = check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).strip()
        # remove all no more existing branches
        check_call(["git", "fetch", "origin", "--prune"])
        branches = check_output(["git", "branch", "--all"]).split("\n")
        if "  remotes/origin/%s" % branch in branches:
            try:
                check_call(["git", "pull", "--rebase", self.options.git_remote, branch])
            except CalledProcessError:
                print(self.color_bar)
                print("")
                print(colorize("The pull (rebase) failed.", RED))
                print("")
                self.print_step(1, intro="Please solve the rebase and run the step 1 again:")
                exit(1)

        check_call(["git", "submodule", "sync"])
        check_call(["git", "submodule", "update", "--init"])
        check_call(["git", "submodule", "foreach", "git", "submodule", "sync"])
        check_call(["git", "submodule", "foreach", "git", "submodule", "update", "--init"])

        if len(check_output(["git", "status", "-z"]).strip()) != 0:
            print(self.color_bar)
            print("")
            print(colorize("The pull isn't fast forward.", RED))
            print("")
            self.print_step(1, intro="Please solve the rebase and run the step 1 again:")
            exit(1)

        check_call(["git", "submodule", "foreach", "git", "submodule", "sync"])
        check_call(["git", "submodule", "foreach", "git", "submodule", "update", "--init"])

        if not self.options.windows:
            check_call(["make", "-f", self.options.file, ".build/requirements.timestamp"])
            pip_cmd = ["%s/pip" % self.venv_bin, "install"]
            if self.options.version == "master":
                check_call(["%s/pip" % self.venv_bin, "uninstall", "--yes", "c2cgeoportal"])
                pip_cmd += ["--pre", "c2cgeoportal"]
            else:
                pip_cmd += ["c2cgeoportal==%s" % (self.options.version)]
            check_call(pip_cmd)

        check_call([
            "%s/pcreate" % self.venv_bin, "--ignore-conflicting-name", "--overwrite",
            "--scaffold=c2cgeoportal_update", "../%s" % self.project["project_folder"]
        ])
        check_call(["make", "-f", self.options.file, self.options.clean])

        with open("changelog.diff", "w") as diff_file:
            check_call(["git", "diff", "--", "CONST_CHANGELOG.txt"], stdout=diff_file)

        check_call(["make", "-f", self.options.file, "update-node-modules"])
        check_call(["make", "-f", self.options.file, ".build/requirements.timestamp"])

        if os.path.getsize("changelog.diff") == 0:
            self.step2()
        else:
            print("")
            print(self.color_bar)
            print(
                "Apply the manual migration steps based on what is in the CONST_CHANGELOG.txt file"
                " (listed in the `changelog.diff` file)."
            )
            self.print_step(2)