def files_to_remove(self, pre=False): error = False for element in self.get_upgrade("files_to_remove"): file_ = element["file"] if os.path.exists(file_): managed = False for pattern in self.project["managed_files"]: if re.match(pattern + '$', file_): print( colorize( "The file '{}' is no more use but not delete " "because he is in the managed_files as '{}'.". format(file_, pattern), RED)) error = True managed = True if not managed and not pre: print( colorize("The file '{}' is removed.".format(file_), GREEN)) if "version" in element and "from" in element: print( "Was used in version {}, to be removed from version {}." .format(element["from"], element["version"])) if os.path.isdir(file_): shutil.rmtree(file_) else: os.remove(file_) return error
def __call__(self, filename, options): messages = [] self.env = bootstrap(filename) with open("project.yaml") as f: self.package = yaml.load(f) self.config = get_config(".build/config.yaml") try: from c2cgeoportal.models import DBSession, Theme, LayerGroup, \ LayerWMS, LayerWMTS, FullTextSearch self._import(Theme, messages) self._import(LayerGroup, messages) self._import(LayerWMS, messages, self._import_layer_wms) self._import(LayerWMTS, messages, self._import_layer_wmts) except ProgrammingError as e: print(colorize( "ERROR: The database is probably not up to date " "(should be ignored when happen during the upgrade)", RED )) print(colorize(e, RED)) for ln, in DBSession.query(FullTextSearch.layer_name).distinct().all(): if ln is not None and ln != "": messages.append(Message( None, ln, None, [], u"", u"", ("fts", ln.encode("ascii", errors="replace")) )) return messages
def __call__(self, filename, options): message_str = subprocess.check_output(["node", "tools/extract-messages.js", filename]) try: messages = loads(message_str) return [Message( None, message, None, [], u"", u"", context.split(":") ) for context, message in messages] except: print(colorize("An error occurred", RED)) print(colorize(message_str, RED)) print("------") raise
def files_to_get(self, pre=False): error = False for root, _, files in os.walk("CONST_create_template"): root = root[len("CONST_create_template/"):] for file_ in files: destination = os.path.join(root, file_) managed = False for pattern in self.project["managed_files"] + self.get_upgrade( "default_project_file"): if re.match(pattern + '$', destination): managed = True for unpattern in self.project.get( "unmanaged_files", []): if re.match(unpattern + '$', destination): managed = False break break source = os.path.join("CONST_create_template", destination) if not managed and (not os.path.exists(destination) or not filecmp.cmp(source, destination)): if not pre: print( colorize( "Get the file '{}' from the create template.". format(destination), GREEN)) if os.path.dirname(destination) != "": os.makedirs(os.path.dirname(destination), exist_ok=True) shutil.copyfile(source, destination) shutil.copymode(source, destination) return error
def _import_layer_wms(self, layer, messages): server = layer.ogc_server url = server.url_wfs or server.url if url is None: return for wms_layer in layer.layer.split(","): self._import_layer_attributes( url, wms_layer, layer.item_type, layer.name, layer.id, messages ) if layer.geo_table is not None: exclude = [] if layer.exclude_properties is None else layer.exclude_properties.split(",") last_update_date = layer.get_metadatas("lastUpdateDateColumn") if len(last_update_date) == 1: exclude.append(last_update_date[0].value) last_update_user = layer.get_metadatas("lastUpdateUserColumn") if len(last_update_user) == 1: exclude.append(last_update_user[0].value) try: cls = get_class(layer.geo_table, exclude_properties=exclude) for column_property in class_mapper(cls).iterate_properties: if isinstance(column_property, ColumnProperty) and \ len(column_property.columns) == 1 and \ not column_property.columns[0].primary_key and \ not column_property.columns[0].foreign_keys and \ not isinstance(column_property.columns[0].type, Geometry): messages.append(Message( None, column_property.key, None, [], "", "", (".".join(["edit", layer.item_type, str(layer.id)]), layer.name) )) except NoSuchTableError: exit(colorize("No such table '{}' for layer '{}'.".format(layer.geo_table, layer.name), RED))
def __call__(self, filename, options): message_str = subprocess.check_output( ["node", "tools/extract-messages.js", filename]) try: messages = [] for contexts, message in loads(message_str): for context in contexts.split(", "): messages.append( Message(None, message, None, [], u"", u"", context.split(":"))) return messages except: print(colorize("An error occurred", RED)) print(colorize(message_str, RED)) print("------") raise
def deploy(self): if not self.test_checkers(): print(colorize("Correct them and run again", RED)) exit(1) check_call(["sudo", "-u", "deploy", "deploy", "-r", "deploy/deploy.cfg", self.options.host]) check_call(["make", "-f", self.options.file, "build"])
def print_step(self, step, error=False, message=None, prompt="To continue type:"): print("") print(self.color_bar) if message is not None: print(colorize(message, RED if error else YELLOW)) if step >= 0: print(colorize(prompt, GREEN)) args = " --makefile=" + self.options.new_makefile if self.options.makefile != "Makefile" else "" print( colorize("./docker-run make{} upgrade{}", GREEN).format( args, step if step != 0 else "", ))
def get_upgrade(section): if not os.path.isfile(".upgrade.yaml"): print( colorize("Unable to find the required '.upgrade.yaml' file.", RED)) exit(1) with open(".upgrade.yaml", "r") as f: return yaml.safe_load(f)[section]
def get_project(): if not os.path.isfile("project.yaml"): print( colorize("Unable to find the required 'project.yaml' file.", RED)) exit(1) with open("project.yaml", "r") as f: return yaml.safe_load(f)
def step0(self): project_template_keys = self.project.get("template_vars").keys() messages = [] for required in REQUIRED_TEMPLATE_KEYS: if required not in project_template_keys: messages.append( "The element '{}' is missing in the 'template_vars' of " "the file 'project.yaml.mako', you should for example: {}: {}.".format( required, required, TEMPLATE_EXAMPLE.get('required', '') ) ) if len(messages) > 0: print("") print(self.color_bar) print(colorize("\n".join(messages), RED)) print("") self.print_step(0, intro="Fix it and run again the upgrade:") exit(1) if re.match(VERSION_RE, self.options.version) is not None: http = httplib2.Http() url = ( "http://raw.github.com/camptocamp/c2cgeoportal/%s/" "c2cgeoportal/scaffolds/update/CONST_versions_requirements.txt" % self.options.version ) headers, _ = http.request(url) if headers.status != 200: print("") print(self.color_bar) print( "Failed downloading the c2cgeoportal " "CONST_versions_requirements.txt file from URL:" ) print(url) print("The upgrade is impossible") exit(1) if os.path.split(os.path.realpath("."))[1] != self.project["project_folder"]: print("") print(self.color_bar) print("Your project isn't in the right folder!") print("It should be in folder '%s' instead of folder '%s'." % ( self.project["project_folder"], os.path.split(os.path.realpath("."))[1] )) print("") self.print_step(0, intro="Fix it and lunch again the upgrade:") exit(1) check_call(["git", "status"]) print("") print(self.color_bar) print( "Here is the output of 'git status'. Please make sure to commit all your changes " "before going further. All uncommited changes will be lost." ) self.print_step(1)
def __call__(self, filename, options): messages = [] self.capabilities_cache = {} self.env = bootstrap(filename) try: from c2cgeoportal.models import Theme, LayerGroup, \ LayerWMS, LayerWMTS self._import(Theme, messages) self._import(LayerGroup, messages) self._import(LayerWMS, messages) self._import(LayerWMTS, messages) except ProgrammingError as e: print(colorize("ERROR: The database is probably not up to date", RED)) print(colorize(e, RED)) return messages
def __call__(self, filename, options): messages = [] self.env = bootstrap(filename) with open("project.yaml") as f: self.package = yaml.load(f) try: from c2cgeoportal.models import Theme, LayerGroup, LayerWMS, LayerWMTS self._import(Theme, messages) self._import(LayerGroup, messages) self._import(LayerWMS, messages, self._import_layer_wms) self._import(LayerWMTS, messages, self._import_layer_wmts) except ProgrammingError as e: print(colorize("ERROR: The database is probably not up to date", RED)) print(colorize(e, RED)) return messages
def post(self, command, output_dir, vars): if os.name == 'posix': dest = os.path.join(output_dir, ".whiskey/action_hooks/pre-build.mako") subprocess.check_call(["chmod", "+x", dest]) """ Overrides the base template class to print "Welcome to c2cgeoportal!" after a successful scaffolding rendering. """ self.out(colorize("\nWelcome to c2cgeoportal!", GREEN)) return BaseTemplate.post(self, command, output_dir, vars)
def step4(self): check_call(["git", "commit", "-m", "Upgrade to GeoMapFish %s" % self.options.version]) print("") print(self.color_bar) print("") print(colorize("Congratulations your upgrade is a success.", GREEN)) print("") branch = check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).strip() print("Now all your files will be commited, you should do a git push %s %s." % ( self.options.git_remote, branch ))
def _import_layer_wms(self, layer, messages): server = layer.ogc_server url = server.url_wfs or server.url if url is None: return for wms_layer in layer.layer.split(","): self._import_layer_attributes(url, wms_layer, layer.item_type, layer.name, messages) if layer.geo_table is not None and layer.geo_table != "": exclude = [] if layer.exclude_properties is None else layer.exclude_properties.split( ",") last_update_date = layer.get_metadatas("lastUpdateDateColumn") if len(last_update_date) == 1: exclude.append(last_update_date[0].value) last_update_user = layer.get_metadatas("lastUpdateUserColumn") if len(last_update_user) == 1: exclude.append(last_update_user[0].value) try: cls = get_class(layer.geo_table, exclude_properties=exclude) for column_property in class_mapper(cls).iterate_properties: if isinstance(column_property, ColumnProperty) and \ len(column_property.columns) == 1 and \ not column_property.columns[0].primary_key and \ not column_property.columns[0].foreign_keys and \ not isinstance(column_property.columns[0].type, Geometry): messages.append( Message( None, column_property.key, None, [], "", "", (".".join( ["edit", layer.item_type, str(layer.id)]), layer.name))) except NoSuchTableError: print( colorize( "ERROR! No such table '{}' for layer '{}'.".format( layer.geo_table, layer.name), RED)) print(colorize(traceback.format_exc(), RED)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE": raise
def main(): """ tool used to help th use in the user tash """ usage = """usage: {prog} [command] [options] Available commands: """ + colorize("help", GREEN) + """: show this page """ + colorize("upgrade", GREEN) + """: upgrade the application to a new version """ + colorize("deploy", GREEN) + """: deploy the application to a server To have some help on a command type: {prog} help [command]""".format(prog=sys.argv[0]) if len(sys.argv) <= 1: print(usage) exit() if sys.argv[1] == "help": if len(sys.argv) > 2: parser = _fill_arguments(sys.argv[2]) parser.print_help() else: print(usage) exit() parser = _fill_arguments(sys.argv[1]) options = parser.parse_args(sys.argv[2:]) c2ctool = C2cTool(options) if sys.argv[1] == "upgrade": c2ctool.upgrade() elif sys.argv[1] == "deploy": c2ctool.deploy() else: print("Unknown command")
def main(): # pragma: no cover """ tool used to help th use in the user tash """ usage = """usage: {prog} [command] [options] Available commands: """ + colorize("help", GREEN) + """: show this page """ + colorize("upgrade", GREEN) + """: upgrade the application to a new version """ + colorize("deploy", GREEN) + """: deploy the application to a server To have some help on a command type: {prog} help [command]""".format(prog=sys.argv[0]) if len(sys.argv) <= 1: print(usage) exit() if sys.argv[1] == "help": if len(sys.argv) > 2: parser = _fill_arguments(sys.argv[2]) parser.print_help() else: print(usage) exit() parser = _fill_arguments(sys.argv[1]) options = parser.parse_args(sys.argv[2:]) c2ctool = C2cTool(options) if sys.argv[1] == "upgrade": c2ctool.upgrade() elif sys.argv[1] == "deploy": c2ctool.deploy() else: print("Unknown command")
def step6(self): check_call(["git", "commit", "-m", "Upgrade to GeoMapFish {}".format( pkg_resources.get_distribution("c2cgeoportal").version )]) print("") print(self.color_bar) print("") print(colorize("Congratulations your upgrade is a success.", GREEN)) print("") branch = check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).strip() print("Now all your files will be commited, you should do a git push {0!s} {1!s}.".format( self.options.git_remote, branch ))
def step6(self): check_call(["git", "commit", "-m", "Upgrade to GeoMapFish {}".format( pkg_resources.get_distribution("c2cgeoportal").version )]) print("") print(self.color_bar) print("") print(colorize("Congratulations your upgrade is a success.", GREEN)) print("") branch = check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).strip() print("Now all your files will be commited, you should do a git push %s %s." % ( self.options.git_remote, branch ))
def files_to_move(self, pre=False): error = False for element in self.get_upgrade("files_to_move"): src = element["from"] dst = element["to"] if os.path.exists(src): type_ = "directory" if os.path.isdir(src) else "file" managed = False for pattern in self.project["managed_files"]: if re.match(pattern + '$', src): print( colorize( "The {} '{}' is present in the managed_files as '{}' but he will move." .format(type_, src, pattern), RED)) error = True managed = True break if not managed and os.path.exists(dst): print( colorize( "The destination '{}' already exists.".format(dst), RED)) error = True if not pre: raise InteruptedException( "The destination '{}' already exists.".format(dst)) if not managed and not pre: print( colorize( "Move the {} '{}' to '{}'.".format( type_, src, dst), GREEN)) if "version" in element: print("Needed from version {}.".format( element["version"])) os.rename(src, dst) return error
def __call__(self, filename, options): messages = [] self.env = bootstrap(filename) with open("project.yaml") as f: self.package = yaml.load(f) self.config = get_config(".build/config.yaml") try: from c2cgeoportal.models import Theme, LayerGroup, \ LayerWMS, LayerWMTS self._import(Theme, messages) self._import(LayerGroup, messages) self._import(LayerWMS, messages, self._import_layer_wms) self._import(LayerWMTS, messages, self._import_layer_wmts) except ProgrammingError as e: print( colorize( "ERROR: The database is probably not up to date " "(should be ignored when happen during the upgrade)", RED)) print(colorize(e, RED)) return messages
def post(self, command, output_dir, vars): """ Overrides the base template class to print the next step. """ if os.name == 'posix': for file in ("post-restore-code", "pre-restore-database.mako"): dest = os.path.join(output_dir, "deploy/hooks", file) subprocess.check_call(["chmod", "+x", dest]) self.out("\nContinue with:") self.out(colorize( ".build/venv/bin/pcreate -s c2cgeoportal_update ../{vars[project]} " "package={vars[package]} srid={vars[srid]} extent={vars[extent]}".format(vars=vars), GREEN )) return BaseTemplate.post(self, command, output_dir, vars)
def __call__(self, filename, options): messages = [] try: self.env = bootstrap(filename) with open("project.yaml") as f: self.package = yaml.safe_load(f) self.config = get_config("config.yaml") try: from c2cgeoportal.models import DBSession, Theme, LayerGroup, \ LayerWMS, LayerWMTS, FullTextSearch self._import(Theme, messages) self._import(LayerGroup, messages) self._import(LayerWMS, messages, self._import_layer_wms) self._import(LayerWMTS, messages, self._import_layer_wmts) for ln, in DBSession.query( FullTextSearch.layer_name).distinct().all(): if ln is not None and ln != "": messages.append( Message( None, ln, None, [], "", "", ("fts", ln.encode("ascii", errors="replace")))) except ProgrammingError as e: print( colorize( "ERROR! The database is probably not up to date " "(should be ignored when happen during the upgrade)", RED)) print(colorize(e, RED)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE": raise except NoSuchTableError as e: print( colorize( "ERROR! The schema didn't seem to exists " "(should be ignored when happen during the deploy)", RED)) print(colorize(e, RED)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE": raise except OperationalError as e: print( colorize( "ERROR! The database didn't seem to exists " "(should be ignored when happen during the deploy)", RED)) print(colorize(e, RED)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE": raise return messages
def _enumerate_attributes_values(cls, dbsessions, layers, layerinfos, fieldname): dbname = layerinfos.get("dbsession", "dbsession") try: dbsession = dbsessions.get(dbname) return layers.query_enumerate_attribute_values( dbsession, layerinfos, fieldname) except Exception as e: table = layerinfos.get("attributes").get(fieldname, {}).get("table") print( colorize( "ERROR! Unable to collect enumerate attributes for " "db: {}, table: {}, column: {} ({})".format( dbname, table, fieldname, e), RED)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE": return [] else: raise
def _import_layer_wmts(self, layer, messages): from c2cgeoportal.models import DBSession, OGCServer layers = [d.value for d in layer.metadatas if d.name == "queryLayers"] if len(layers) == 0: layers = [d.value for d in layer.metadatas if d.name == "wmsLayer"] server = [d.value for d in layer.metadatas if d.name == "ogcServer"] if len(server) >= 1 and len(layers) >= 1: for wms_layer in layers: try: db_server = DBSession.query(OGCServer).filter( OGCServer.name == server[0]).one() self._import_layer_attributes( db_server.url_wfs or db_server.url, wms_layer, layer.item_type, layer.name, messages) except NoResultFound: print( colorize( "ERROR! the OGC server '{}' from the WMTS layer '{}' does not exist." .format(server[0], layer.name), RED)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE": raise
def print_step(self, step, intro="To continue type:"): print(intro) print(colorize("make -f %s upgrade%i", YELLOW) % ( self.options.file if self.options.file is not None else "<user.mk>", step ))
def __call__(self, filename, options): config = get_config("config.yaml") class Registry: settings = config class Request: registry = Registry() params = {} GET = {} user_agent = "" @staticmethod def static_url(*args, **kwargs): return "" @staticmethod def static_path(*args, **kwargs): return "" @staticmethod def route_url(*args, **kwargs): return "" @staticmethod def current_route_url(*args, **kwargs): return "" init_region({"backend": "dogpile.cache.memory"}) int_filename = filename if re.match( "^" + re.escape("./{}/templates".format(config["package"])), filename): try: empty_template = Template("") class Lookup(TemplateLookup): @staticmethod def get_template(uri): return empty_template class MyTemplate(MakoTemplate): def prepare(self, **options): options.update({"input_encoding": self.encoding}) lookup = Lookup(**options) if self.source: self.tpl = Template(self.source, lookup=lookup, **options) else: self.tpl = Template(uri=self.name, filename=self.filename, lookup=lookup, **options) try: processed = template(filename, { "request": Request(), "lang": "fr", "debug": False, "extra_params": {}, "permalink_themes": "", "fulltextsearch_groups": [], "wfs_types": [], "_": lambda x: x, }, template_adapter=MyTemplate) int_filename = os.path.join( os.path.dirname(filename), "_" + os.path.basename(filename)) with open(int_filename, "wb") as file_open: file_open.write(processed.encode("utf-8")) except: print( colorize( "ERROR! Occurred during the '{}' template generation" .format(filename), RED)) print(colorize(traceback.format_exc(), RED)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE": # Continue with the original one int_filename = filename else: raise except: print(traceback.format_exc()) message_str = subprocess.check_output( ["node", "tools/extract-messages.js", int_filename]).decode("utf-8") if int_filename != filename: os.unlink(int_filename) try: messages = [] for contexts, message in loads(message_str): for context in contexts.split(", "): messages.append( Message(None, message, None, [], "", "", context.split(":"))) return messages except: print(colorize("An error occurred", RED)) print(colorize(message_str, RED)) print("------") raise
def _layer_attributes(self, url, layer): errors = set() class Registry: setting = None class Request: registry = Registry() request = Request() request.registry.settings = self.config # static schema will not be supported url = get_url2("Layer", url, request, errors) if len(errors) > 0: print("\n".join(errors)) return [] wms_getcap_url = add_url_params(url, { "SERVICE": "WMS", "VERSION": "1.1.1", "REQUEST": "GetCapabilities", }) hostname = urlsplit(url).hostname if url not in self.wmscap_cache: print("Get WMS GetCapabilities for URL: {}".format(url)) self.wmscap_cache[url] = None # forward request to target (without Host Header) http = httplib2.Http() h = {} if hostname == "localhost": # pragma: no cover h["Host"] = self.package["host"] try: resp, content = http.request(wms_getcap_url, method="GET", headers=h) try: self.wmscap_cache[url] = WebMapService(None, xml=content) except Exception as e: print( colorize( "ERROR! an error occurred while trying to " "parse the GetCapabilities document.", RED)) print(colorize(str(e), RED)) print("URL: {}\nxml:\n{}".format(wms_getcap_url, content)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE": raise except Exception as e: # pragma: no cover print(colorize(str(e), RED)) print( colorize( "ERROR! Unable to GetCapabilities from URL: {}".format( wms_getcap_url), RED, )) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") != "TRUE": raise wmscap = self.wmscap_cache[url] wfs_descrfeat_url = add_url_params( url, { "SERVICE": "WFS", "VERSION": "1.1.0", "REQUEST": "DescribeFeatureType", }) if url not in self.featuretype_cache: print("Get WFS DescribeFeatureType for URL: {}".format( wfs_descrfeat_url)) self.featuretype_cache[url] = None # forward request to target (without Host Header) http = httplib2.Http() h = {} if hostname == "localhost": # pragma: no cover h["Host"] = self.package["host"] try: resp, content = http.request(wfs_descrfeat_url, method="GET", headers=h) except Exception as e: # pragma: no cover print(colorize(str(e), RED)) print( colorize( "ERROR! Unable to DescribeFeatureType from URL: {}". format(wfs_descrfeat_url), RED, )) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE": return [] else: raise if resp.status < 200 or resp.status >= 300: # pragma: no cover print( colorize( "ERROR! DescribeFeatureType from URL {} return the error: {1:d} {}" .format(wfs_descrfeat_url, resp.status, resp.reason), RED, )) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE": return [] else: raise Exception("Aborted") try: describe = parseString(content) featurestype = {} self.featuretype_cache[url] = featurestype for type_element in describe.getElementsByTagNameNS( "http://www.w3.org/2001/XMLSchema", "complexType"): featurestype[type_element.getAttribute( "name")] = type_element except ExpatError as e: print( colorize( "ERROR! an error occurred while trying to " "parse the DescribeFeatureType document.", RED)) print(colorize(str(e), RED)) print("URL: {}\nxml:\n{}".format(wfs_descrfeat_url, content)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE": return [] else: raise except AttributeError: print( colorize( "ERROR! an error occurred while trying to " "read the Mapfile and recover the themes.", RED)) print("URL: {}\nxml:\n{}".format(wfs_descrfeat_url, content)) if os.environ.get("IGNORE_I18N_ERRORS", "FALSE") == "TRUE": return [] else: raise else: featurestype = self.featuretype_cache[url] if featurestype is None: return [] layers = [layer] if wmscap is not None and layer in list(wmscap.contents): layer_obj = wmscap[layer] if len(layer_obj.layers) > 0: layers = [l.name for l in layer_obj.layers] attributes = [] for sub_layer in layers: # Should probably be adapted for other king of servers type_element = featurestype.get("{}Type".format(sub_layer)) if type_element is not None: for element in type_element.getElementsByTagNameNS( "http://www.w3.org/2001/XMLSchema", "element"): if not element.getAttribute("type").startswith("gml:"): attributes.append(element.getAttribute("name")) return attributes
def step1(self): notes = [] check_call(["git", "reset", "--hard"]) check_call(["git", "clean", "--force", "-d"]) check_call(["git", "submodule", "foreach", "--recursive", "git", "reset", "--hard"]) check_call(["git", "submodule", "foreach", "--recursive", "git", "clean", "--force", "-d"]) branch = check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).strip() # remove all no more existing branches check_call(["git", "fetch", "origin", "--prune"]) branches = check_output(["git", "branch", "--all"]).split("\n") if " remotes/origin/%s" % branch in branches: try: check_call(["git", "pull", "--rebase", self.options.git_remote, branch]) except CalledProcessError: print(self.color_bar) print(colorize("The pull (rebase) failed.", RED)) print(colorize("Please solve the rebase and run the step again.", YELLOW)) exit(1) check_call(["git", "submodule", "sync"]) check_call(["git", "submodule", "update", "--init"]) check_call(["git", "submodule", "foreach", "git", "submodule", "sync"]) check_call(["git", "submodule", "foreach", "git", "submodule", "update", "--init"]) if len(check_output(["git", "status", "-z"]).strip()) != 0: print(self.color_bar) print(colorize("The pull isn't fast forward.", RED)) print(colorize("Please solve the rebase and run the step again.", YELLOW)) exit(1) check_call(["git", "submodule", "foreach", "git", "submodule", "sync"]) check_call(["git", "submodule", "foreach", "git", "submodule", "update", "--init"]) if not self.options.windows: check_call(["make", "-f", self.options.file, ".build/requirements.timestamp"]) pip_cmd = ["%s/pip" % self.venv_bin, "install"] if self.options.version == "master": check_call(["%s/pip" % self.venv_bin, "uninstall", "--yes", "c2cgeoportal"]) pip_cmd += ["--pre", "c2cgeoportal"] else: pip_cmd += ["c2cgeoportal==%s" % (self.options.version)] check_call(pip_cmd) check_call([ "%s/pcreate" % self.venv_bin, "--ignore-conflicting-name", "--overwrite", "--scaffold=c2cgeoportal_update", "../%s" % self.project["project_folder"] ]) pcreate_cmd = [ "%s/pcreate" % self.venv_bin, "--ignore-conflicting-name", "--scaffold=c2cgeoportal_create", "/tmp/%s" % self.project["project_folder"], ] for name, value in self.project["template_vars"].items(): if isinstance(value, basestring): value = value.encode('utf-8') pcreate_cmd.append("{}={}".format(name, value)) check_call(pcreate_cmd) check_call(["make", "-f", self.options.file, self.options.clean]) diff_file = open("changelog.diff", "w") check_call(["git", "diff", "CONST_CHANGELOG.txt"], stdout=diff_file) diff_file.close() check_call(["make", "-f", self.options.file, "update-node-modules"]) check_call(["make", "-f", self.options.file, ".build/requirements.timestamp"]) print("") print(self.color_bar) print("\n".join(notes)) print( "Apply the manual migration steps based on what is in the CONST_CHANGELOG.txt file" " (listed in the `changelog.diff` file)." ) self.print_step(2)
def step1(self): notes = [] check_call(["git", "reset", "--hard"]) check_call(["git", "clean", "--force", "-d"]) check_call(["git", "submodule", "foreach", "--recursive", "git", "reset", "--hard"]) check_call(["git", "submodule", "foreach", "--recursive", "git", "clean", "--force", "-d"]) branch = check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).strip() # remove all no more existing branches check_call(["git", "update", "origin", "--prune"]) branches = check_output(["git", "branch", "--all"]).split("\n") if " remotes/origin/%s" % branch in branches: try: check_call(["git", "pull", "--rebase", self.options.git_remote, branch]) except CalledProcessError: print(self.color_bar) print(colorize("The pull (rebase) failed.", RED)) print(colorize("Please solve the rebase and run the step again.", YELLOW)) exit(1) check_call(["git", "submodule", "sync"]) check_call(["git", "submodule", "update", "--init"]) check_call(["git", "submodule", "foreach", "git", "submodule", "sync"]) check_call(["git", "submodule", "foreach", "git", "submodule", "update", "--init"]) if len(check_output(["git", "status", "-z"]).strip()) != 0: print(self.color_bar) print(colorize("The pull isn't fast forward.", RED)) print(colorize("Please solve the rebase and run the step again.", YELLOW)) exit(1) check_call(["git", "submodule", "foreach", "git", "fetch", "origin"]) if self.options.version == "master": check_call([ "git", "submodule", "foreach", "git", "reset", "--hard", "origin/%s" % self.options.version, "--" ]) elif re.match(VERSION_RE, self.options.version) is not None: check_call([ "git", "submodule", "foreach", "git", "reset", "--hard", self.options.version, "--" ]) else: notes.append( "We can't define the cgxp revision, than you should manually do:\n" "git submodule foreach git reset --hard <revision>" ) check_call(["git", "submodule", "foreach", "git", "submodule", "sync"]) check_call(["git", "submodule", "foreach", "git", "submodule", "update", "--init"]) if not self.options.windows: check_call(["make", "-f", self.options.file, ".build/requirements.timestamp"]) pip_cmd = [ "%s/pip" % self.venv_bin, "install", "--find-links", self.options.index_url, ] if self.options.version == "master": check_call(["%s/pip" % self.venv_bin, "uninstall", "--yes", self.package]) pip_cmd += ["--pre", self.package] else: pip_cmd += ["%s==%s" % (self.package, self.options.version)] check_call(pip_cmd) check_call([ "%s/pcreate" % self.venv_bin, "--overwrite", "--scaffold=c2cgeoportal_update", "../%s" % self.project["project_folder"], "package=%s" % self.project["project_package"] ]) check_call([ "%s/pcreate" % self.venv_bin, "-s", "c2cgeoportal_create", "/tmp/%s" % self.project["project_folder"], "package=%s" % self.project["project_package"], "mobile_application_title=%s" % self.project["template_vars"]["mobile_application_title"], "srid=%s" % self.project["template_vars"].get("srid", 21781), ]) check_call(["make", "-f", self.options.file, self.options.clean]) diff_file = open("changelog.diff", "w") check_call(["git", "diff", "CONST_CHANGELOG.txt"], stdout=diff_file) diff_file.close() check_call(["make", "-f", self.options.file, ".build/requirements.timestamp"]) print("") print(self.color_bar) print("\n".join(notes)) print( "Apply the manual migration steps based on what is in the CONST_CHANGELOG.txt file" " (listed in the `changelog.diff` file)." ) self.print_step(2)
def __call__(self, filename, options): config = get_config(".build/config.yaml") class Registry: settings = config class Request: registry = Registry() params = {} GET = {} user_agent = "" def static_url(*args, **kwargs): return "" def static_path(*args, **kwargs): return "" def route_url(*args, **kwargs): return "" def current_route_url(*args, **kwargs): return "" init_region({"backend": "dogpile.cache.memory"}) int_filename = filename if re.match("^" + re.escape("./{}/templates".format(config["package"])), filename): try: empty_template = Template("") class Lookup(TemplateLookup): @staticmethod def get_template(uri): return empty_template class MyTemplate(MakoTemplate): def prepare(self, **options): options.update({"input_encoding": self.encoding}) lookup = Lookup(**options) if self.source: self.tpl = Template(self.source, lookup=lookup, **options) else: self.tpl = Template( uri=self.name, filename=self.filename, lookup=lookup, **options) processed = template( filename, { "request": Request(), "lang": "fr", "debug": False, "extra_params": {}, "permalink_themes": "", "fulltextsearch_groups": [], "wfs_types": [], "_": lambda x: x, }, template_adapter=MyTemplate ) int_filename = os.path.join(os.path.dirname(filename), "_" + os.path.basename(filename)) with open(int_filename, "wb") as file_open: file_open.write(processed.encode("utf-8")) except: print(traceback.format_exc()) message_str = subprocess.check_output(["node", "tools/extract-messages.js", int_filename]) if int_filename != filename: os.unlink(int_filename) try: messages = [] for contexts, message in loads(message_str): for context in contexts.split(", "): messages.append(Message( None, message, None, [], u"", u"", context.split(":") )) return messages except: print(colorize("An error occurred", RED)) print(colorize(message_str, RED)) print("------") raise
class C2cTool: color_bar = colorize("================================================================", GREEN) def print_step(self, step, intro="To continue type:"): print(colorize(intro, YELLOW)) print(colorize("make -f {} upgrade{}", GREEN).format( self.options.file if self.options.file is not None else "<user.mk>", step if step != 0 else "", )) def get_project(self): if not os.path.isfile("project.yaml"): print("Unable to find the required 'project.yaml' file.") exit(1) with open("project.yaml", "r") as f: return yaml.load(f) def test_checkers(self): http = httplib2.Http() for check_type in ("", "type=all"): resp, content = http.request( "http://localhost%s%s" % (self.project["checker_path"], check_type), method="GET", headers={ "Host": self.project["host"] } ) if resp.status < 200 or resp.status >= 300: print(self.color_bar) print("Checker error:") print("Open `http://%s%s%s` for more informations." % ( self.project["host"], self.project["checker_path"], check_type )) return False return True def __init__(self, options): self.options = options self.project = self.get_project() def upgrade(self): self.venv_bin = ".build/venv/bin" if self.options.windows: self.options.clean = "clean" self.venv_bin = ".build/venv/Scripts" if self.options.step == 0: self.step0() elif self.options.step == 1: self.step1() elif self.options.step == 2: self.step2() elif self.options.step == 3: self.step3() elif self.options.step == 4: self.step4() elif self.options.step == 5: self.step5() elif self.options.step == 6: self.step6() def step0(self): project_template_keys = self.project.get("template_vars").keys() messages = [] for required in REQUIRED_TEMPLATE_KEYS: if required not in project_template_keys: messages.append( "The element '{}' is missing in the 'template_vars' of " "the file 'project.yaml.mako', you should for example: {}: {}.".format( required, required, TEMPLATE_EXAMPLE.get('required', '') ) ) if len(messages) > 0: print("") print(self.color_bar) print(colorize("\n".join(messages), RED)) print("") self.print_step(0, intro="Fix it and run again the upgrade:") exit(1) if self.options.version is None: print("") print(self.color_bar) print("The VERSION environment variable is required for this upgrade step") exit(1) if re.match(VERSION_RE, self.options.version) is not None: http = httplib2.Http() url = ( "http://raw.github.com/camptocamp/c2cgeoportal/%s/" "c2cgeoportal/scaffolds/update/CONST_versions_requirements.txt" % self.options.version ) headers, _ = http.request(url) if headers.status != 200: print("") print(self.color_bar) print( "Failed downloading the c2cgeoportal " "CONST_versions_requirements.txt file from URL:" ) print(url) print("The upgrade is impossible") exit(1) if os.path.split(os.path.realpath("."))[1] != self.project["project_folder"]: print("") print(self.color_bar) print("Your project isn't in the right folder!") print("It should be in folder '%s' instead of folder '%s'." % ( self.project["project_folder"], os.path.split(os.path.realpath("."))[1] )) print("") self.print_step(0, intro="Fix it and lunch again the upgrade:") exit(1) if check_output(["git", "status", "--short"]) == "": self.step1() else: check_call(["git", "status"]) print("") print(self.color_bar) print( "Here is the output of 'git status'. Please make sure to commit all your changes " "before going further. All uncommited changes will be lost." ) self.print_step(1) def step1(self): if self.options.version is None: print("") print(self.color_bar) print("The VERSION environment variable is required for this upgrade step") exit(1) check_call(["git", "reset", "--hard"]) check_call(["git", "clean", "--force", "-d"]) check_call(["git", "submodule", "foreach", "--recursive", "git", "reset", "--hard"]) check_call(["git", "submodule", "foreach", "--recursive", "git", "clean", "--force", "-d"]) branch = check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).strip() # remove all no more existing branches check_call(["git", "fetch", "origin", "--prune"]) branches = check_output(["git", "branch", "--all"]).split("\n") if " remotes/origin/%s" % branch in branches: try: check_call(["git", "pull", "--rebase", self.options.git_remote, branch]) except CalledProcessError: print(self.color_bar) print("") print(colorize("The pull (rebase) failed.", RED)) print("") self.print_step(1, intro="Please solve the rebase and run the step 1 again:") exit(1) check_call(["git", "submodule", "sync"]) check_call(["git", "submodule", "update", "--init"]) check_call(["git", "submodule", "foreach", "git", "submodule", "sync"]) check_call(["git", "submodule", "foreach", "git", "submodule", "update", "--init"]) if len(check_output(["git", "status", "-z"]).strip()) != 0: print(self.color_bar) print("") print(colorize("The pull isn't fast forward.", RED)) print("") self.print_step(1, intro="Please solve the rebase and run the step 1 again:") exit(1) check_call(["git", "submodule", "foreach", "git", "submodule", "sync"]) check_call(["git", "submodule", "foreach", "git", "submodule", "update", "--init"]) if not self.options.windows: check_call(["make", "-f", self.options.file, ".build/requirements.timestamp"]) pip_cmd = ["%s/pip" % self.venv_bin, "install"] if self.options.version == "master": check_call(["%s/pip" % self.venv_bin, "uninstall", "--yes", "c2cgeoportal"]) pip_cmd += ["--pre", "c2cgeoportal"] else: pip_cmd += ["c2cgeoportal==%s" % (self.options.version)] check_call(pip_cmd) check_call([ "%s/pcreate" % self.venv_bin, "--ignore-conflicting-name", "--overwrite", "--scaffold=c2cgeoportal_update", "../%s" % self.project["project_folder"] ]) check_call(["make", "-f", self.options.file, self.options.clean]) with open("changelog.diff", "w") as diff_file: check_call(["git", "diff", "--", "CONST_CHANGELOG.txt"], stdout=diff_file) check_call(["make", "-f", self.options.file, "update-node-modules"]) check_call(["make", "-f", self.options.file, ".build/requirements.timestamp"]) if os.path.getsize("changelog.diff") == 0: self.step2() else: print("") print(self.color_bar) print( "Apply the manual migration steps based on what is in the CONST_CHANGELOG.txt file" " (listed in the `changelog.diff` file)." ) self.print_step(2) def step2(self): if os.path.isfile("changelog.diff"): os.unlink("changelog.diff") with open("ngeo.diff", "w") as diff_file: check_call([ "git", "diff", "--", "CONST_create_template/{}/templates".format(self.project["project_package"]), "CONST_create_template/{}/static-ngeo".format(self.project["project_package"]), ], stdout=diff_file) if os.path.getsize("ngeo.diff") == 0: self.step3() else: print("") print(self.color_bar) print( "Apply the ngeo application diff available in the `ngeo.diff` file." ) self.print_step(3) def step3(self): if os.path.isfile("ngeo.diff"): os.unlink("ngeo.diff") status = check_output(["git", "status", "--short", "CONST_create_template"]) status = [s for s in status.split("\n") if len(s) > 3] status = [s[3:] for s in status if not s.startswith("?? ")] status = [s for s in status if not s.startswith( "CONST_create_template/{}/templates/".format(self.project["project_package"]), )] status = [s for s in status if not s.startswith( "CONST_create_template/{}/static-ngeo/".format(self.project["project_package"]), )] matcher = re.compile(r"CONST_create_tremplate.*/CONST_.+") status = [s for s in status if not matcher.match(s)] status = [s for s in status if s != "CONST_create_template/package.json"] with open("create.diff", "w") as diff_file: check_call(["git", "diff", "--"] + status, stdout=diff_file) if os.path.getsize("create.diff") == 0: self.step4() else: print("") print(self.color_bar) print( "This is an optional step but it helps to have a standard project.\n" "In the `create.diff` file, there are all the changes made in the create template " "that are recommended to be reported in your project.\n" "An advise to be more effective: In most of the casees it concerned a file that " "you never customise, or a file that you customised in depth, then respectively " "copy the new file from CONST_create_template, respectively ignore the changes." ) self.print_step(4) def step4(self): if self.options.file is None: print("") print(self.color_bar) print("The makefile is missing") print("") self.print_step(4, intro="Fix it and run again the step 4:") exit(1) if os.path.isfile("create.diff"): os.unlink("create.diff") check_call(["make", "-f", self.options.file, "build"]) command.upgrade(Config("alembic.ini"), "head") command.upgrade(Config("alembic_static.ini"), "head") if not self.options.windows: check_call(self.project.get("cmds", {}).get( "apache_graceful", ["sudo", "/usr/sbin/apache2ctl", "graceful"] )) print("") print(self.color_bar) print("The upgrade is nearly done, now you should:") print("- Test your application.") if self.options.windows: print("You are running on Windows, please restart your Apache server,") print("because we can not do that automatically.") self.print_step(5) def step5(self): if not self.test_checkers(): print("") self.print_step(5, intro="Correct the checker, the step 5 again:") exit(1) # Required to remove from the Git stage the ignored file when we lunch the step again check_call(["git", "reset", "--mixed"]) check_call(["git", "add", "-A"]) check_call(["git", "status"]) print("") print(self.color_bar) print("We will commit all the above files!") print( "If there are some files which should not be commited, then you should " "add them into the `.gitignore` file and launch step 5 again." ) self.print_step(6, intro="Then to commit your changes type:") def step6(self): check_call(["git", "commit", "-m", "Upgrade to GeoMapFish {}".format( pkg_resources.get_distribution("c2cgeoportal").version )]) print("") print(self.color_bar) print("") print(colorize("Congratulations your upgrade is a success.", GREEN)) print("") branch = check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).strip() print("Now all your files will be commited, you should do a git push %s %s." % ( self.options.git_remote, branch )) def deploy(self): if not self.test_checkers(): print(colorize("Correct them and run again", RED)) exit(1) check_call(["sudo", "-u", "deploy", "deploy", "-r", "deploy/deploy.cfg", self.options.host]) check_call(["make", "-f", self.options.file, "build"])
def step1(self): if self.options.version is None: print("") print(self.color_bar) print("The VERSION environment variable is required for this upgrade step") exit(1) check_call(["git", "reset", "--hard"]) check_call(["git", "clean", "--force", "-d"]) check_call(["git", "submodule", "foreach", "--recursive", "git", "reset", "--hard"]) check_call(["git", "submodule", "foreach", "--recursive", "git", "clean", "--force", "-d"]) branch = check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).strip() # remove all no more existing branches check_call(["git", "fetch", "origin", "--prune"]) branches = check_output(["git", "branch", "--all"]).split("\n") if " remotes/origin/%s" % branch in branches: try: check_call(["git", "pull", "--rebase", self.options.git_remote, branch]) except CalledProcessError: print(self.color_bar) print("") print(colorize("The pull (rebase) failed.", RED)) print("") self.print_step(1, intro="Please solve the rebase and run the step 1 again:") exit(1) check_call(["git", "submodule", "sync"]) check_call(["git", "submodule", "update", "--init"]) check_call(["git", "submodule", "foreach", "git", "submodule", "sync"]) check_call(["git", "submodule", "foreach", "git", "submodule", "update", "--init"]) if len(check_output(["git", "status", "-z"]).strip()) != 0: print(self.color_bar) print("") print(colorize("The pull isn't fast forward.", RED)) print("") self.print_step(1, intro="Please solve the rebase and run the step 1 again:") exit(1) check_call(["git", "submodule", "foreach", "git", "submodule", "sync"]) check_call(["git", "submodule", "foreach", "git", "submodule", "update", "--init"]) if not self.options.windows: check_call(["make", "-f", self.options.file, ".build/requirements.timestamp"]) pip_cmd = ["%s/pip" % self.venv_bin, "install"] if self.options.version == "master": check_call(["%s/pip" % self.venv_bin, "uninstall", "--yes", "c2cgeoportal"]) pip_cmd += ["--pre", "c2cgeoportal"] else: pip_cmd += ["c2cgeoportal==%s" % (self.options.version)] check_call(pip_cmd) check_call([ "%s/pcreate" % self.venv_bin, "--ignore-conflicting-name", "--overwrite", "--scaffold=c2cgeoportal_update", "../%s" % self.project["project_folder"] ]) check_call(["make", "-f", self.options.file, self.options.clean]) with open("changelog.diff", "w") as diff_file: check_call(["git", "diff", "--", "CONST_CHANGELOG.txt"], stdout=diff_file) check_call(["make", "-f", self.options.file, "update-node-modules"]) check_call(["make", "-f", self.options.file, ".build/requirements.timestamp"]) if os.path.getsize("changelog.diff") == 0: self.step2() else: print("") print(self.color_bar) print( "Apply the manual migration steps based on what is in the CONST_CHANGELOG.txt file" " (listed in the `changelog.diff` file)." ) self.print_step(2)
def step0(self): project_template_keys = self.project.get("template_vars").keys() messages = [] for required in REQUIRED_TEMPLATE_KEYS: if required not in project_template_keys: messages.append( "The element '{}' is missing in the 'template_vars' of " "the file 'project.yaml.mako', you should for example: {}: {}.".format( required, required, TEMPLATE_EXAMPLE.get('required', '') ) ) if len(messages) > 0: print("") print(self.color_bar) print(colorize("\n".join(messages), RED)) print("") self.print_step(0, intro="Fix it and run again the upgrade:") exit(1) if self.options.version is None: print("") print(self.color_bar) print("The VERSION environment variable is required for this upgrade step") exit(1) if re.match(VERSION_RE, self.options.version) is not None: http = httplib2.Http() url = ( "http://raw.github.com/camptocamp/c2cgeoportal/%s/" "c2cgeoportal/scaffolds/update/CONST_versions_requirements.txt" % self.options.version ) headers, _ = http.request(url) if headers.status != 200: print("") print(self.color_bar) print( "Failed downloading the c2cgeoportal " "CONST_versions_requirements.txt file from URL:" ) print(url) print("The upgrade is impossible") exit(1) if os.path.split(os.path.realpath("."))[1] != self.project["project_folder"]: print("") print(self.color_bar) print("Your project isn't in the right folder!") print("It should be in folder '%s' instead of folder '%s'." % ( self.project["project_folder"], os.path.split(os.path.realpath("."))[1] )) print("") self.print_step(0, intro="Fix it and lunch again the upgrade:") exit(1) if check_output(["git", "status", "--short"]) == "": self.step1() else: check_call(["git", "status"]) print("") print(self.color_bar) print( "Here is the output of 'git status'. Please make sure to commit all your changes " "before going further. All uncommited changes will be lost." ) self.print_step(1)
def print_step(self, step, intro="To continue type:"): print(colorize(intro, YELLOW)) print(colorize("make -f {} upgrade{}", GREEN).format( self.options.file if self.options.file is not None else "<user.mk>", step if step != 0 else "", ))
def step1(self): if self.options.version is None: print("") print(self.color_bar) print("The VERSION environment variable is required for this upgrade step") exit(1) check_call(["git", "reset", "--hard"]) check_call(["git", "clean", "--force", "-d"]) check_call(["git", "submodule", "foreach", "--recursive", "git", "reset", "--hard"]) check_call(["git", "submodule", "foreach", "--recursive", "git", "clean", "--force", "-d"]) branch = check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).strip() # remove all no more existing branches check_call(["git", "fetch", "origin", "--prune"]) branches = check_output(["git", "branch", "--all"]).split("\n") if " remotes/origin/{0!s}".format(branch) in branches: try: check_call(["git", "pull", "--rebase", self.options.git_remote, branch]) except CalledProcessError: print(self.color_bar) print("") print(colorize("The pull (rebase) failed.", RED)) print("") self.print_step(1, intro="Please solve the rebase and run the step 1 again:") exit(1) check_call(["git", "submodule", "sync"]) check_call(["git", "submodule", "update", "--init"]) check_call(["git", "submodule", "foreach", "git", "submodule", "sync"]) check_call(["git", "submodule", "foreach", "git", "submodule", "update", "--init"]) if len(check_output(["git", "status", "-z"]).strip()) != 0: print(self.color_bar) print("") print(colorize("The pull is not fast forward.", RED)) print("") self.print_step(1, intro="Please solve the rebase and run the step 1 again:") exit(1) check_call(["git", "submodule", "foreach", "git", "submodule", "sync"]) check_call(["git", "submodule", "foreach", "git", "submodule", "update", "--init"]) if not self.options.windows: check_call(["make", "-f", self.options.file, ".build/requirements.timestamp"]) pip_cmd = ["{0!s}/pip".format(self.venv_bin), "install"] if self.options.version == "master": check_call(["{0!s}/pip".format(self.venv_bin), "uninstall", "--yes", "c2cgeoportal"]) pip_cmd += ["--pre", "c2cgeoportal"] else: pip_cmd += ["c2cgeoportal=={0!s}".format((self.options.version))] check_call(pip_cmd) check_call([ "{0!s}/pcreate".format(self.venv_bin), "--ignore-conflicting-name", "--overwrite", "--scaffold=c2cgeoportal_update", "../{0!s}".format(self.project["project_folder"]) ]) check_call(["make", "-f", self.options.file, self.options.clean]) # Update the package.json file if os.path.exists("package.json") and os.path.getsize("package.json") > 0: with open("package.json", "r") as package_json_file: package_json = json.loads(package_json_file.read(), encoding="utf-8") with open("CONST_create_template/package.json", "r") as package_json_file: template_package_json = json.loads(package_json_file.read(), encoding="utf-8") if "devDependencies" not in package_json: package_json["devDependencies"] = {} for package, version in template_package_json.get("devDependencies", {}).items(): package_json["devDependencies"][package] = version with open("package.json", "w") as package_json_file: json.dump( package_json, package_json_file, encoding="utf-8", sort_keys=True, separators=(',', ': '), indent=2 ) package_json_file.write("\n") else: shutil.copyfile("CONST_create_template/package.json", "package.json") with open("changelog.diff", "w") as diff_file: check_call(["git", "diff", "--", "CONST_CHANGELOG.txt"], stdout=diff_file) check_call(["make", "-f", self.options.file, "update-node-modules"]) check_call(["make", "-f", self.options.file, ".build/requirements.timestamp"]) if os.path.getsize("changelog.diff") == 0: self.step2() else: print("") print(self.color_bar) print( "Apply the manual migration steps based on what is in the CONST_CHANGELOG.txt file" " (listed in the `changelog.diff` file)." ) self.print_step(2)
class C2cUpgradeTool: color_bar = colorize( "================================================================", GREEN) project = None def __init__(self, options): self.options = options self.project = self.get_project() @staticmethod def get_project(): if not os.path.isfile("project.yaml"): print( colorize("Unable to find the required 'project.yaml' file.", RED)) exit(1) with open("project.yaml", "r") as f: return yaml.safe_load(f) @staticmethod def get_upgrade(section): if not os.path.isfile(".upgrade.yaml"): print( colorize("Unable to find the required '.upgrade.yaml' file.", RED)) exit(1) with open(".upgrade.yaml", "r") as f: return yaml.safe_load(f)[section] def print_step(self, step, error=False, message=None, prompt="To continue type:"): print("") print(self.color_bar) if message is not None: print(colorize(message, RED if error else YELLOW)) if step >= 0: print(colorize(prompt, GREEN)) args = " --makefile=" + self.options.new_makefile if self.options.makefile != "Makefile" else "" print( colorize("./docker-run make{} upgrade{}", GREEN).format( args, step if step != 0 else "", )) def run_step(self, step): getattr(self, "step{}".format(step))() def test_checkers(self): http = httplib2.Http() for check_type in ("", "type=all"): resp, _ = http.request("http://localhost{}{}".format( self.project["checker_path"], check_type), method="GET", headers={"Host": self.project["host"]}) if resp.status < 200 or resp.status >= 300: return False, "\n".join([ "Checker error:", "Open `http://{}{}{}` for more informations." ]).format(self.project["host"], self.project["checker_path"], check_type) return True, None def upgrade(self): self.run_step(self.options.step) @Step(0) def step0(self, step): project_template_keys = list(self.project.get("template_vars").keys()) messages = [] for required in REQUIRED_TEMPLATE_KEYS: if required not in project_template_keys: messages.append( "The element '{}' is missing in the 'template_vars' of " "the file 'project.yaml.mako', you should for example: {}: {}." .format(required, required, TEMPLATE_EXAMPLE.get('required', ''))) if len(messages) > 0: self.print_step(step, error=True, message="\n".join(messages), prompt="Fix it and run again the upgrade:") exit(1) if check_output(["git", "status", "--short"]).decode("utf-8") == "": self.run_step(step + 1) else: check_call(["git", "status"]) self.print_step( step + 1, message= "Here is the output of 'git status'. Please make sure to commit all your " "changes before going further. All uncommited changes will be lost." ) @Step(1) def step1(self, step): check_call(["git", "reset", "--hard"]) check_call(["git", "clean", "--force", "-d"]) check_call([ "git", "submodule", "foreach", "--recursive", "git", "reset", "--hard" ]) check_call([ "git", "submodule", "foreach", "--recursive", "git", "clean", "--force", "-d" ]) branch = check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).decode("utf-8").strip() # remove all no more existing branches check_call(["git", "fetch", "origin", "--prune"]) branches = check_output(["git", "branch", "--all"]).decode("utf-8").split("\n") if " remotes/origin/{0!s}".format(branch) in branches: try: check_call([ "git", "pull", "--rebase", self.options.git_remote, branch ]) except subprocess.CalledProcessError: self.print_step( step, error=True, message="The pull (rebase) failed.", prompt="Please solve the rebase and run it again:") exit(1) check_call(["git", "submodule", "sync"]) check_call(["git", "submodule", "update", "--init"]) check_call(["git", "submodule", "foreach", "git", "submodule", "sync"]) check_call([ "git", "submodule", "foreach", "git", "submodule", "update", "--init" ]) if len(check_output(["git", "status", "-z" ]).decode("utf-8").strip()) != 0: self.print_step(step, error=True, message="The pull is not fast forward.", prompt="Please solve the rebase and run it again:") exit(1) check_call(["git", "submodule", "foreach", "git", "submodule", "sync"]) check_call([ "git", "submodule", "foreach", "git", "submodule", "update", "--init" ]) if os.path.exists("CONST_create_template"): shutil.rmtree("CONST_create_template") project_path = os.path.join("/tmp", self.project["project_folder"]) check_call(["ln", "-s", "/src", project_path]) check_call([ "pcreate", "--ignore-conflicting-name", "--overwrite", "--scaffold=c2cgeoportal_update", project_path ]) if self.options.nondocker: check_call([ "pcreate", "--ignore-conflicting-name", "--overwrite", "--scaffold=c2cgeoportal_nondockerupdate", project_path ]) os.remove(project_path) check_call( ["make", "--makefile=" + self.options.makefile, "clean-all"]) self.options.makefile = self.options.new_makefile # Update the package.json file if os.path.exists( "package.json") and os.path.getsize("package.json") > 0: with open("package.json", "r") as package_json_file: package_json = json.loads(package_json_file.read(), encoding="utf-8") with open("CONST_create_template/package.json", "r") as package_json_file: template_package_json = json.loads(package_json_file.read(), encoding="utf-8") if "devDependencies" not in package_json: package_json["devDependencies"] = {} for package, version in list( template_package_json.get("devDependencies", {}).items()): package_json["devDependencies"][package] = version with open("package.json", "w") as package_json_file: json.dump(package_json, package_json_file, sort_keys=True, separators=(',', ': '), indent=2) package_json_file.write("\n") else: shutil.copyfile("CONST_create_template/package.json", "package.json") self.run_step(step + 1) @Step(2) def step2(self, step): error = False print("Files to remove") error |= self.files_to_remove(pre=True) print("Files to move") error |= self.files_to_move(pre=True) print("Files to get") error |= self.files_to_get(pre=True) if error: self.print_step( step, error=True, message= "There was some error on your project configuration, see above", prompt="Fix it and run it again:") exit(1) else: if "managed_files" not in self.project: self.print_step( step, message= "In the new version we will also manage almost all the create " "template files.\n" "By default following regex pattern will not be replaced:\n{}" "Than you should fill the 'managed_files' in you 'project.yaml' file with at least " "`[]`.".format("\n".join([ "- " + e for e in self.project.get('unmanaged_files', []) ])), prompt="Fill it and run it again:") else: self.run_step(step + 1) @Step(3) def step3(self, step): self.files_to_remove() self.run_step(step + 1) def files_to_remove(self, pre=False): error = False for element in self.get_upgrade("files_to_remove"): file_ = element["file"] if os.path.exists(file_): managed = False for pattern in self.project["managed_files"]: if re.match(pattern + '$', file_): print( colorize( "The file '{}' is no more use but not delete " "because he is in the managed_files as '{}'.". format(file_, pattern), RED)) error = True managed = True if not managed and not pre: print( colorize("The file '{}' is removed.".format(file_), GREEN)) if "version" in element and "from" in element: print( "Was used in version {}, to be removed from version {}." .format(element["from"], element["version"])) if os.path.isdir(file_): shutil.rmtree(file_) else: os.remove(file_) return error @Step(4) def step4(self, step): self.files_to_move() self.run_step(step + 1) def files_to_move(self, pre=False): error = False for element in self.get_upgrade("files_to_move"): src = element["from"] dst = element["to"] if os.path.exists(src): type_ = "directory" if os.path.isdir(src) else "file" managed = False for pattern in self.project["managed_files"]: if re.match(pattern + '$', src): print( colorize( "The {} '{}' is present in the managed_files as '{}' but he will move." .format(type_, src, pattern), RED)) error = True managed = True break if not managed and os.path.exists(dst): print( colorize( "The destination '{}' already exists.".format(dst), RED)) error = True if not pre: raise InteruptedException( "The destination '{}' already exists.".format(dst)) if not managed and not pre: print( colorize( "Move the {} '{}' to '{}'.".format( type_, src, dst), GREEN)) if "version" in element: print("Needed from version {}.".format( element["version"])) os.rename(src, dst) return error @Step(5) def step5(self, step): self.files_to_get() self.run_step(step + 1) def files_to_get(self, pre=False): error = False for root, _, files in os.walk("CONST_create_template"): root = root[len("CONST_create_template/"):] for file_ in files: destination = os.path.join(root, file_) managed = False for pattern in self.project["managed_files"] + self.get_upgrade( "default_project_file"): if re.match(pattern + '$', destination): managed = True for unpattern in self.project.get( "unmanaged_files", []): if re.match(unpattern + '$', destination): managed = False break break source = os.path.join("CONST_create_template", destination) if not managed and (not os.path.exists(destination) or not filecmp.cmp(source, destination)): if not pre: print( colorize( "Get the file '{}' from the create template.". format(destination), GREEN)) if os.path.dirname(destination) != "": os.makedirs(os.path.dirname(destination), exist_ok=True) shutil.copyfile(source, destination) shutil.copymode(source, destination) return error @Step(6) def step6(self, step): with open("changelog.diff", "w") as diff_file: check_call(["git", "diff", "--", "CONST_CHANGELOG.txt"], stdout=diff_file) from210 = False try: check_call(["grep", "--", "-Version 2.1.0", "changelog.diff"]) from210 = True except subprocess.CalledProcessError: pass if from210: check_call(["cp", "CONST_CHANGELOG.txt", "changelog.diff"]) if os.path.getsize("changelog.diff") == 0: self.run_step(step + 1) else: self.print_step( step + 1, message= "Apply the manual migration steps based on what is in the CONST_CHANGELOG.txt " "file (listed in the `changelog.diff` file).") @Step(7) def step7(self, step): if os.path.isfile("changelog.diff"): os.unlink("changelog.diff") status = check_output( ["git", "status", "--short", "CONST_nondocker_CHANGELOG.txt"]).decode("utf-8") if status.strip() == "?? CONST_nondocker_CHANGELOG.txt": check_call([ "cp", "CONST_nondocker_CHANGELOG.txt", "nondocker-changelog.diff" ]) else: with open("nondocker-changelog.diff", "w") as diff_file: check_call( ["git", "diff", "--", "CONST_nondocker_CHANGELOG.txt"], stdout=diff_file) if os.path.getsize("nondocker-changelog.diff") == 0: self.run_step(step + 1) else: self.print_step( step + 1, message= "Apply the manual migration steps based on what is in the " "CONST_nondocker_CHANGELOG.txt file (listed in the `nondocker-changelog.diff` file)." ) @Step(8) def step8(self, step): if os.path.isfile("nondocker-changelog.diff"): os.unlink("nondocker-changelog.diff") with open("ngeo.diff", "w") as diff_file: check_call([ "git", "diff", "--", "CONST_create_template/{}/templates".format( self.project["project_package"]), "CONST_create_template/{}/static-ngeo".format( self.project["project_package"]), ], stdout=diff_file) if os.path.getsize("ngeo.diff") == 0: self.run_step(step + 1) else: self.print_step( step + 1, message= "Manually apply the ngeo application changes as shown in the `ngeo.diff` file.\n" + DIFF_NOTICE) @Step(9) def step9(self, step): if os.path.isfile("ngeo.diff"): os.unlink("ngeo.diff") status = check_output( ["git", "status", "--short", "CONST_create_template"]).decode("utf-8") status = [s for s in status.split("\n") if len(s) > 3] status = [s[3:] for s in status if s.startswith(" M ")] status = [ s for s in status if not s.startswith( "CONST_create_template/{}/templates/".format( self.project["project_package"]), ) ] status = [ s for s in status if not s.startswith( "CONST_create_template/{}/static-ngeo/".format( self.project["project_package"]), ) ] matcher = re.compile(r"CONST_create_tremplate.*/CONST_.+") status = [s for s in status if not matcher.match(s)] status = [ s for s in status if s != "CONST_create_template/package.json" ] status = [ s for s in status if not filecmp.cmp(s, s[len("CONST_create_template/"):]) ] if len(status) > 0: with open("create.diff", "w") as diff_file: check_call(["git", "diff", "--"] + status, stdout=diff_file) if os.path.getsize("create.diff") == 0: self.run_step(step + 1) else: self.print_step( step + 1, message= "This is an optional step but it helps to have a standard project.\n" "The `create.diff` file is a recommendation of the changes that you should apply " "to your project.\n" + DIFF_NOTICE) else: self.run_step(step + 1) @Step(10) def step10(self, step): if os.path.isfile("create.diff"): os.unlink("create.diff") os.environ["IGNORE_I18N_ERRORS"] = "TRUE" check_call( ["make", "--makefile=" + self.options.new_makefile, "build"]) del os.environ["IGNORE_I18N_ERRORS"] check_call([ "git", "checkout", "{0}/locale/*/LC_MESSAGES/{0}-client.po".format( self.project["project_package"]) ]) if self.options.nondocker: command.upgrade(Config("alembic.ini"), "head") command.upgrade(Config("alembic_static.ini"), "head") message = [ "The upgrade is nearly done, now you should:", "- Test your application." ] else: message = [ "The upgrade is nearly done, now you should:", "- run `docker-compose up`", "- Test your application on 'http://localhost:8480/desktop'." ] if self.options.windows: message.append( "You are running on Windows, please restart your Apache server," "because we can not do that automatically.") if os.path.isfile(".upgrade.yaml"): os.unlink(".upgrade.yaml") with open(".UPGRADE_SUCCESS", "w"): pass self.print_step(step + 1, message="\n".join(message)) @Step(11) def step11(self, step): if os.path.isfile(".UPGRADE_SUCCESS"): os.unlink(".UPGRADE_SUCCESS") ok, message = self.test_checkers() if not ok: self.print_step(step, error=True, message=message, prompt="Correct the checker, the it again:") exit(1) # Required to remove from the Git stage the ignored file when we lunch the step again check_call(["git", "reset", "--mixed"]) check_call(["git", "add", "-A"]) check_call(["git", "status"]) self.print_step( step + 1, message="We will commit all the above files!\n" "If there are some files which should not be committed, then you should " "add them into the `.gitignore` file and launch upgrade5 again.", prompt="Then to commit your changes type:") @Step(12) def step12(self, _): check_call([ "git", "commit", "-m", "Upgrade to GeoMapFish {}".format( pkg_resources.get_distribution("c2cgeoportal").version) ]) print("") print(self.color_bar) print("") print(colorize("Congratulations your upgrade is a success.", GREEN)) print("") branch = check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).decode("utf-8").strip() print( "Now all your files will be committed, you should do a git push:") print("git push {0!s} {1!s}.".format(self.options.git_remote, branch))