def delete(self, id_, local_only=False): """Delete the local copy and the origin of a suite. It takes the suite ID as an argument. Return the SuiteId of the suite on success. """ if isinstance(id_, str): id_ = SuiteId(id_text=id_) local_copy = id_.to_local_copy() if os.path.exists(local_copy): if not self.force_mode: status = self.popen("svn", "status", local_copy)[0] if status: raise LocalCopyStatusError(id_, status) if os.getcwd() == local_copy: self.fs_util.chdir(os.path.expanduser("~")) self.fs_util.delete(local_copy) if not local_only: self.popen( "svn", "delete", "-q", "-m", self.COMMIT_MESSAGE_DELETE % str(id_), id_.to_origin(), ) self.event_handler(SuiteDeleteEvent(id_)) return id_
def checkout(self, id_): """Create a local copy of a suite with the given ID. Return the SuiteId of the suite on success. """ if isinstance(id_, str): id_ = SuiteId(id_text=id_) if id_.revision is None: id_.revision = id_.REV_HEAD if id_.branch is None: id_.branch = id_.BRANCH_TRUNK local_copy = id_.to_local_copy() if os.path.exists(local_copy): id0 = SuiteId(location=local_copy) if id_.to_string_with_version() == id0.to_string_with_version(): self.event_handler(LocalCopyCreateSkipEvent(id_)) return id_ elif self.force_mode: self.fs_util.delete(local_copy) else: raise FileExistError(local_copy) local_copy_dir = os.path.dirname(local_copy) if not os.path.isdir(local_copy_dir): self.fs_util.makedirs(os.path.dirname(local_copy)) origin = "%s/%s@%s" % (id_.to_origin(), id_.branch, id_.revision) self.popen("svn", "checkout", "-q", origin, local_copy) self.event_handler(LocalCopyCreateEvent(id_)) return id_
def _copy1(self, info_config, from_id): """Copy a suite from the same repository.""" from_id_url = "%s/%s@%s" % ( from_id.to_origin(), from_id.branch, from_id.revision, ) self.popen("svn", "info", from_id_url) # Die if from_id not exists prefix = from_id.prefix temp_local_copy = os.path.join(self._get_work_dir(), "work") new_id = None # N.B. This is probably the simplest logic to maintain, # but not the most efficient for runtime. Does it matter? while new_id is None: if os.path.exists(temp_local_copy): shutil.rmtree(temp_local_copy) self.popen( "svn", "checkout", "-q", "--depth", "empty", SuiteId.get_prefix_location(prefix), temp_local_copy, ) new_id = SuiteId.get_next(prefix) for i in range(len(new_id.sid)): dir_ = os.path.join( temp_local_copy, os.sep.join(new_id.sid[0 : i + 1]) ) self.popen("svn", "update", "-q", "--depth", "empty", dir_) if not os.path.isdir(dir_): os.mkdir(dir_) self.popen("svn", "add", "-q", dir_) dir_ = os.path.join(temp_local_copy, os.sep.join(new_id.sid)) self.popen( "svn", "cp", "-q", from_id_url, os.path.join(dir_, "trunk") ) metomi.rose.config.dump( info_config, os.path.join(dir_, "trunk", "rose-suite.info") ) message = self.COMMIT_MESSAGE_COPY % ( new_id, from_id.to_string_with_version(), ) try: self.popen( "svn", "commit", "-q", "-m", message, temp_local_copy ) self.event_handler(SuiteCreateEvent(new_id)) self.event_handler(SuiteCopyEvent(new_id, from_id)) except RosePopenError as exc: try: self.popen("svn", "info", new_id.to_origin()) new_id = None except RosePopenError: raise exc finally: self._delete_work_dir() return new_id
def delete(): """CLI function: delete.""" opt_parser = RoseOptionParser(usage='rosie delete [OPTIONS] [--] [ID ...]', description=''' Delete suites. Check the standard working copy location for a checked out suite matching `ID` and remove it if there is no uncommitted change (or if `--force` is specified). Delete the suite directory structure from the HEAD of the central repository matching the `ID`. If no `ID` is specified and `$PWD` is a working copy of a suite, use the `ID` of the suite in the working copy. ''').add_my_options("force_mode", "non_interactive", "local_only") opt_parser.modify_option( 'force_mode', help=("Remove working copies even if there are uncommitted changes." "\nContinue with the next `ID` if delete of a suite fails."), ) opts, args = opt_parser.parse_args() report = Reporter(opts.verbosity - opts.quietness) client = RosieVCClient(event_handler=report, force_mode=opts.force_mode) SuiteId.svn.event_handler = client.event_handler if not args: args.append(SuiteId(location=os.getcwd())) interactive_mode = not opts.non_interactive prompt = PROMPT_DELETE if opts.local_only: prompt = PROMPT_DELETE_LOCAL ret_code = 0 for arg in args: if interactive_mode: try: response = input(prompt % arg) except EOFError: ret_code = 1 continue if response == YES_TO_ALL: interactive_mode = False elif response != YES: ret_code = 1 continue if opts.debug_mode: client.delete(arg, opts.local_only) else: try: client.delete(arg, opts.local_only) except ( LocalCopyStatusError, RosePopenError, SuiteIdPrefixError, ) as exc: client.event_handler(exc) ret_code = 1 if not opts.force_mode: sys.exit(1) if ret_code: sys.exit(ret_code)
def _render(self, all_revs=0, data=None, filters=None, s=None): """Render return data with a template.""" if data: for item in data: suite_id = SuiteId.from_idx_branch_revision( item["idx"], item["branch"], item["revision"]) item["href"] = suite_id.to_web() item["date"] = str( get_timepoint_from_seconds_since_unix_epoch(item["date"])) tmpl = self.props["template_env"].get_template("prefix-index.html") self.write( tmpl.render( title=self.props["title"], host=self.props["host_name"], rose_version=self.props["rose_version"], script="/static", service_root=self.service_root, prefix=self.prefix, prefix_source_url=self.source_url, known_keys=self.dao.get_known_keys(), query_operators=self.dao.get_query_operators(), all_revs=all_revs, filters=filters, s=s, data=data, ))
def query_local_copies(self, user=None): """Returns details of the local suites. As if they had been obtained using a search or query. """ suite_ids = [] for suite_id in SuiteId.get_checked_out_suite_ids(user=user): if suite_id.prefix in self.prefixes: suite_ids.append(suite_id) if not suite_ids: return [] # Simple query results = [] queued_suite_ids = list(suite_ids) while queued_suite_ids: # Batch up queries q_list = [] for _ in range(self.MAX_LOCAL_QUERIES): if not queued_suite_ids: break suite_id = queued_suite_ids.pop() q_list.append("or ( idx eq %s" % suite_id.idx) q_list.append("and branch eq %s )" % suite_id.branch) for data, _ in self.query(q_list): results.extend(data) result_idx_branches = [] for result in results: result_idx_branches.append((result["idx"], result["branch"])) # A branch may have been deleted - query with all_revs=1. # We only want to use all_revs on demand as it's slow. queued_suite_ids = [] for suite_id in suite_ids: if (suite_id.idx, suite_id.branch) not in result_idx_branches: queued_suite_ids.append(suite_id) if not queued_suite_ids: return results while queued_suite_ids: # Batch up queries q_list = [] for _ in range(self.MAX_LOCAL_QUERIES): if not queued_suite_ids: break suite_id = queued_suite_ids.pop() q_list.append("or ( idx eq %s" % suite_id.idx) q_list.append("and branch eq %s )" % suite_id.branch) more_results = [] for data, _ in self.query(q_list, all_revs=1): more_results.extend(data) new_results = {} for result in more_results: idx_branch = (result["idx"], result["branch"]) if (idx_branch not in new_results or result["revision"] > new_results[idx_branch]["revision"]): new_results.update({idx_branch: result}) for _, result in sorted(new_results.items()): results.append(result) return results
def list_local_suites(argv): """CLI command to list all the locally checked out suites""" opt_parser = RoseOptionParser().add_my_options( "no_headers", "prefixes", "print_format", "reverse", "sort", "user") opts = opt_parser.parse_args(argv)[0] report = Reporter(opts.verbosity - opts.quietness) if opts.user: alternative_roses_dir = SuiteId.get_local_copy_root(opts.user) report(UserSpecificRoses(alternative_roses_dir), prefix=None) ws_client = RosieWSClient(prefixes=opts.prefixes, event_handler=report) if ws_client.unreachable_prefixes: bad_prefix_string = " ".join(ws_client.unreachable_prefixes) report( RosieWSClientError( ERR_PREFIX_UNREACHABLE.format(bad_prefix_string))) _display_maps(opts, ws_client, ws_client.query_local_copies(opts.user))
def delete(argv): """CLI function: delete.""" opt_parser = RoseOptionParser().add_my_options( "force_mode", "non_interactive", "local_only" ) opts, args = opt_parser.parse_args(argv) report = Reporter(opts.verbosity - opts.quietness) client = RosieVCClient(event_handler=report, force_mode=opts.force_mode) SuiteId.svn.event_handler = client.event_handler if not args: args.append(SuiteId(location=os.getcwd())) interactive_mode = not opts.non_interactive prompt = PROMPT_DELETE if opts.local_only: prompt = PROMPT_DELETE_LOCAL ret_code = 0 for arg in args: if interactive_mode: try: response = input(prompt % arg) except EOFError: ret_code = 1 continue if response == YES_TO_ALL: interactive_mode = False elif response != YES: ret_code = 1 continue if opts.debug_mode: client.delete(arg, opts.local_only) else: try: client.delete(arg, opts.local_only) except ( LocalCopyStatusError, RosePopenError, SuiteIdPrefixError, ) as exc: client.event_handler(exc) ret_code = 1 if not opts.force_mode: sys.exit(1) if ret_code: sys.exit(ret_code)
def list_local_suites(): """CLI command to list all the locally checked out suites""" opt_parser = RoseOptionParser(description=''' List the local suites. Search for locally checked out suites and print their details. The default format includes a local working copy status field (`%local`) in the first column. A blank field means there is no related suite checked out. * `=` means that the suite is checked out at this branch and revision. * `<` means that the suite is checked out but at an older revision. * `>` means that the suite is checked out but at a newer revision. * `S` means that the suite is checked out but on a different branch. * `M` means that the suite is checked out and modified. * `X` means that the suite is checked out but is corrupted. ''', ).add_my_options("no_headers", "prefixes", "print_format", "reverse", "sort", "user") opt_parser.modify_option( 'verbosity', help=('Display full info for each returned suite.'), ) opts = opt_parser.parse_args()[0] report = Reporter(opts.verbosity - opts.quietness) if opts.user: alternative_roses_dir = SuiteId.get_local_copy_root(opts.user) report(UserSpecificRoses(alternative_roses_dir), prefix=None) ws_client = RosieWSClient(prefixes=opts.prefixes, event_handler=report) if ws_client.unreachable_prefixes: bad_prefix_string = " ".join(ws_client.unreachable_prefixes) report( RosieWSClientError( ERR_PREFIX_UNREACHABLE.format(bad_prefix_string))) _display_maps(opts, ws_client, ws_client.query_local_copies(opts.user))
def create(): """CLI function: create and copy.""" opt_parser = RoseOptionParser( usage=('rosie create [OPTIONS]' '\n rosie copy [OPTIONS] ID-OF-EXISTING-SUITE'), description=''' rosie create: Create a new suite rosie copy : Create a new suite and copy content from an existing one. Assign a new `ID` and create the directory structure in the central repository for a new suite. The location of the repository for the new suite is determined in order of preference: 1. `--prefix=PREFIX` option 2. prefix of the `ID-OF-EXISTING-SUITE` 3. `[rosie-id]prefix-default` option in the site/user configuration. If `ID-OF-EXISTING-SUITE` is specified, copy items from the existing suite `ID-OF-EXISTING-SUITE` when the suite is created. It is worth noting that revision history of the copied items can only be preserved if `ID-OF-EXISTING-SUITE` is in the same repository of the new suite The syntax of the ID-OF-EXISTING-SUITE is PREFIX-xxNNN[/BRANCH][@REV] (e.g. my-su173, my-su173/trunk, my-su173/trunk@HEAD). If REV is not specified, the last changed revision of the branch is used. If BRANCH is not specified, "trunk" is used. NOTE: ID-OF-EXISTING-SUITE is _not_ a filepath. ''', ) opt_parser.add_my_options( "checkout_mode", "info_file", "meta_suite_mode", "non_interactive", "prefix", "project", ) opts, args = opt_parser.parse_args() verbosity = opts.verbosity - opts.quietness client = RosieVCClient(event_handler=Reporter(verbosity)) SuiteId.svn.event_handler = client.event_handler from_id = None if args: from_id = SuiteId(id_text=args[0]) if from_id.branch is None: from_id.branch = from_id.BRANCH_TRUNK if from_id.revision is None: from_id.revision = from_id.REV_HEAD from_id = SuiteId(id_text=from_id.to_string_with_version()) interactive_mode = not opts.non_interactive if opts.info_file is None: info_config = client.generate_info_config(from_id, opts.prefix, opts.project) if from_id is not None: meta_config = load_meta_config( info_config, directory=None, config_type=metomi.rose.INFO_CONFIG_NAME, error_handler=None, ignore_meta_error=False, ) for node_keys, node in meta_config.walk(no_ignore=True): if isinstance(node.value, dict): continue sect, key = node_keys value = node.value sect = sect.replace("=", "") if key == "copy-mode" and value == "clear": info_config.set([sect], "") if key == "copy-mode" and value == "never": info_config.unset([sect]) info_config = _edit_info_config(opts, client, info_config) else: file_ = opts.info_file if opts.info_file == "-": file_ = sys.stdin info_config = metomi.rose.config.load(file_) info_config = _validate_info_config(opts, client, info_config) if interactive_mode: prefix = opts.prefix if from_id: if not prefix: prefix = from_id.prefix question = PROMPT_COPY % (from_id.to_string_with_version(), prefix) else: if not prefix: prefix = SuiteId.get_prefix_default() question = PROMPT_CREATE % prefix try: response = input(question) except EOFError: sys.exit(1) if response != YES: sys.exit(1) try: id_ = client.create(info_config, from_id, opts.prefix, opts.meta_suite_mode) except (RosePopenError, SuiteIdOverflowError) as exc: client.event_handler(exc) sys.exit(1) if opts.checkout_mode: try: client.checkout(id_) except (FileExistError, RosePopenError) as exc: client.event_handler(exc) sys.exit(1)
def generate_info_config(self, from_id=None, prefix=None, project=None): """Generate a metomi.rose.config.ConfigNode for a rose-suite.info. This is suitable for passing into the create method of this class. If from_id is defined, copy items from it. Return the metomi.rose.config.ConfigNode instance. """ from_project = None from_title = None if from_id is not None: from_info_url = "%s/%s/rose-suite.info@%s" % ( from_id.to_origin(), from_id.branch, from_id.revision, ) out_data = self.popen("svn", "cat", from_info_url)[0] from_config = metomi.rose.config.load(StringIO(out_data)) res_loc = ResourceLocator.default() older_config = None info_config = metomi.rose.config.ConfigNode() # Determine project if given as a command-line option on create if from_id is None and project is not None: info_config.set(["project"], project) # Set the compulsory fields and use the project and metadata if # available. meta_config = load_meta_config( info_config, config_type=metomi.rose.INFO_CONFIG_NAME) if from_id is None and project is not None: for node_keys, node in meta_config.walk(no_ignore=True): if isinstance(node.value, dict): continue sect, key = node_keys value = node.value sect = sect.translate(None, "=") if key == "compulsory" and value == "true": info_config.set([sect], "") info_config.set(["project"], project) else: if from_project is None: info_config.set(["project"], "") if from_title is None: info_config.set(["title"], "") # Determine prefix if prefix is None: if from_id is None: prefix = SuiteId.get_prefix_default() else: prefix = from_id.prefix # Determine owner: # 1. From user configuration [rosie-id]prefix-username # 2. From username of a matching group in [groups] in # ~/.subversion/servers # 3. Current user ID owner = res_loc.get_conf().get_value( ["rosie-id", "prefix-username." + prefix]) if not owner and self.subversion_servers_conf: servers_conf = metomi.rose.config.load( self.subversion_servers_conf) groups_node = servers_conf.get(["groups"]) if groups_node is not None: prefix_loc = SuiteId.get_prefix_location(prefix) prefix_host = urlparse(prefix_loc).hostname for key, node in groups_node.value.items(): if fnmatch(prefix_host, node.value): owner = servers_conf.get_value([key, "username"]) break if not owner: owner = pwd.getpwuid(os.getuid())[0] info_config.set(["owner"], owner) # Copy description try: from_id.to_string_with_version() info_config.set( ["description"], "Copy of %s" % (from_id.to_string_with_version()), ) except AttributeError: pass # Copy fields provided by the user try: from_config.walk(no_ignore=False) for node_keys, node in from_config.walk(no_ignore=False): if isinstance(node.value, dict): continue sect, key = node_keys value = node.value if key in [ "description", "owner", "access-list" ] or (key == "project" and from_project is not None): pass else: info_config.set([key], value) except UnboundLocalError: pass # Determine access list access_list_str = res_loc.get_conf().get_value( ["rosie-vc", "access-list-default"]) if access_list_str: info_config.set(["access-list"], access_list_str) if from_id is None and project is not None: for node_keys, node in meta_config.walk(no_ignore=True): if isinstance(node.value, dict): continue sect, key = node_keys value = node.value sect = sect.translate(None, "=") if key == "value-hints" or key == "values": reminder = ("please remove all commented hints/lines " + "in the main/top section before saving.") info_config.set( [sect], metomi.rose.variable.array_split(value)[0], comments=[value, reminder], ) if older_config is not None: for node_keys, node in older_config.walk(no_ignore=True): if isinstance(node.value, dict): continue sect, key = node_keys value = node.value info_config.set([key], value) return info_config
def create(self, info_config, from_id=None, prefix=None, meta_suite_mode=False): """Create a suite. info_config -- A metomi.rose.config.ConfigNode object, which will be used as the content of the "rose-suite.info" file of the new suite. from_id -- If defined, copy items from it. prefix -- If defined, create the suite in the suite repository named by the prefix instead of the default one. meta_suite_mode -- If True, create the special metadata suite. Ignored if from_id is not None. Return the SuiteId of the suite on success. """ if from_id is not None and (not prefix or from_id.prefix == prefix): return self._copy1(info_config, from_id) dir_ = self._get_work_dir() try: # Create a temporary suite in the file system if from_id: from_id_url = "%s/%s@%s" % ( from_id.to_origin(), from_id.branch, from_id.revision, ) self.popen("svn", "export", "-q", "--force", from_id_url, dir_) else: open(os.path.join(dir_, "rose-suite.conf"), "w").close() metomi.rose.config.dump(info_config, os.path.join(dir_, "rose-suite.info")) # Attempt to import the temporary suite to the repository new_id = None while new_id is None: if meta_suite_mode: if prefix is None: new_id = SuiteId(id_text="ROSIE") else: idx = SuiteId.FORMAT_IDX % (prefix, "ROSIE") new_id = SuiteId(id_text=idx) else: new_id = SuiteId.get_next(prefix) new_origin = new_id.to_origin() + "/" + new_id.BRANCH_TRUNK try: if from_id: message = self.COMMIT_MESSAGE_COPY % ( new_id, from_id.to_string_with_version(), ) else: message = self.COMMIT_MESSAGE_CREATE % str(new_id) self.popen("svn", "import", "-q", "-m", message, dir_, new_origin) self.event_handler(SuiteCreateEvent(new_id)) if from_id: self.event_handler(SuiteCopyEvent(new_id, from_id)) except RosePopenError as exc: try: self.popen("svn", "info", new_origin) if not meta_suite_mode: new_id = None except RosePopenError: raise exc return new_id finally: self._delete_work_dir()
def test_parse_cylc_vc_file(vcs_info: dict, expected: Optional[str], tmp_path: Path): vcs_file = tmp_path / 'gimli.json' vcs_file.write_text(json.dumps(vcs_info)) assert SuiteId._parse_cylc_vc_file(str(vcs_file)) == expected
def create(argv): """CLI function: create and copy.""" opt_parser = RoseOptionParser() opt_parser.add_my_options("checkout_mode", "info_file", "meta_suite_mode", "non_interactive", "prefix", "project") opts, args = opt_parser.parse_args(argv) verbosity = opts.verbosity - opts.quietness client = RosieVCClient(event_handler=Reporter(verbosity)) SuiteId.svn.event_handler = client.event_handler from_id = None if args: from_id = SuiteId(id_text=args[0]) if from_id.branch is None: from_id.branch = from_id.BRANCH_TRUNK if from_id.revision is None: from_id.revision = from_id.REV_HEAD from_id = SuiteId(id_text=from_id.to_string_with_version()) interactive_mode = not opts.non_interactive if opts.info_file is None: info_config = client.generate_info_config(from_id, opts.prefix, opts.project) if from_id is not None: meta_config = load_meta_config( info_config, directory=None, config_type=metomi.rose.INFO_CONFIG_NAME, error_handler=None, ignore_meta_error=False) for node_keys, node in meta_config.walk(no_ignore=True): if isinstance(node.value, dict): continue sect, key = node_keys value = node.value sect = sect.replace("=", "") if key == "copy-mode" and value == "clear": info_config.set([sect], "") if key == "copy-mode" and value == "never": info_config.unset([sect]) info_config = _edit_info_config(opts, client, info_config) else: file_ = opts.info_file if opts.info_file == "-": file_ = sys.stdin info_config = metomi.rose.config.load(file_) info_config = _validate_info_config(opts, client, info_config) if interactive_mode: prefix = opts.prefix if from_id: if not prefix: prefix = from_id.prefix question = PROMPT_COPY % (from_id.to_string_with_version(), prefix) else: if not prefix: prefix = SuiteId.get_prefix_default() question = PROMPT_CREATE % prefix try: response = input(question) except EOFError: sys.exit(1) if response != YES: sys.exit(1) try: id_ = client.create(info_config, from_id, opts.prefix, opts.meta_suite_mode) except (RosePopenError, SuiteIdOverflowError) as exc: client.event_handler(exc) sys.exit(1) if opts.checkout_mode: try: client.checkout(id_) except (FileExistError, RosePopenError) as exc: client.event_handler(exc) sys.exit(1)
def _display_maps(opts, ws_client, dict_rows, url=None): """Display returned suite details.""" report = ws_client.event_handler try: terminal_cols = int(ws_client.popen("stty", "size")[0].split()[1]) except (IndexError, RosePopenError, ValueError): terminal_cols = None if terminal_cols == 0: terminal_cols = None if opts.quietness and not opts.print_format: opts.print_format = PRINT_FORMAT_QUIET elif not opts.print_format: opts.print_format = PRINT_FORMAT_DEFAULT all_keys = ws_client.get_known_keys() for dict_row in dict_rows: suite_id = SuiteId.from_idx_branch_revision(dict_row["idx"], dict_row["branch"], dict_row["revision"]) dict_row["suite"] = suite_id.to_string_with_version() if "%local" in opts.print_format: dict_row["local"] = suite_id.get_status(getattr( opts, "user", None)) all_keys += ["suite"] if "%local" in opts.print_format: all_keys += ["local"] more_keys = [] for key in REC_COL_IN_FORMAT.findall(opts.print_format): if key not in all_keys: more_keys.append(key) all_keys += more_keys if opts.sort is None or opts.sort not in all_keys: opts.sort = "revision" dict_rows.sort(key=lambda x: x[opts.sort]) if opts.reverse: dict_rows.reverse() keylist = [] for key in all_keys: if "%" + key in opts.print_format: keylist.append(key) if not opts.no_headers: dummy_row = {} for key in all_keys: dummy_row[key] = key dict_rows.insert(0, dummy_row) dict_rows = _align(dict_rows, keylist) for dict_row in dict_rows: out = opts.print_format for key, value in dict_row.items(): if "%" + key in out: out = str(out).replace("%" + str(key), str(value), 1) out = str(out.replace("%%", "%").expandtabs().rstrip()) report(SuiteEvent(out.expandtabs() + "\n"), prefix="", clip=terminal_cols) report(SuiteInfo(dict_row), prefix="") if url is not None: report(URLEvent(url + "\n"), prefix="")