def _coerce_cycleinterval(value, keys, _): """Coerce value to a cycle interval.""" if not value: return None value = _strip_and_unquote(keys, value) if value.isdigit(): # Old runahead limit format. set_syntax_version(VERSION_PREV, "integer interval for %s" % itemstr( keys[:-1], keys[-1], value)) return value if REC_INTEGER_INTERVAL.match(value): # New integer cycling format. set_syntax_version(VERSION_NEW, "integer interval for %s" % itemstr( keys[:-1], keys[-1], value)) return value parser = DurationParser() try: parser.parse(value) except ValueError: raise IllegalValueError("interval", keys, value) set_syntax_version(VERSION_NEW, "ISO 8601 interval for %s" % itemstr( keys[:-1], keys[-1], value)) return value
def _coerce_cycleinterval(value, keys, _): """Coerce value to a cycle interval.""" if not value: return None value = _strip_and_unquote(keys, value) if value.isdigit(): # Old runahead limit format. set_syntax_version( VERSION_PREV, "integer interval for %s" % itemstr(keys[:-1], keys[-1], value)) return value if REC_INTEGER_INTERVAL.match(value): # New integer cycling format. set_syntax_version( VERSION_NEW, "integer interval for %s" % itemstr(keys[:-1], keys[-1], value)) return value parser = DurationParser() try: parser.parse(value) except ValueError: raise IllegalValueError("interval", keys, value) set_syntax_version( VERSION_NEW, "ISO 8601 interval for %s" % itemstr(keys[:-1], keys[-1], value)) return value
def _coerce_cycleinterval(value, keys, _): """Coerce value to a cycle interval.""" if not value: return None value = _strip_and_unquote(keys, value) parser = DurationParser() try: parser.parse(value) except ValueError: raise IllegalValueError("interval", keys, value) return value
def coerce_cycle_point(cls, value, keys): """Coerce value to a cycle point.""" if not value: return None value = cls.strip_and_unquote(keys, value) if value == 'now': # Handle this later in config.py when the suite UTC mode is known. return value if "next" in value or "previous" in value: # Handle this later, as for "now". return value if value.isdigit(): # Could be an old date-time cycle point format, or integer format. return value if "P" not in value and ( value.startswith('-') or value.startswith('+')): # We don't know the value given for num expanded year digits... for i in range(1, 101): try: TimePointParser(num_expanded_year_digits=i).parse(value) except ValueError: continue return value raise IllegalValueError('cycle point', keys, value) if "P" in value: # ICP is an offset parser = DurationParser() try: if value.startswith("-"): # parser doesn't allow negative duration with this setup? parser.parse(value[1:]) else: parser.parse(value) return value except ValueError: raise IllegalValueError("cycle point", keys, value) try: TimePointParser().parse(value) except ValueError: raise IllegalValueError('cycle point', keys, value) return value
def add_offset(cycle_point, offset): """Add a (positive or negative) offset to a cycle point. Return the result. """ my_parser = TimePointParser() my_target_point = my_parser.parse(cycle_point, dump_as_parsed=True) my_offset_parser = DurationParser() oper = "+" if offset.startswith("-") or offset.startswith("+"): oper = offset[0] offset = offset[1:] if offset.startswith("P"): my_shift = my_offset_parser.parse(offset) if oper == "-": my_target_point -= my_shift else: my_target_point += my_shift else: raise ValueError("ERROR, bad offset format: %s" % offset) return my_target_point
class RoseDateTimeOperator(object): """A class to parse and print date string with an offset.""" CURRENT_TIME_DUMP_FORMAT = u"CCYY-MM-DDThh:mm:ss+hh:mm" CURRENT_TIME_DUMP_FORMAT_Z = u"CCYY-MM-DDThh:mm:ssZ" NEGATIVE = "-" # strptime formats and their compatibility with the ISO 8601 parser. PARSE_FORMATS = [ ("%a %b %d %H:%M:%S %Y", True), # ctime ("%a %b %d %H:%M:%S %Z %Y", True), # Unix "date" ("%Y-%m-%dT%H:%M:%S", False), # ISO8601, extended ("%Y%m%dT%H%M%S", False), # ISO8601, basic ("%Y%m%d%H", False) # Cylc (current) ] REC_OFFSET = re.compile(r"""\A[\+\-]?(?:\d+[wdhms])+\Z""", re.I) REC_OFFSET_FIND = re.compile(r"""(?P<num>\d+)(?P<unit>[wdhms])""") STR_NOW = "now" STR_REF = "ref" TASK_CYCLE_TIME_ENV = "ROSE_TASK_CYCLE_TIME" UNITS = { "w": "weeks", "d": "days", "h": "hours", "m": "minutes", "s": "seconds" } def __init__(self, parse_format=None, utc_mode=False, calendar_mode=None, ref_point_str=None): """Constructor. parse_format -- If specified, parse with the specified format. Otherwise, parse with one of the format strings in self.PARSE_FORMATS. The format should be a string compatible to strptime(3). utc_mode -- If True, parse/print in UTC mode rather than local or other timezones. calendar_mode -- Set calendar mode, for isodatetime.data.Calendar. ref_point_str -- Set the reference time point for operations. If not specified, operations use current date time. """ self.parse_formats = self.PARSE_FORMATS self.custom_parse_format = parse_format self.utc_mode = utc_mode if self.utc_mode: assumed_time_zone = (0, 0) else: assumed_time_zone = None self.set_calendar_mode(calendar_mode) self.time_point_dumper = TimePointDumper() self.time_point_parser = TimePointParser( assumed_time_zone=assumed_time_zone) self.duration_parser = DurationParser() self.ref_point_str = ref_point_str def date_format(self, print_format, time_point=None): """Reformat time_point according to print_format. time_point -- The time point to format. Otherwise, use ref date time. """ if time_point is None: time_point = self.date_parse()[0] if print_format is None: return str(time_point) if "%" in print_format: try: return time_point.strftime(print_format) except ValueError: return self.get_datetime_strftime(time_point, print_format) return self.time_point_dumper.dump(time_point, print_format) def date_parse(self, time_point_str=None): """Parse time_point_str. Return (t, format) where t is a isodatetime.data.TimePoint object and format is the format that matches time_point_str. time_point_str -- The time point string to parse. Otherwise, use ref time. """ if time_point_str is None or time_point_str == self.STR_REF: time_point_str = self.ref_point_str if time_point_str is None or time_point_str == self.STR_NOW: time_point = get_timepoint_for_now() time_point.set_time_zone_to_local() if self.utc_mode or time_point.get_time_zone_utc(): # is in UTC parse_format = self.CURRENT_TIME_DUMP_FORMAT_Z else: parse_format = self.CURRENT_TIME_DUMP_FORMAT elif self.custom_parse_format is not None: parse_format = self.custom_parse_format time_point = self.strptime(time_point_str, parse_format) else: parse_formats = list(self.parse_formats) time_point = None while parse_formats: parse_format, should_use_datetime = parse_formats.pop(0) try: if should_use_datetime: time_point = self.get_datetime_strptime( time_point_str, parse_format) else: time_point = self.time_point_parser.strptime( time_point_str, parse_format) break except ValueError: pass if time_point is None: time_point = self.time_point_parser.parse(time_point_str, dump_as_parsed=True) parse_format = time_point.dump_format if self.utc_mode: time_point.set_time_zone_to_utc() return time_point, parse_format def date_shift(self, time_point=None, offset=None): """Return a date string with an offset. time_point -- A time point or time point string. Otherwise, use current time. offset -- If specified, it should be a string containing the offset that has the format "[+/-]nU[nU...]" where "n" is an integer, and U is a unit matching a key in self.UNITS. """ if time_point is None: time_point = self.date_parse()[0] # Offset if offset: sign = "+" if offset.startswith("-") or offset.startswith("+"): sign = offset[0] offset = offset[1:] if offset.startswith("P"): # Parse and apply. try: duration = self.duration_parser.parse(offset) except ValueError: raise OffsetValueError(offset) if sign == "-": time_point -= duration else: time_point += duration else: # Backwards compatibility for e.g. "-1h" if not self.is_offset(offset): raise OffsetValueError(offset) for num, unit in self.REC_OFFSET_FIND.findall(offset.lower()): num = int(num) if sign == "-": num = -num key = self.UNITS[unit] time_point += Duration(**{key: num}) return time_point def date_diff(self, time_point_1=None, time_point_2=None): """Return (duration, is_negative) between two TimePoint objects. duration -- is a Duration instance. is_negative -- is a RoseDateTimeOperator.NEGATIVE if time_point_2 is in the past of time_point_1. """ if time_point_2 < time_point_1: return (time_point_1 - time_point_2, self.NEGATIVE) else: return (time_point_2 - time_point_1, "") @classmethod def date_diff_format(cls, print_format, duration, sign): """Format a duration.""" if print_format: delta_lookup = { "y": duration.years, "m": duration.months, "d": duration.days, "h": duration.hours, "M": duration.minutes, "s": duration.seconds } expression = "" for item in print_format: if item in delta_lookup: if float(delta_lookup[item]).is_integer(): expression += str(int(delta_lookup[item])) else: expression += str(delta_lookup[item]) else: expression += item return sign + expression else: return sign + str(duration) @staticmethod def get_calendar_mode(): """Get current calendar mode.""" return Calendar.default().mode def is_offset(self, offset): """Return True if the string offset can be parsed as an offset.""" return (self.REC_OFFSET.match(offset) is not None) @staticmethod def set_calendar_mode(calendar_mode=None): """Set calendar mode for subsequent operations. Raise KeyError if calendar_mode is invalid. """ if not calendar_mode: calendar_mode = os.getenv("ROSE_CYCLING_MODE") if calendar_mode and calendar_mode in Calendar.MODES: Calendar.default().set_mode(calendar_mode) def strftime(self, time_point, print_format): """Use either the isodatetime or datetime strftime time formatting.""" try: return time_point.strftime(print_format) except ValueError: return self.get_datetime_strftime(time_point, print_format) def strptime(self, time_point_str, parse_format): """Use either the isodatetime or datetime strptime time parsing.""" try: return self.time_point_parser.strptime(time_point_str, parse_format) except ValueError: return self.get_datetime_strptime(time_point_str, parse_format) @classmethod def get_datetime_strftime(cls, time_point, print_format): """Use the datetime library's strftime as a fallback.""" calendar_date = time_point.copy().to_calendar_date() year, month, day = calendar_date.get_calendar_date() hour, minute, second = time_point.get_hour_minute_second() microsecond = int(1.0e6 * (second - int(second))) hour = int(hour) minute = int(minute) second = int(second) date_time = datetime(year, month, day, hour, minute, second, microsecond) return date_time.strftime(print_format) def get_datetime_strptime(self, time_point_str, parse_format): """Use the datetime library's strptime as a fallback.""" date_time = datetime.strptime(time_point_str, parse_format) return self.time_point_parser.parse(date_time.isoformat())
class AppRunner(Runner): """Invoke a Rose application.""" OLD_DURATION_UNITS = {"h": 3600, "m": 60, "s": 1} NAME = "app" OPTIONS = ["app_mode", "command_key", "conf_dir", "defines", "install_only_mode", "new_mode", "no_overwrite_mode", "opt_conf_keys"] def __init__(self, *args, **kwargs): Runner.__init__(self, *args, **kwargs) path = os.path.dirname(os.path.dirname(sys.modules["rose"].__file__)) self.builtins_manager = SchemeHandlersManager( [path], "rose.apps", ["run"], None, *args, **kwargs) self.duration_parser = DurationParser() def run_impl(self, opts, args, uuid, work_files): """The actual logic for a run.""" # Preparation. conf_tree = self.config_load(opts) self._prep(conf_tree, opts) self._poll(conf_tree) # Run the application or the command. app_mode = conf_tree.node.get_value(["mode"]) if app_mode is None: app_mode = opts.app_mode if app_mode in [None, "command"]: return self._command(conf_tree, opts, args) else: builtin_app = self.builtins_manager.get_handler(app_mode) if builtin_app is None: raise UnknownBuiltinAppError(app_mode) return builtin_app.run(self, conf_tree, opts, args, uuid, work_files) def _poll(self, conf_tree): """Poll for prerequisites of applications.""" # Poll configuration poll_test = conf_tree.node.get_value(["poll", "test"]) poll_all_files_value = conf_tree.node.get_value(["poll", "all-files"]) poll_all_files = [] if poll_all_files_value: try: poll_all_files = shlex.split( env_var_process(poll_all_files_value)) except UnboundEnvironmentVariableError as exc: raise ConfigValueError(["poll", "all-files"], poll_all_files_value, exc) poll_any_files_value = conf_tree.node.get_value(["poll", "any-files"]) poll_any_files = [] if poll_any_files_value: try: poll_any_files = shlex.split( env_var_process(poll_any_files_value)) except UnboundEnvironmentVariableError as exc: raise ConfigValueError(["poll", "any-files"], poll_any_files_value, exc) poll_file_test = None if poll_all_files or poll_any_files: poll_file_test = conf_tree.node.get_value(["poll", "file-test"]) if poll_file_test and "{}" not in poll_file_test: raise ConfigValueError(["poll", "file-test"], poll_file_test, ConfigValueError.SYNTAX) poll_delays = [] if poll_test or poll_all_files or poll_any_files: # Parse something like this: delays=10,4*PT30S,PT2M30S,2*PT1H # R*DURATION: repeat the value R times conf_keys = ["poll", "delays"] poll_delays_value = conf_tree.node.get_value( conf_keys, default="").strip() if poll_delays_value: is_legacy0 = None for item in poll_delays_value.split(","): value = item.strip() repeat = 1 if "*" in value: repeat, value = value.split("*", 1) try: repeat = int(repeat) except ValueError as exc: raise ConfigValueError(conf_keys, poll_delays_value, ConfigValueError.SYNTAX) try: value = self.duration_parser.parse(value).get_seconds() is_legacy = False except ISO8601SyntaxError: # Legacy mode: nnnU # nnn is a float, U is the unit # No unit or s: seconds # m: minutes # h: hours unit = None if value[-1].lower() in self.OLD_DURATION_UNITS: unit = self.OLD_DURATION_UNITS[value[-1].lower()] value = value[:-1] try: value = float(value) except ValueError as exc: raise ConfigValueError(conf_keys, poll_delays_value, ConfigValueError.SYNTAX) if unit: value *= unit is_legacy = True if is_legacy0 is None: is_legacy0 = is_legacy elif is_legacy0 != is_legacy: raise ConfigValueError( conf_keys, poll_delays_value, ConfigValueError.DURATION_LEGACY_MIX) poll_delays += [value] * repeat else: poll_delays = [0] # poll once without a delay # Poll t_init = get_timepoint_for_now() while poll_delays and (poll_test or poll_any_files or poll_all_files): poll_delay = poll_delays.pop(0) if poll_delay: sleep(poll_delay) if poll_test: ret_code = self.popen.run( poll_test, shell=True, stdout=sys.stdout, stderr=sys.stderr)[0] self.handle_event(PollEvent(time(), poll_test, ret_code == 0)) if ret_code == 0: poll_test = None any_files = list(poll_any_files) for file_ in any_files: if self._poll_file(file_, poll_file_test): self.handle_event(PollEvent(time(), "any-files", True)) poll_any_files = [] break all_files = list(poll_all_files) for file_ in all_files: if self._poll_file(file_, poll_file_test): poll_all_files.remove(file_) if all_files and not poll_all_files: self.handle_event(PollEvent(time(), "all-files", True)) failed_items = [] if poll_test: failed_items.append("test") if poll_any_files: failed_items.append("any-files") if poll_all_files: failed_items.append("all-files:" + self.popen.list_to_shell_str(poll_all_files)) if failed_items: now = get_timepoint_for_now() raise PollTimeoutError(now, now - t_init, failed_items) def _poll_file(self, file_, poll_file_test): """Poll for existence of a file.""" is_done = False if poll_file_test: test = poll_file_test.replace( "{}", self.popen.list_to_shell_str([file_])) is_done = self.popen.run( test, shell=True, stdout=sys.stdout, stderr=sys.stderr)[0] == 0 else: is_done = bool(glob(file_)) self.handle_event(PollEvent(time(), "file:" + file_, is_done)) return is_done def _prep(self, conf_tree, opts): """Prepare to run the application.""" if opts.new_mode: conf_dir = opts.conf_dir if not conf_dir or os.path.abspath(conf_dir) == os.getcwd(): raise NewModeError(os.getcwd()) for path in os.listdir("."): self.fs_util.delete(path) # Dump the actual configuration as rose-app-run.conf ConfigDumper()(conf_tree.node, "rose-app-run.conf") # Environment variables: PATH paths = [] for conf_dir in conf_tree.conf_dirs: conf_bin_dir = os.path.join(conf_dir, "bin") if os.path.isdir(conf_bin_dir): paths.append(conf_bin_dir) if paths: value = os.pathsep.join(paths + [os.getenv("PATH")]) conf_tree.node.set(["env", "PATH"], value) else: conf_tree.node.set(["env", "PATH"], os.getenv("PATH")) # Free format files not defined in the configuration file file_section_prefix = self.config_pm.get_handler("file").PREFIX for rel_path, conf_dir in conf_tree.files.items(): if not rel_path.startswith("file" + os.sep): continue name = rel_path[len("file" + os.sep):] # No sub-directories, very slow otherwise if os.sep in name: name = name.split(os.sep, 1)[0] target_key = file_section_prefix + name target_node = conf_tree.node.get([target_key]) if target_node is None: conf_tree.node.set([target_key]) target_node = conf_tree.node.get([target_key]) elif target_node.is_ignored(): continue source_node = target_node.get("source") if source_node is None: target_node.set(["source"], os.path.join(conf_dir, "file", name)) elif source_node.is_ignored(): continue # Process Environment Variables self.config_pm(conf_tree, "env") # Process Files self.config_pm(conf_tree, "file", no_overwrite_mode=opts.no_overwrite_mode) def _command(self, conf_tree, opts, args): """Run the command.""" command = self.popen.list_to_shell_str(args) if not command: names = [opts.command_key, os.getenv("ROSE_TASK_NAME"), "default"] for name in names: if not name: continue command = conf_tree.node.get_value(["command", name]) if command is not None: break else: self.handle_event(CommandNotDefinedEvent()) return if os.access("STDIN", os.F_OK | os.R_OK): command += " <STDIN" self.handle_event("command: %s" % command) if opts.install_only_mode: return self.popen(command, shell=True, stdout=sys.stdout, stderr=sys.stderr)
class RoseDateTimeOperator(object): """A class to parse and print date string with an offset.""" CURRENT_TIME_DUMP_FORMAT = u"CCYY-MM-DDThh:mm:ss+hh:mm" CURRENT_TIME_DUMP_FORMAT_Z = u"CCYY-MM-DDThh:mm:ssZ" NEGATIVE = "-" # strptime formats and their compatibility with the ISO 8601 parser. PARSE_FORMATS = [ ("%a %b %d %H:%M:%S %Y", True), # ctime ("%a %b %d %H:%M:%S %Z %Y", True), # Unix "date" ("%Y-%m-%dT%H:%M:%S", False), # ISO8601, extended ("%Y%m%dT%H%M%S", False), # ISO8601, basic ("%Y%m%d%H", False) # Cylc (current) ] REC_OFFSET = re.compile(r"""\A[\+\-]?(?:\d+[wdhms])+\Z""", re.I) REC_OFFSET_FIND = re.compile(r"""(?P<num>\d+)(?P<unit>[wdhms])""") STR_NOW = "now" STR_REF = "ref" TASK_CYCLE_TIME_ENV = "ROSE_TASK_CYCLE_TIME" UNITS = {"w": "weeks", "d": "days", "h": "hours", "m": "minutes", "s": "seconds"} def __init__(self, parse_format=None, utc_mode=False, calendar_mode=None, ref_point_str=None): """Constructor. parse_format -- If specified, parse with the specified format. Otherwise, parse with one of the format strings in self.PARSE_FORMATS. The format should be a string compatible to strptime(3). utc_mode -- If True, parse/print in UTC mode rather than local or other timezones. calendar_mode -- Set calendar mode, for isodatetime.data.Calendar. ref_point_str -- Set the reference time point for operations. If not specified, operations use current date time. """ self.parse_formats = self.PARSE_FORMATS self.custom_parse_format = parse_format self.utc_mode = utc_mode if self.utc_mode: assumed_time_zone = (0, 0) else: assumed_time_zone = None self.set_calendar_mode(calendar_mode) self.time_point_dumper = TimePointDumper() self.time_point_parser = TimePointParser( assumed_time_zone=assumed_time_zone) self.duration_parser = DurationParser() self.ref_point_str = ref_point_str def date_format(self, print_format, time_point=None): """Reformat time_point according to print_format. time_point -- The time point to format. Otherwise, use ref date time. """ if time_point is None: time_point = self.date_parse()[0] if print_format is None: return str(time_point) if "%" in print_format: try: return time_point.strftime(print_format) except ValueError: return self.get_datetime_strftime(time_point, print_format) return self.time_point_dumper.dump(time_point, print_format) def date_parse(self, time_point_str=None): """Parse time_point_str. Return (t, format) where t is a isodatetime.data.TimePoint object and format is the format that matches time_point_str. time_point_str -- The time point string to parse. Otherwise, use ref time. """ if time_point_str is None or time_point_str == self.STR_REF: time_point_str = self.ref_point_str if time_point_str is None or time_point_str == self.STR_NOW: time_point = get_timepoint_for_now() time_point.set_time_zone_to_local() if self.utc_mode or time_point.get_time_zone_utc(): # is in UTC parse_format = self.CURRENT_TIME_DUMP_FORMAT_Z else: parse_format = self.CURRENT_TIME_DUMP_FORMAT elif self.custom_parse_format is not None: parse_format = self.custom_parse_format time_point = self.strptime(time_point_str, parse_format) else: parse_formats = list(self.parse_formats) time_point = None while parse_formats: parse_format, should_use_datetime = parse_formats.pop(0) try: if should_use_datetime: time_point = self.get_datetime_strptime( time_point_str, parse_format) else: time_point = self.time_point_parser.strptime( time_point_str, parse_format) break except ValueError: pass if time_point is None: time_point = self.time_point_parser.parse( time_point_str, dump_as_parsed=True) parse_format = time_point.dump_format if self.utc_mode: time_point.set_time_zone_to_utc() return time_point, parse_format def date_shift(self, time_point=None, offset=None): """Return a date string with an offset. time_point -- A time point or time point string. Otherwise, use current time. offset -- If specified, it should be a string containing the offset that has the format "[+/-]nU[nU...]" where "n" is an integer, and U is a unit matching a key in self.UNITS. """ if time_point is None: time_point = self.date_parse()[0] # Offset if offset: sign = "+" if offset.startswith("-") or offset.startswith("+"): sign = offset[0] offset = offset[1:] if offset.startswith("P"): # Parse and apply. try: duration = self.duration_parser.parse(offset) except ValueError: raise OffsetValueError(offset) if sign == "-": time_point -= duration else: time_point += duration else: # Backwards compatibility for e.g. "-1h" if not self.is_offset(offset): raise OffsetValueError(offset) for num, unit in self.REC_OFFSET_FIND.findall(offset.lower()): num = int(num) if sign == "-": num = -num key = self.UNITS[unit] time_point += Duration(**{key: num}) return time_point def date_diff(self, time_point_1=None, time_point_2=None): """Return (duration, is_negative) between two TimePoint objects. duration -- is a Duration instance. is_negative -- is a RoseDateTimeOperator.NEGATIVE if time_point_2 is in the past of time_point_1. """ if time_point_2 < time_point_1: return (time_point_1 - time_point_2, self.NEGATIVE) else: return (time_point_2 - time_point_1, "") @classmethod def date_diff_format(cls, print_format, duration, sign): """Format a duration.""" if print_format: delta_lookup = {"y": duration.years, "m": duration.months, "d": duration.days, "h": duration.hours, "M": duration.minutes, "s": duration.seconds} expression = "" for item in print_format: if item in delta_lookup: if float(delta_lookup[item]).is_integer(): expression += str(int(delta_lookup[item])) else: expression += str(delta_lookup[item]) else: expression += item return sign + expression else: return sign + str(duration) @staticmethod def get_calendar_mode(): """Get current calendar mode.""" return Calendar.default().mode def is_offset(self, offset): """Return True if the string offset can be parsed as an offset.""" return (self.REC_OFFSET.match(offset) is not None) @staticmethod def set_calendar_mode(calendar_mode=None): """Set calendar mode for subsequent operations. Raise KeyError if calendar_mode is invalid. """ if not calendar_mode: calendar_mode = os.getenv("ROSE_CYCLING_MODE") if calendar_mode and calendar_mode in Calendar.MODES: Calendar.default().set_mode(calendar_mode) def strftime(self, time_point, print_format): """Use either the isodatetime or datetime strftime time formatting.""" try: return time_point.strftime(print_format) except ValueError: return self.get_datetime_strftime(time_point, print_format) def strptime(self, time_point_str, parse_format): """Use either the isodatetime or datetime strptime time parsing.""" try: return self.time_point_parser.strptime(time_point_str, parse_format) except ValueError: return self.get_datetime_strptime(time_point_str, parse_format) @classmethod def get_datetime_strftime(cls, time_point, print_format): """Use the datetime library's strftime as a fallback.""" calendar_date = time_point.copy().to_calendar_date() year, month, day = calendar_date.get_calendar_date() hour, minute, second = time_point.get_hour_minute_second() microsecond = int(1.0e6 * (second - int(second))) hour = int(hour) minute = int(minute) second = int(second) date_time = datetime(year, month, day, hour, minute, second, microsecond) return date_time.strftime(print_format) def get_datetime_strptime(self, time_point_str, parse_format): """Use the datetime library's strptime as a fallback.""" date_time = datetime.strptime(time_point_str, parse_format) return self.time_point_parser.parse(date_time.isoformat())
class ControlTree(object): """Text Treeview suite control interface.""" def __init__(self, cfg, updater, theme, dot_size, info_bar, get_right_click_menu, log_colors, insert_task_popup): self.cfg = cfg self.updater = updater self.theme = theme self.dot_size = dot_size self.info_bar = info_bar self.get_right_click_menu = get_right_click_menu self.log_colors = log_colors self.insert_task_popup = insert_task_popup self.interval_parser = DurationParser() self.gcapture_windows = [] self.ttree_paths = {} # Cache dict of tree paths & states, names. def get_control_widgets(self): main_box = gtk.VBox() main_box.pack_start(self.treeview_widgets(), expand=True, fill=True) self.t = TreeUpdater( self.cfg, self.updater, self.ttreeview, self.ttree_paths, self.info_bar, self.theme, self.dot_size ) self.t.start() return main_box def toggle_grouping(self, toggle_item): """Toggle grouping by visualisation families.""" group_on = toggle_item.get_active() if group_on == self.t.should_group_families: return False if group_on: if "text" in self.cfg.ungrouped_views: self.cfg.ungrouped_views.remove("text") elif "text" not in self.cfg.ungrouped_views: self.cfg.ungrouped_views.append("text") self.t.should_group_families = group_on if isinstance(toggle_item, gtk.ToggleToolButton): if group_on: tip_text = "Tree View - Click to ungroup families" else: tip_text = "Tree View - Click to group tasks by families" self._set_tooltip(toggle_item, tip_text) self.group_menu_item.set_active(group_on) else: if toggle_item != self.group_menu_item: self.group_menu_item.set_active(group_on) self.group_toolbutton.set_active(group_on) self.t.update_gui() return False def stop(self): self.t.quit = True def toggle_autoexpand(self, w): self.t.autoexpand = not self.t.autoexpand def treeview_widgets(self): self.sort_col_num = 0 self.ttreestore = gtk.TreeStore( str, str, str, str, str, str, str, str, str, str, str, gtk.gdk.Pixbuf, int) self.ttreeview = gtk.TreeView() self.ttreeview.set_rules_hint(True) # TODO - REMOVE FILTER HERE? self.tmodelfilter = self.ttreestore.filter_new() self.tmodelsort = gtk.TreeModelSort(self.tmodelfilter) self.ttreeview.set_model(self.tmodelsort) ts = self.ttreeview.get_selection() ts.set_mode(gtk.SELECTION_SINGLE) self.ttreeview.connect( 'button_press_event', self.on_treeview_button_pressed) headings = [ None, 'task', 'state', 'host', 'job system', 'job ID', 'T-submit', 'T-start', 'T-finish', 'dT-mean', 'latest message', ] for n in range(1, len(headings)): # Skip first column (cycle point) tvc = gtk.TreeViewColumn(headings[n]) if n == 1: crp = gtk.CellRendererPixbuf() tvc.pack_start(crp, False) tvc.set_attributes(crp, pixbuf=11) if n == 8: # Pack in progress and text cell renderers. prog_cr = gtk.CellRendererProgress() tvc.pack_start(prog_cr, True) tvc.set_cell_data_func(prog_cr, self._set_cell_text_time, n) cr = gtk.CellRendererText() tvc.pack_start(cr, True) if n == 6 or n == 7 or n == 8: tvc.set_cell_data_func(cr, self._set_cell_text_time, n) else: tvc.set_attributes(cr, text=n) tvc.set_resizable(True) tvc.set_clickable(True) self.ttreeview.append_column(tvc) tvc.set_sort_column_id(n - 1) self.tmodelsort.set_sort_func(n - 1, self.sort_column, n - 1) sw = gtk.ScrolledWindow() sw.set_policy(gtk.POLICY_AUTOMATIC, gtk.POLICY_AUTOMATIC) sw.add(self.ttreeview) vbox = gtk.VBox() vbox.pack_start(sw, True) return vbox def on_treeview_button_pressed(self, treeview, event): # DISPLAY MENU ONLY ON RIGHT CLICK ONLY if event.button != 3: return False # the following sets selection to the position at which the # right click was done (otherwise selection lags behind the # right click): x = int(event.x) y = int(event.y) time = event.time pth = treeview.get_path_at_pos(x, y) if pth is None: return False treeview.grab_focus() path, col, cellx, celly = pth treeview.set_cursor(path, col, 0) selection = treeview.get_selection() treemodel, iter = selection.get_selected() point_string = treemodel.get_value(iter, 0) name = treemodel.get_value(iter, 1) if point_string == name: # must have clicked on the top level point_string return task_id = TaskID.get(name, point_string) is_fam = (name in self.t.descendants) if is_fam: task_state = self.t.fam_state_summary[task_id]['state'] submit_num = None else: task_state = self.t.state_summary[task_id]['state'] submit_num = self.t.state_summary[task_id]['submit_num'] menu = self.get_right_click_menu( task_id, t_state=task_state, task_is_family=is_fam, submit_num=submit_num) sep = gtk.SeparatorMenuItem() sep.show() menu.append(sep) group_item = gtk.CheckMenuItem('Toggle Family Grouping') group_item.set_active(self.t.should_group_families) menu.append(group_item) group_item.connect('toggled', self.toggle_grouping) group_item.show() menu.popup(None, None, None, event.button, event.time) # TODO - popup menus are not automatically destroyed and can be # reused if saved; however, we need to reconstruct or at least # alter ours dynamically => should destroy after each use to # prevent a memory leak? But I'm not sure how to do this as yet.) return True def sort_column(self, model, iter1, iter2, col_num): cols = self.ttreeview.get_columns() point_string1 = model.get_value(iter1, 0) point_string2 = model.get_value(iter2, 0) if point_string1 != point_string2: # TODO ISO: worth a proper comparison here? if cols[col_num].get_sort_order() == gtk.SORT_DESCENDING: return cmp(point_string2, point_string1) return cmp(point_string1, point_string2) # Columns do not include the cycle point (0th col), so add 1. if (col_num + 1) == 9: prop1 = (model.get_value(iter1, col_num + 1)) prop2 = (model.get_value(iter2, col_num + 1)) prop1 = self._get_interval_in_seconds(prop1) prop2 = self._get_interval_in_seconds(prop2) else: prop1 = model.get_value(iter1, col_num + 1) prop2 = model.get_value(iter2, col_num + 1) return cmp(prop1, prop2) def _get_interval_in_seconds(self, val): """Convert the IOS 8601 date/time to seconds.""" if val == "*" or val == "": secsout = val else: interval = self.interval_parser.parse(val) seconds = interval.get_seconds() secsout = seconds return secsout def change_sort_order(self, col, event=None, n=0): if hasattr(event, "button") and event.button != 1: return False cols = self.ttreeview.get_columns() self.sort_col_num = n if cols[n].get_sort_order() == gtk.SORT_ASCENDING: cols[n].set_sort_order(gtk.SORT_DESCENDING) else: cols[n].set_sort_order(gtk.SORT_ASCENDING) return False def on_popup_quit(self, b, lv, w): lv.quit() self.quitters.remove(lv) w.destroy() def refresh(self): self.t.update_gui() def get_menuitems(self): """Return the menu items specific to this view.""" items = [] autoex_item = gtk.CheckMenuItem('Toggle _Auto-Expand Tree') autoex_item.set_active(self.t.autoexpand) items.append(autoex_item) autoex_item.connect('activate', self.toggle_autoexpand) self.group_menu_item = gtk.CheckMenuItem('Toggle _Family Grouping') self.group_menu_item.set_active(self.t.should_group_families) items.append(self.group_menu_item) self.group_menu_item.connect('toggled', self.toggle_grouping) return items def _set_tooltip(self, widget, tip_text): """Convenience function to add hover over text to a widget.""" tip = gtk.Tooltips() tip.enable() tip.set_tip(widget, tip_text) def _set_cell_text_time(self, column, cell, model, iter_, n): """Remove the date part if it matches the last update date.""" date_time_string = model.get_value(iter_, n) if "T" in self.updater.dt: last_update_date = self.updater.dt.split("T")[0] date_time_string = date_time_string.replace( last_update_date + "T", "", 1) if n == 8: # Progress bar for estimated completion time. if isinstance(cell, gtk.CellRendererText): if date_time_string.endswith("?"): # Task running -show progress bar instead. cell.set_property('visible', False) else: # Task not running - just show text cell.set_property('visible', True) cell.set_property('text', date_time_string) if isinstance(cell, gtk.CellRendererProgress): if date_time_string.endswith("?"): # Task running -show progress bar to estimated finish time. cell.set_property('visible', True) percent = model.get_value(iter_, 12) cell.set_property('value', percent) else: # Task not running - show text cell instead. cell.set_property('visible', False) cell.set_property('value', 0) cell.set_property("text", date_time_string) def get_toolitems(self): """Return the tool bar items specific to this view.""" items = [] expand_button = gtk.ToolButton() image = gtk.image_new_from_stock( gtk.STOCK_ADD, gtk.ICON_SIZE_SMALL_TOOLBAR) expand_button.set_icon_widget(image) expand_button.set_label("Expand") self._set_tooltip(expand_button, "Tree View - Expand all") expand_button.connect('clicked', lambda x: self.ttreeview.expand_all()) items.append(expand_button) collapse_button = gtk.ToolButton() image = gtk.image_new_from_stock( gtk.STOCK_REMOVE, gtk.ICON_SIZE_SMALL_TOOLBAR) collapse_button.set_icon_widget(image) collapse_button.set_label("Collapse") collapse_button.connect( 'clicked', lambda x: self.ttreeview.collapse_all()) self._set_tooltip(collapse_button, "Tree View - Collapse all") items.append(collapse_button) self.group_toolbutton = gtk.ToggleToolButton() self.group_toolbutton.set_active(self.t.should_group_families) g_image = gtk.image_new_from_stock( 'group', gtk.ICON_SIZE_SMALL_TOOLBAR) self.group_toolbutton.set_icon_widget(g_image) self.group_toolbutton.set_label("Group") self.group_toolbutton.connect('toggled', self.toggle_grouping) self._set_tooltip( self.group_toolbutton, "Tree View - Click to group tasks by families") items.append(self.group_toolbutton) return items
class ControlTree(object): """Text Treeview suite control interface.""" def __init__(self, cfg, updater, theme, dot_size, info_bar, get_right_click_menu, insert_task_popup): self.cfg = cfg self.updater = updater self.theme = theme self.dot_size = dot_size self.info_bar = info_bar self.get_right_click_menu = get_right_click_menu self.insert_task_popup = insert_task_popup self.interval_parser = DurationParser() self.gcapture_windows = [] self.ttree_paths = {} # Cache dict of tree paths & states, names. self.group_toolbutton = None self.group_menu_item = None self.tmodelfilter = None self.t = None self.sort_col_num = None self.tmodelsort = None self.ttreeview = None self.ttreestore = None def get_control_widgets(self): main_box = gtk.VBox() main_box.pack_start(self.treeview_widgets(), expand=True, fill=True) self.t = TreeUpdater(self.cfg, self.updater, self.ttreeview, self.ttree_paths, self.info_bar, self.theme, self.dot_size) self.t.start() return main_box def toggle_grouping(self, toggle_item): """Toggle grouping by visualisation families.""" group_on = toggle_item.get_active() if group_on == self.t.should_group_families: return False if group_on: if "text" in self.cfg.ungrouped_views: self.cfg.ungrouped_views.remove("text") elif "text" not in self.cfg.ungrouped_views: self.cfg.ungrouped_views.append("text") self.t.should_group_families = group_on if isinstance(toggle_item, gtk.ToggleToolButton): if group_on: tip_text = "Tree View - Click to ungroup families" else: tip_text = "Tree View - Click to group tasks by families" self._set_tooltip(toggle_item, tip_text) self.group_menu_item.set_active(group_on) else: if toggle_item != self.group_menu_item: self.group_menu_item.set_active(group_on) self.group_toolbutton.set_active(group_on) self.t.update_gui() return False def stop(self): self.t.quit = True def toggle_autoexpand(self, w): self.t.autoexpand = not self.t.autoexpand def treeview_widgets(self): self.sort_col_num = 0 self.ttreestore = gtk.TreeStore(str, str, str, str, str, str, str, str, str, str, str, gtk.gdk.Pixbuf, int) self.ttreeview = gtk.TreeView() self.ttreeview.set_rules_hint(True) # TODO - REMOVE FILTER HERE? self.tmodelfilter = self.ttreestore.filter_new() self.tmodelsort = gtk.TreeModelSort(self.tmodelfilter) self.ttreeview.set_model(self.tmodelsort) # multiple selection ts = self.ttreeview.get_selection() self.ttreeview.set_rubber_banding(True) if ts: ts.set_mode(gtk.SELECTION_MULTIPLE) self.ttreeview.connect('button_press_event', self.on_treeview_button_pressed) for n in range(1, len(HEADINGS)): # Skip first column (cycle point) tvc = gtk.TreeViewColumn(HEADINGS[n]) if n == 1: crp = gtk.CellRendererPixbuf() tvc.pack_start(crp, False) tvc.set_attributes(crp, pixbuf=11) if n == 8: # Pack in progress and text cell renderers. prog_cr = gtk.CellRendererProgress() tvc.pack_start(prog_cr, True) tvc.set_cell_data_func(prog_cr, self._set_cell_text_time, n) cr = gtk.CellRendererText() tvc.pack_start(cr, True) if n == 6 or n == 7 or n == 8: tvc.set_cell_data_func(cr, self._set_cell_text_time, n) else: tvc.set_attributes(cr, text=n) tvc.set_resizable(True) tvc.set_clickable(True) self.ttreeview.append_column(tvc) tvc.set_sort_column_id(n - 1) self.tmodelsort.set_sort_func(n - 1, self.sort_column, n - 1) sw = gtk.ScrolledWindow() sw.set_policy(gtk.POLICY_AUTOMATIC, gtk.POLICY_AUTOMATIC) sw.add(self.ttreeview) vbox = gtk.VBox() vbox.pack_start(sw, True) return vbox def on_treeview_button_pressed(self, treeview, event): # DISPLAY MENU ONLY ON RIGHT CLICK ONLY if event.button != 3: return False # If clicking on a task that is not selected, set the selection to be # that task. x = int(event.x) y = int(event.y) pth = treeview.get_path_at_pos(x, y) if pth is None: return False treeview.grab_focus() path, col = pth[:2] tvte = TreeViewTaskExtractor(treeview) if path not in (row[0] for row in tvte.get_selected_rows()): treeview.set_cursor(path, col, 0) # Populate lists of task info from the selected tasks. task_ids = [] t_states = [] task_is_family = [] # List of boolean values. for task in tvte.get_selected_tasks(): # get_selected_tasks() does not return tasks if their parent node # is also returned, i.e. no duplicates. point_string, name = task if point_string == name: name = 'root' task_id = TaskID.get(name, point_string) task_ids.append(task_id) is_fam = (name in self.t.descendants) task_is_family.append(is_fam) if is_fam: if task_id not in self.t.fam_state_summary: return False t_states.append(self.t.fam_state_summary[task_id]['state']) else: if task_id not in self.t.state_summary: return False t_states.append(self.t.state_summary[task_id]['state']) menu = self.get_right_click_menu(task_ids, t_states, task_is_family=task_is_family) sep = gtk.SeparatorMenuItem() sep.show() menu.append(sep) group_item = gtk.CheckMenuItem('Toggle Family Grouping') group_item.set_active(self.t.should_group_families) menu.append(group_item) group_item.connect('toggled', self.toggle_grouping) group_item.show() menu.popup(None, None, None, event.button, event.time) # TODO - popup menus are not automatically destroyed and can be # reused if saved; however, we need to reconstruct or at least # alter ours dynamically => should destroy after each use to # prevent a memory leak? But I'm not sure how to do this as yet.) return True def sort_by_column(self, col_name=None, col_no=None, ascending=True): """Sort this ControlTree by the column selected by the string col_name OR by the index col_no.""" if col_name is not None and col_name in HEADINGS: col_no = HEADINGS.index(col_name) if col_no is not None: self.sort_col_num = col_no cols = self.ttreeview.get_columns() order = gtk.SORT_ASCENDING if ascending else gtk.SORT_DESCENDING cols[col_no].set_sort_order(order) self.tmodelsort.set_sort_column_id(col_no - 1, order) def sort_column(self, model, iter1, iter2, col_num): cols = self.ttreeview.get_columns() point_string1 = model.get_value(iter1, 0) point_string2 = model.get_value(iter2, 0) if point_string1 != point_string2: # TODO ISO: worth a proper comparison here? if cols[col_num].get_sort_order() == gtk.SORT_DESCENDING: return cmp(point_string2, point_string1) return cmp(point_string1, point_string2) # Columns do not include the cycle point (0th col), so add 1. prop1 = model.get_value(iter1, col_num + 1) prop2 = model.get_value(iter2, col_num + 1) if col_num == 8: # dT-mean column, convert intervals to seconds prop1 = self._get_interval_in_seconds(prop1) prop2 = self._get_interval_in_seconds(prop2) return cmp(prop1, prop2) def _get_interval_in_seconds(self, val): """Convert the IOS 8601 date/time to seconds.""" try: return self.interval_parser.parse(str(val)).get_seconds() except ISO8601SyntaxError: return 0.0 def change_sort_order(self, col, event=None, n=0): if hasattr(event, "button") and event.button != 1: return False cols = self.ttreeview.get_columns() self.sort_col_num = n if cols[n].get_sort_order() == gtk.SORT_ASCENDING: cols[n].set_sort_order(gtk.SORT_DESCENDING) else: cols[n].set_sort_order(gtk.SORT_ASCENDING) return False def on_popup_quit(self, b, lv, w): lv.quit() w.destroy() def refresh(self): self.t.update_gui() self.t.action_required = True def get_menuitems(self): """Return the menu items specific to this view.""" items = [] autoex_item = gtk.CheckMenuItem('Toggle _Auto-Expand Tree') autoex_item.set_active(self.t.autoexpand) items.append(autoex_item) autoex_item.connect('activate', self.toggle_autoexpand) self.group_menu_item = gtk.CheckMenuItem('Toggle _Family Grouping') self.group_menu_item.set_active(self.t.should_group_families) items.append(self.group_menu_item) self.group_menu_item.connect('toggled', self.toggle_grouping) return items @staticmethod def _set_tooltip(widget, tip_text): """Convenience function to add hover over text to a widget.""" tip = gtk.Tooltips() tip.enable() tip.set_tip(widget, tip_text) def _set_cell_text_time(self, column, cell, model, iter_, n): """Remove the date part if it matches the last update date.""" date_time_string = model.get_value(iter_, n) if date_time_string is None: return if "T" in self.updater.update_time_str: last_update_date = self.updater.update_time_str.split("T")[0] date_time_string = date_time_string.replace( last_update_date + "T", "", 1) if n == 8: # Progress bar for estimated completion time. if isinstance(cell, gtk.CellRendererText): if date_time_string.endswith("?"): # Task running -show progress bar instead. cell.set_property('visible', False) else: # Task not running - just show text cell.set_property('visible', True) cell.set_property('text', date_time_string) if isinstance(cell, gtk.CellRendererProgress): if date_time_string.endswith("?"): # Task running -show progress bar to estimated finish time. cell.set_property('visible', True) percent = model.get_value(iter_, 12) cell.set_property('value', percent) else: # Task not running - show text cell instead. cell.set_property('visible', False) cell.set_property('value', 0) cell.set_property("text", date_time_string) def get_toolitems(self): """Return the tool bar items specific to this view.""" items = [] expand_button = gtk.ToolButton() image = gtk.image_new_from_stock(gtk.STOCK_ADD, gtk.ICON_SIZE_SMALL_TOOLBAR) expand_button.set_icon_widget(image) expand_button.set_label("Expand") self._set_tooltip(expand_button, "Tree View - Expand all") expand_button.connect('clicked', lambda x: self.ttreeview.expand_all()) items.append(expand_button) collapse_button = gtk.ToolButton() image = gtk.image_new_from_stock(gtk.STOCK_REMOVE, gtk.ICON_SIZE_SMALL_TOOLBAR) collapse_button.set_icon_widget(image) collapse_button.set_label("Collapse") collapse_button.connect('clicked', lambda x: self.ttreeview.collapse_all()) self._set_tooltip(collapse_button, "Tree View - Collapse all") items.append(collapse_button) self.group_toolbutton = gtk.ToggleToolButton() self.group_toolbutton.set_active(self.t.should_group_families) g_image = gtk.image_new_from_stock('group', gtk.ICON_SIZE_SMALL_TOOLBAR) self.group_toolbutton.set_icon_widget(g_image) self.group_toolbutton.set_label("Group") self.group_toolbutton.connect('toggled', self.toggle_grouping) self._set_tooltip(self.group_toolbutton, "Tree View - Click to group tasks by families") items.append(self.group_toolbutton) return items
class AppRunner(Runner): """Invoke a Rose application.""" OLD_DURATION_UNITS = {"h": 3600, "m": 60, "s": 1} NAME = "app" OPTIONS = [ "app_mode", "command_key", "conf_dir", "defines", "install_only_mode", "new_mode", "no_overwrite_mode", "opt_conf_keys" ] def __init__(self, *args, **kwargs): Runner.__init__(self, *args, **kwargs) path = os.path.dirname(os.path.dirname(sys.modules["rose"].__file__)) self.builtins_manager = SchemeHandlersManager([path], "rose.apps", ["run"], None, *args, **kwargs) self.duration_parser = DurationParser() def run_impl(self, opts, args, uuid, work_files): """The actual logic for a run.""" # Preparation. conf_tree = self.config_load(opts) self._prep(conf_tree, opts) self._poll(conf_tree) # Run the application or the command. app_mode = conf_tree.node.get_value(["mode"]) if app_mode is None: app_mode = opts.app_mode if app_mode in [None, "command"]: return self._command(conf_tree, opts, args) else: builtin_app = self.builtins_manager.get_handler(app_mode) if builtin_app is None: raise UnknownBuiltinAppError(app_mode) return builtin_app.run(self, conf_tree, opts, args, uuid, work_files) def _poll(self, conf_tree): """Poll for prerequisites of applications.""" # Poll configuration poll_test = conf_tree.node.get_value(["poll", "test"]) poll_all_files_value = conf_tree.node.get_value(["poll", "all-files"]) poll_all_files = [] if poll_all_files_value: try: poll_all_files = shlex.split( env_var_process(poll_all_files_value)) except UnboundEnvironmentVariableError as exc: raise ConfigValueError(["poll", "all-files"], poll_all_files_value, exc) poll_any_files_value = conf_tree.node.get_value(["poll", "any-files"]) poll_any_files = [] if poll_any_files_value: try: poll_any_files = shlex.split( env_var_process(poll_any_files_value)) except UnboundEnvironmentVariableError as exc: raise ConfigValueError(["poll", "any-files"], poll_any_files_value, exc) poll_file_test = None if poll_all_files or poll_any_files: poll_file_test = conf_tree.node.get_value(["poll", "file-test"]) if poll_file_test and "{}" not in poll_file_test: raise ConfigValueError(["poll", "file-test"], poll_file_test, ConfigValueError.SYNTAX) poll_delays = [] if poll_test or poll_all_files or poll_any_files: # Parse something like this: delays=10,4*PT30S,PT2M30S,2*PT1H # R*DURATION: repeat the value R times conf_keys = ["poll", "delays"] poll_delays_value = conf_tree.node.get_value(conf_keys, default="").strip() if poll_delays_value: is_legacy0 = None for item in poll_delays_value.split(","): value = item.strip() repeat = 1 if "*" in value: repeat, value = value.split("*", 1) try: repeat = int(repeat) except ValueError as exc: raise ConfigValueError(conf_keys, poll_delays_value, ConfigValueError.SYNTAX) try: value = self.duration_parser.parse(value).get_seconds() is_legacy = False except ISO8601SyntaxError: # Legacy mode: nnnU # nnn is a float, U is the unit # No unit or s: seconds # m: minutes # h: hours unit = None if value[-1].lower() in self.OLD_DURATION_UNITS: unit = self.OLD_DURATION_UNITS[value[-1].lower()] value = value[:-1] try: value = float(value) except ValueError as exc: raise ConfigValueError(conf_keys, poll_delays_value, ConfigValueError.SYNTAX) if unit: value *= unit is_legacy = True if is_legacy0 is None: is_legacy0 = is_legacy elif is_legacy0 != is_legacy: raise ConfigValueError( conf_keys, poll_delays_value, ConfigValueError.DURATION_LEGACY_MIX) poll_delays += [value] * repeat else: poll_delays = [0] # poll once without a delay # Poll t_init = get_timepoint_for_now() while poll_delays and (poll_test or poll_any_files or poll_all_files): poll_delay = poll_delays.pop(0) if poll_delay: sleep(poll_delay) if poll_test: ret_code = self.popen.run(poll_test, shell=True, stdout=sys.stdout, stderr=sys.stderr)[0] self.handle_event(PollEvent(time(), poll_test, ret_code == 0)) if ret_code == 0: poll_test = None any_files = list(poll_any_files) for file_ in any_files: if self._poll_file(file_, poll_file_test): self.handle_event(PollEvent(time(), "any-files", True)) poll_any_files = [] break all_files = list(poll_all_files) for file_ in all_files: if self._poll_file(file_, poll_file_test): poll_all_files.remove(file_) if all_files and not poll_all_files: self.handle_event(PollEvent(time(), "all-files", True)) failed_items = [] if poll_test: failed_items.append("test") if poll_any_files: failed_items.append("any-files") if poll_all_files: failed_items.append("all-files:" + self.popen.list_to_shell_str(poll_all_files)) if failed_items: now = get_timepoint_for_now() raise PollTimeoutError(now, now - t_init, failed_items) def _poll_file(self, file_, poll_file_test): """Poll for existence of a file.""" is_done = False if poll_file_test: test = poll_file_test.replace( "{}", self.popen.list_to_shell_str([file_])) is_done = self.popen.run(test, shell=True, stdout=sys.stdout, stderr=sys.stderr)[0] == 0 else: is_done = bool(glob(file_)) self.handle_event(PollEvent(time(), "file:" + file_, is_done)) return is_done def _prep(self, conf_tree, opts): """Prepare to run the application.""" if opts.new_mode: conf_dir = opts.conf_dir if not conf_dir or os.path.abspath(conf_dir) == os.getcwd(): raise NewModeError(os.getcwd()) for path in os.listdir("."): self.fs_util.delete(path) # Dump the actual configuration as rose-app-run.conf ConfigDumper()(conf_tree.node, "rose-app-run.conf") # Environment variables: PATH paths = [] for conf_dir in conf_tree.conf_dirs: conf_bin_dir = os.path.join(conf_dir, "bin") if os.path.isdir(conf_bin_dir): paths.append(conf_bin_dir) if paths: value = os.pathsep.join(paths + [os.getenv("PATH")]) conf_tree.node.set(["env", "PATH"], value) else: conf_tree.node.set(["env", "PATH"], os.getenv("PATH")) # Free format files not defined in the configuration file file_section_prefix = self.config_pm.get_handler("file").PREFIX for rel_path, conf_dir in conf_tree.files.items(): if not rel_path.startswith("file" + os.sep): continue name = rel_path[len("file" + os.sep):] # No sub-directories, very slow otherwise if os.sep in name: name = name.split(os.sep, 1)[0] target_key = file_section_prefix + name target_node = conf_tree.node.get([target_key]) if target_node is None: conf_tree.node.set([target_key]) target_node = conf_tree.node.get([target_key]) elif target_node.is_ignored(): continue source_node = target_node.get("source") if source_node is None: target_node.set(["source"], os.path.join(conf_dir, "file", name)) elif source_node.is_ignored(): continue # Process Environment Variables self.config_pm(conf_tree, "env") # Process Files self.config_pm(conf_tree, "file", no_overwrite_mode=opts.no_overwrite_mode) def _command(self, conf_tree, opts, args): """Run the command.""" command = self.popen.list_to_shell_str(args) if not command: names = [opts.command_key, os.getenv("ROSE_TASK_NAME"), "default"] for name in names: if not name: continue command = conf_tree.node.get_value(["command", name]) if command is not None: break else: self.handle_event(CommandNotDefinedEvent()) return if os.access("STDIN", os.F_OK | os.R_OK): command += " <STDIN" self.handle_event("command: %s" % command) if opts.install_only_mode: return self.popen(command, shell=True, stdout=sys.stdout, stderr=sys.stderr)
class ControlTree(object): """Text Treeview suite control interface.""" def __init__(self, cfg, updater, theme, dot_size, info_bar, get_right_click_menu, log_colors, insert_task_popup): self.cfg = cfg self.updater = updater self.theme = theme self.dot_size = dot_size self.info_bar = info_bar self.get_right_click_menu = get_right_click_menu self.log_colors = log_colors self.insert_task_popup = insert_task_popup self.interval_parser = DurationParser() self.gcapture_windows = [] self.ttree_paths = {} # Cache dict of tree paths & states, names. def get_control_widgets(self): main_box = gtk.VBox() main_box.pack_start(self.treeview_widgets(), expand=True, fill=True) self.t = TreeUpdater(self.cfg, self.updater, self.ttreeview, self.ttree_paths, self.info_bar, self.theme, self.dot_size) self.t.start() return main_box def toggle_grouping(self, toggle_item): """Toggle grouping by visualisation families.""" group_on = toggle_item.get_active() if group_on == self.t.should_group_families: return False if group_on: if "text" in self.cfg.ungrouped_views: self.cfg.ungrouped_views.remove("text") elif "text" not in self.cfg.ungrouped_views: self.cfg.ungrouped_views.append("text") self.t.should_group_families = group_on if isinstance(toggle_item, gtk.ToggleToolButton): if group_on: tip_text = "Tree View - Click to ungroup families" else: tip_text = "Tree View - Click to group tasks by families" self._set_tooltip(toggle_item, tip_text) self.group_menu_item.set_active(group_on) else: if toggle_item != self.group_menu_item: self.group_menu_item.set_active(group_on) self.group_toolbutton.set_active(group_on) self.t.update_gui() return False def stop(self): self.t.quit = True def toggle_autoexpand(self, w): self.t.autoexpand = not self.t.autoexpand def treeview_widgets(self): self.sort_col_num = 0 self.ttreestore = gtk.TreeStore(str, str, str, str, str, str, str, str, str, str, str, gtk.gdk.Pixbuf, int) self.ttreeview = gtk.TreeView() self.ttreeview.set_rules_hint(True) # TODO - REMOVE FILTER HERE? self.tmodelfilter = self.ttreestore.filter_new() self.tmodelsort = gtk.TreeModelSort(self.tmodelfilter) self.ttreeview.set_model(self.tmodelsort) ts = self.ttreeview.get_selection() ts.set_mode(gtk.SELECTION_SINGLE) self.ttreeview.connect('button_press_event', self.on_treeview_button_pressed) headings = [ None, 'task', 'state', 'host', 'job system', 'job ID', 'T-submit', 'T-start', 'T-finish', 'dT-mean', 'latest message', ] for n in range(1, len(headings)): # Skip first column (cycle point) tvc = gtk.TreeViewColumn(headings[n]) if n == 1: crp = gtk.CellRendererPixbuf() tvc.pack_start(crp, False) tvc.set_attributes(crp, pixbuf=11) if n == 8: # Pack in progress and text cell renderers. prog_cr = gtk.CellRendererProgress() tvc.pack_start(prog_cr, True) tvc.set_cell_data_func(prog_cr, self._set_cell_text_time, n) cr = gtk.CellRendererText() tvc.pack_start(cr, True) if n == 6 or n == 7 or n == 8: tvc.set_cell_data_func(cr, self._set_cell_text_time, n) else: tvc.set_attributes(cr, text=n) tvc.set_resizable(True) tvc.set_clickable(True) self.ttreeview.append_column(tvc) tvc.set_sort_column_id(n - 1) self.tmodelsort.set_sort_func(n - 1, self.sort_column, n - 1) sw = gtk.ScrolledWindow() sw.set_policy(gtk.POLICY_AUTOMATIC, gtk.POLICY_AUTOMATIC) sw.add(self.ttreeview) vbox = gtk.VBox() vbox.pack_start(sw, True) return vbox def on_treeview_button_pressed(self, treeview, event): # DISPLAY MENU ONLY ON RIGHT CLICK ONLY if event.button != 3: return False # the following sets selection to the position at which the # right click was done (otherwise selection lags behind the # right click): x = int(event.x) y = int(event.y) time = event.time pth = treeview.get_path_at_pos(x, y) if pth is None: return False treeview.grab_focus() path, col, cellx, celly = pth treeview.set_cursor(path, col, 0) selection = treeview.get_selection() treemodel, iter = selection.get_selected() point_string = treemodel.get_value(iter, 0) name = treemodel.get_value(iter, 1) if point_string == name: # must have clicked on the top level point_string return task_id = TaskID.get(name, point_string) is_fam = (name in self.t.descendants) if is_fam: task_state = self.t.fam_state_summary[task_id]['state'] submit_num = None else: task_state = self.t.state_summary[task_id]['state'] submit_num = self.t.state_summary[task_id]['submit_num'] menu = self.get_right_click_menu(task_id, t_state=task_state, task_is_family=is_fam, submit_num=submit_num) sep = gtk.SeparatorMenuItem() sep.show() menu.append(sep) group_item = gtk.CheckMenuItem('Toggle Family Grouping') group_item.set_active(self.t.should_group_families) menu.append(group_item) group_item.connect('toggled', self.toggle_grouping) group_item.show() menu.popup(None, None, None, event.button, event.time) # TODO - popup menus are not automatically destroyed and can be # reused if saved; however, we need to reconstruct or at least # alter ours dynamically => should destroy after each use to # prevent a memory leak? But I'm not sure how to do this as yet.) return True def sort_column(self, model, iter1, iter2, col_num): cols = self.ttreeview.get_columns() point_string1 = model.get_value(iter1, 0) point_string2 = model.get_value(iter2, 0) if point_string1 != point_string2: # TODO ISO: worth a proper comparison here? if cols[col_num].get_sort_order() == gtk.SORT_DESCENDING: return cmp(point_string2, point_string1) return cmp(point_string1, point_string2) # Columns do not include the cycle point (0th col), so add 1. if (col_num + 1) == 9: prop1 = (model.get_value(iter1, col_num + 1)) prop2 = (model.get_value(iter2, col_num + 1)) prop1 = self._get_interval_in_seconds(prop1) prop2 = self._get_interval_in_seconds(prop2) else: prop1 = model.get_value(iter1, col_num + 1) prop2 = model.get_value(iter2, col_num + 1) return cmp(prop1, prop2) def _get_interval_in_seconds(self, val): """Convert the IOS 8601 date/time to seconds.""" if val == "*" or val == "": secsout = val else: interval = self.interval_parser.parse(val) seconds = interval.get_seconds() secsout = seconds return secsout def change_sort_order(self, col, event=None, n=0): if hasattr(event, "button") and event.button != 1: return False cols = self.ttreeview.get_columns() self.sort_col_num = n if cols[n].get_sort_order() == gtk.SORT_ASCENDING: cols[n].set_sort_order(gtk.SORT_DESCENDING) else: cols[n].set_sort_order(gtk.SORT_ASCENDING) return False def on_popup_quit(self, b, lv, w): lv.quit() self.quitters.remove(lv) w.destroy() def refresh(self): self.t.update_gui() def get_menuitems(self): """Return the menu items specific to this view.""" items = [] autoex_item = gtk.CheckMenuItem('Toggle _Auto-Expand Tree') autoex_item.set_active(self.t.autoexpand) items.append(autoex_item) autoex_item.connect('activate', self.toggle_autoexpand) self.group_menu_item = gtk.CheckMenuItem('Toggle _Family Grouping') self.group_menu_item.set_active(self.t.should_group_families) items.append(self.group_menu_item) self.group_menu_item.connect('toggled', self.toggle_grouping) return items def _set_tooltip(self, widget, tip_text): """Convenience function to add hover over text to a widget.""" tip = gtk.Tooltips() tip.enable() tip.set_tip(widget, tip_text) def _set_cell_text_time(self, column, cell, model, iter_, n): """Remove the date part if it matches the last update date.""" date_time_string = model.get_value(iter_, n) if "T" in self.updater.dt: last_update_date = self.updater.dt.split("T")[0] date_time_string = date_time_string.replace( last_update_date + "T", "", 1) if n == 8: # Progress bar for estimated completion time. if isinstance(cell, gtk.CellRendererText): if date_time_string.endswith("?"): # Task running -show progress bar instead. cell.set_property('visible', False) else: # Task not running - just show text cell.set_property('visible', True) cell.set_property('text', date_time_string) if isinstance(cell, gtk.CellRendererProgress): if date_time_string.endswith("?"): # Task running -show progress bar to estimated finish time. cell.set_property('visible', True) percent = model.get_value(iter_, 12) cell.set_property('value', percent) else: # Task not running - show text cell instead. cell.set_property('visible', False) cell.set_property('value', 0) cell.set_property("text", date_time_string) def get_toolitems(self): """Return the tool bar items specific to this view.""" items = [] expand_button = gtk.ToolButton() image = gtk.image_new_from_stock(gtk.STOCK_ADD, gtk.ICON_SIZE_SMALL_TOOLBAR) expand_button.set_icon_widget(image) expand_button.set_label("Expand") self._set_tooltip(expand_button, "Tree View - Expand all") expand_button.connect('clicked', lambda x: self.ttreeview.expand_all()) items.append(expand_button) collapse_button = gtk.ToolButton() image = gtk.image_new_from_stock(gtk.STOCK_REMOVE, gtk.ICON_SIZE_SMALL_TOOLBAR) collapse_button.set_icon_widget(image) collapse_button.set_label("Collapse") collapse_button.connect('clicked', lambda x: self.ttreeview.collapse_all()) self._set_tooltip(collapse_button, "Tree View - Collapse all") items.append(collapse_button) self.group_toolbutton = gtk.ToggleToolButton() self.group_toolbutton.set_active(self.t.should_group_families) g_image = gtk.image_new_from_stock('group', gtk.ICON_SIZE_SMALL_TOOLBAR) self.group_toolbutton.set_icon_widget(g_image) self.group_toolbutton.set_label("Group") self.group_toolbutton.connect('toggled', self.toggle_grouping) self._set_tooltip(self.group_toolbutton, "Tree View - Click to group tasks by families") items.append(self.group_toolbutton) return items