def coerce_cycle_point_time_zone(cls, value, keys): """Coerce value to a cycle point time zone format. Examples: >>> CylcConfigValidator.coerce_cycle_point_time_zone( ... 'Z', None) 'Z' >>> CylcConfigValidator.coerce_cycle_point_time_zone( ... '+13', None) '+13' >>> CylcConfigValidator.coerce_cycle_point_time_zone( ... '-0800', None) '-0800' """ value = cls.strip_and_unquote(keys, value) if not value: return None test_timepoint = TimePoint(year=2001, month_of_year=3, day_of_month=1, hour_of_day=4, minute_of_hour=30, second_of_minute=54) dumper = TimePointDumper() test_timepoint_string = dumper.dump(test_timepoint, 'CCYYMMDDThhmmss') test_timepoint_string += value parser = TimePointParser(allow_only_basic=True) try: parser.parse(test_timepoint_string) except ValueError: raise IllegalValueError( 'cycle point time zone format', keys, value) return value
def get_unix_time_from_time_string(datetime_string): """Convert a datetime string into a unix timestamp. The datetime_string must match DATE_TIME_FORMAT_EXTENDED above, which is the extended ISO 8601 year-month-dayThour:minute:second format, plus a valid ISO 8601 time zone. For example, 2016-09-07T11:21:00+01:00, 2016-12-25T06:00:00Z, or 2016-12-25T06:00:00+13. isodatetime is not used to do the whole parsing, partly for performance, but mostly because the calendar may be in non-Gregorian mode. """ try: date_time_utc = datetime.strptime( datetime_string, DATE_TIME_FORMAT_EXTENDED + "Z") except ValueError: global PARSER if PARSER is None: from metomi.isodatetime.parsers import TimePointParser PARSER = TimePointParser() time_zone_info = PARSER.get_info(datetime_string)[1] time_zone_hour = int(time_zone_info["time_zone_hour"]) time_zone_minute = int(time_zone_info.get("time_zone_minute", 0)) offset_seconds = 3600 * time_zone_hour + 60 * time_zone_minute if "+" in datetime_string: datetime_string = datetime_string.split("+")[0] else: datetime_string = datetime_string.rsplit("-", 1)[0] date_time = datetime.strptime( datetime_string, DATE_TIME_FORMAT_EXTENDED) date_time_utc = date_time - timedelta(seconds=offset_seconds) return timegm(date_time_utc.timetuple())
def add_offset(cycle_point, offset): """Add a (positive or negative) offset to a cycle point. Return the result. """ my_parser = TimePointParser() my_target_point = my_parser.parse(cycle_point, dump_as_parsed=True) my_offset_parser = DurationParser() oper = "+" if offset.startswith(("-", "+")): oper = offset[0] offset = offset[1:] if not offset.startswith("P"): # TODO - raise appropriate exception raise ValueError("ERROR, bad offset format: %s" % offset) my_shift = my_offset_parser.parse(offset) if oper == "-": my_target_point -= my_shift else: my_target_point += my_shift return my_target_point
def __init__(self, parse_format=None, utc_mode=False, calendar_mode=None, ref_point_str=None): """Constructor. parse_format -- If specified, parse with the specified format. Otherwise, parse with one of the format strings in self.PARSE_FORMATS. The format should be a string compatible to strptime(3). utc_mode -- If True, parse/print in UTC mode rather than local or other timezones. calendar_mode -- Set calendar mode for metomi.isodatetime.data.Calendar. ref_point_str -- Set the reference time point for operations. If not specified, operations use current date time. """ self.parse_formats = self.PARSE_FORMATS self.custom_parse_format = parse_format self.utc_mode = utc_mode if self.utc_mode: assumed_time_zone = (0, 0) else: assumed_time_zone = None self.set_calendar_mode(calendar_mode) self.time_point_dumper = TimePointDumper() self.time_point_parser = TimePointParser( assumed_time_zone=assumed_time_zone) self.duration_parser = DurationParser() self.ref_point_str = ref_point_str
def strftime(iso8601_datetime, strftime_str, strptime_str=None): """Format an iso8601 datetime string using an strftime string. Args: iso8601_datetime (str): Any valid ISO8601 datetime as a string. strftime_str (str): A valid strftime string to format the output datetime. strptime_str (str - optional): A valid strptime string defining the format of the provided iso8601_datetime. Return: The result of applying the strftime to the iso8601_datetime as parsed by the strptime string if provided. Raises: ISO8601SyntaxError: In the event of an invalid datetime string. StrftimeSyntaxError: In the event of an invalid strftime string. Examples: >>> # Basic usage. >>> strftime('2000-01-01T00Z', '%H') '00' >>> strftime('2000', '%H') '00' >>> strftime('2000', '%Y/%m/%d %H:%M:%S') '2000/01/01 00:00:00' >>> strftime('10661014T08+01', '%z') # Timezone offset. '+0100' >>> strftime('10661014T08+01', '%j') # Day of the year '287' >>> # Strptime. >>> strftime('12,30,2000', '%m', '%m,%d,%Y') '12' >>> strftime('1066/10/14 08:00:00', '%Y%m%dT%H', '%Y/%m/%d %H:%M:%S') '10661014T08' >>> # Exceptions. >>> strftime('invalid', '%H') # doctest: +NORMALIZE_WHITESPACE Traceback (most recent call last): <class 'metomi.isodatetime.parsers.ISO8601SyntaxError'> metomi.isodatetime.parsers.ISO8601SyntaxError: Invalid ISO 8601 date \ representation: invalid >>> strftime('2000', '%invalid') # doctest: +NORMALIZE_WHITESPACE Traceback (most recent call last): metomi.isodatetime.parser_spec.StrftimeSyntaxError: Invalid \ strftime/strptime representation: %i >>> strftime('2000', '%Y', '%invalid') ... # doctest: +NORMALIZE_WHITESPACE Traceback (most recent call last): metomi.isodatetime.parser_spec.StrftimeSyntaxError: Invalid \ strftime/strptime representation: %i """ if not strptime_str: return TimePointParser().parse(iso8601_datetime).strftime(strftime_str) return TimePointParser().strptime(iso8601_datetime, strptime_str).strftime(strftime_str)
def coerce_cycle_point(cls, value, keys): """Coerce value to a cycle point. Examples: >>> CylcConfigValidator.coerce_cycle_point('2000', None) '2000' >>> CylcConfigValidator.coerce_cycle_point('now', None) 'now' >>> CylcConfigValidator.coerce_cycle_point('next(T-00)', None) 'next(T-00)' """ if not value: return None value = cls.strip_and_unquote(keys, value) if value == 'now': # Handle this later in config.py when workflow UTC mode is known. return value if "next" in value or "previous" in value: # Handle this later, as for "now". return value if value.isdigit(): # Could be an old date-time cycle point format, or integer format. return value if "P" not in value and ( value.startswith('-') or value.startswith('+')): # We don't know the value given for num expanded year digits... for i in range(1, 101): try: TimePointParser(num_expanded_year_digits=i).parse(value) except IsodatetimeError: continue return value raise IllegalValueError('cycle point', keys, value) if "P" in value: # ICP is an offset parser = DurationParser() try: if value.startswith("-"): # parser doesn't allow negative duration with this setup? parser.parse(value[1:]) else: parser.parse(value) return value except IsodatetimeError as exc: raise IllegalValueError('cycle point', keys, value, exc=exc) try: TimePointParser().parse(value) except IsodatetimeError as exc: if isinstance(exc, ISO8601SyntaxError): # Don't know cycling mode yet, so override ISO8601-specific msg details = {'msg': "Invalid cycle point"} else: details = {'exc': exc} raise IllegalValueError('cycle point', keys, value, **details) return value
def coerce_cycle_point(cls, value, keys): """Coerce value to a cycle point. Examples: >>> CylcConfigValidator.coerce_cycle_point('2000', None) '2000' >>> CylcConfigValidator.coerce_cycle_point('now', None) 'now' >>> CylcConfigValidator.coerce_cycle_point('next(T-00)', None) 'next(T-00)' """ if not value: return None value = cls.strip_and_unquote(keys, value) if value == 'now': # Handle this later in config.py when the suite UTC mode is known. return value if "next" in value or "previous" in value: # Handle this later, as for "now". return value if value.isdigit(): # Could be an old date-time cycle point format, or integer format. return value if "P" not in value and ( value.startswith('-') or value.startswith('+')): # We don't know the value given for num expanded year digits... for i in range(1, 101): try: TimePointParser(num_expanded_year_digits=i).parse(value) except ValueError: continue return value raise IllegalValueError('cycle point', keys, value) if "P" in value: # ICP is an offset parser = DurationParser() try: if value.startswith("-"): # parser doesn't allow negative duration with this setup? parser.parse(value[1:]) else: parser.parse(value) return value except ValueError: raise IllegalValueError("cycle point", keys, value) try: TimePointParser().parse(value) except ValueError: raise IllegalValueError('cycle point', keys, value) return value
def is_valid_datetime(date): """Try and see if isodatetime can validate date.""" exc = IsodatetimeError if PY_3 else ISO8601SyntaxError # 23compat try: TimePointParser().parse(date) except exc: # 23compat - should be `except IsodatetimeError:` return False else: return True
def test_invalid_components(): parser = TimePointParser() for date, invalid in { '2000-01-01T00:00:60': ['second_of_minute=60'], '2000-01-01T00:60:00': ['minute_of_hour=60'], '2000-01-01T60:00:00': ['hour_of_day=60'], '2000-01-32T00:00:00': ['day_of_month=32'], '2000-13-00T00:00:00': ['month_of_year=13'], '2000-13-32T60:60:60': [ 'month_of_year=13', 'day_of_month=32', 'hour_of_day=60', 'minute_of_hour=60', 'second_of_minute=60' ] }.items(): with pytest.raises(ValueError) as exc: parser.parse(date) # import pdb; pdb.set_trace() for item in invalid: assert item in str(exc.value)
def connect(self): """Connect to the suite db, polling if necessary in case the suite has not been started up yet.""" # Returns True if connected, otherwise (one-off failed to # connect, or max number of polls exhausted) False connected = False if cylc.flow.flags.verbose: sys.stdout.write( "connecting to suite db for " + self.args['run_dir'] + "/" + self.args['suite']) # Attempt db connection even if no polls for condition are # requested, as failure to connect is useful information. max_polls = self.max_polls or 1 # max_polls*interval is equivalent to a timeout, and we # include time taken to connect to the run db in this... while not connected: self.n_polls += 1 try: self.checker = CylcSuiteDBChecker( self.args['run_dir'], self.args['suite']) connected = True # ... but ensure at least one poll after connection: self.n_polls -= 1 except (OSError, sqlite3.Error): if self.n_polls >= max_polls: raise if cylc.flow.flags.verbose: sys.stdout.write('.') sleep(self.interval) if cylc.flow.flags.verbose: sys.stdout.write('\n') if connected and self.args['cycle']: fmt = self.checker.get_remote_point_format() if fmt: my_parser = TimePointParser() my_point = my_parser.parse(self.args['cycle'], dump_format=fmt) self.args['cycle'] = str(my_point) return connected, self.args['cycle']
def suite_state(suite, task, point, offset=None, status='succeeded', message=None, cylc_run_dir=None, debug=False): """Connect to a suite DB and query the requested task state. Reports satisfied only if the remote suite state has been achieved. Returns all suite state args to pass on to triggering tasks. """ cylc_run_dir = os.path.expandvars( os.path.expanduser(cylc_run_dir or glbl_cfg().get_host_item('run directory'))) if offset is not None: point = str(add_offset(point, offset)) try: checker = CylcSuiteDBChecker(cylc_run_dir, suite) except (OSError, sqlite3.Error): # Failed to connect to DB; target suite may not be started. return (False, None) fmt = checker.get_remote_point_format() if fmt: my_parser = TimePointParser() point = str(my_parser.parse(point, dump_format=fmt)) if message is not None: satisfied = checker.task_state_met(task, point, message=message) else: satisfied = checker.task_state_met(task, point, status=status) results = { 'suite': suite, 'task': task, 'point': point, 'offset': offset, 'status': status, 'message': message, 'cylc_run_dir': cylc_run_dir } return (satisfied, results)
def workflow_state(workflow, task, point, offset=None, status='succeeded', message=None, cylc_run_dir=None): """Connect to a workflow DB and query the requested task state. * Reports satisfied only if the remote workflow state has been achieved. * Returns all workflow state args to pass on to triggering tasks. Arguments: workflow (str): The workflow to interrogate. task (str): The name of the task to query. point (str): The cycle point. offset (str): The offset between the cycle this xtrigger is used in and the one it is querying for as an ISO8601 time duration. e.g. PT1H (one hour). status (str): The task status required for this xtrigger to be satisfied. message (str): The custom task output required for this xtrigger to be satisfied. .. note:: This cannot be specified in conjunction with ``status``. cylc_run_dir (str): The directory in which the workflow to interrogate. .. note:: This only needs to be supplied if the workflow is running in a different location to what is specified in the global configuration (usually ``~/cylc-run``). Returns: tuple: (satisfied, results) satisfied (bool): True if ``satisfied`` else ``False``. results (dict): Dictionary containing the args / kwargs which were provided to this xtrigger. """ if cylc_run_dir: cylc_run_dir = expand_path(cylc_run_dir) else: cylc_run_dir = get_workflow_run_dir('') if offset is not None: point = str(add_offset(point, offset)) try: checker = CylcWorkflowDBChecker(cylc_run_dir, workflow) except (OSError, sqlite3.Error): # Failed to connect to DB; target workflow may not be started. return (False, None) fmt = checker.get_remote_point_format() if fmt: my_parser = TimePointParser() point = str(my_parser.parse(point, dump_format=fmt)) if message is not None: satisfied = checker.task_state_met(task, point, message=message) else: satisfied = checker.task_state_met(task, point, status=status) results = { 'workflow': workflow, 'task': task, 'point': point, 'offset': offset, 'status': status, 'message': message, 'cylc_run_dir': cylc_run_dir } return satisfied, results
class RoseDateTimeOperator: """A class to parse and print date string with an offset.""" CURRENT_TIME_DUMP_FORMAT = "CCYY-MM-DDThh:mm:ss+hh:mm" CURRENT_TIME_DUMP_FORMAT_Z = "CCYY-MM-DDThh:mm:ssZ" NEGATIVE = "-" # strptime formats and their compatibility with the ISO 8601 parser. PARSE_FORMATS = [ ("%a %b %d %H:%M:%S %Y", True), # ctime ("%a %b %d %H:%M:%S %Z %Y", True), # Unix "date" ("%Y-%m-%dT%H:%M:%S", False), # ISO8601, extended ("%Y%m%dT%H%M%S", False), # ISO8601, basic ("%Y%m%d%H", False), # Cylc (current) ] REC_OFFSET = re.compile(r"""\A[\+\-]?(?:\d+[wdhms])+\Z""", re.I) REC_OFFSET_FIND = re.compile(r"""(?P<num>\d+)(?P<unit>[wdhms])""") STR_NOW = "now" STR_REF = "ref" TASK_CYCLE_TIME_ENV = "ROSE_TASK_CYCLE_TIME" UNITS = { "w": "weeks", "d": "days", "h": "hours", "m": "minutes", "s": "seconds", } def __init__( self, parse_format=None, utc_mode=False, calendar_mode=None, ref_point_str=None, ): """Constructor. parse_format -- If specified, parse with the specified format. Otherwise, parse with one of the format strings in self.PARSE_FORMATS. The format should be a string compatible to strptime(3). utc_mode -- If True, parse/print in UTC mode rather than local or other timezones. calendar_mode -- Set calendar mode for metomi.isodatetime.data.Calendar. ref_point_str -- Set the reference time point for operations. If not specified, operations use current date time. """ self.parse_formats = self.PARSE_FORMATS self.custom_parse_format = parse_format self.utc_mode = utc_mode if self.utc_mode: assumed_time_zone = (0, 0) else: assumed_time_zone = None self.set_calendar_mode(calendar_mode) self.time_point_dumper = TimePointDumper() self.time_point_parser = TimePointParser( assumed_time_zone=assumed_time_zone) self.duration_parser = DurationParser() self.ref_point_str = ref_point_str def date_format(self, print_format, time_point=None): """Reformat time_point according to print_format. time_point -- The time point to format. Otherwise, use ref date time. """ if time_point is None: time_point = self.date_parse()[0] if print_format is None: return str(time_point) if "%" in print_format: try: return time_point.strftime(print_format) except ValueError: return self.get_datetime_strftime(time_point, print_format) return self.time_point_dumper.dump(time_point, print_format) def date_parse(self, time_point_str=None): """Parse time_point_str. Return (t, format) where t is a metomi.isodatetime.data.TimePoint object and format is the format that matches time_point_str. time_point_str -- The time point string to parse. Otherwise, use ref time. """ if time_point_str is None or time_point_str == self.STR_REF: time_point_str = self.ref_point_str if time_point_str is None or time_point_str == self.STR_NOW: time_point = get_timepoint_for_now() if self.utc_mode or time_point.get_time_zone_utc(): # is in UTC parse_format = self.CURRENT_TIME_DUMP_FORMAT_Z else: parse_format = self.CURRENT_TIME_DUMP_FORMAT elif self.custom_parse_format is not None: parse_format = self.custom_parse_format time_point = self.strptime(time_point_str, parse_format) else: parse_formats = list(self.parse_formats) time_point = None while parse_formats: parse_format, should_use_datetime = parse_formats.pop(0) try: if should_use_datetime: time_point = self.get_datetime_strptime( time_point_str, parse_format) else: time_point = self.time_point_parser.strptime( time_point_str, parse_format) break except ValueError: pass if time_point is None: time_point = self.time_point_parser.parse(time_point_str, dump_as_parsed=True) parse_format = time_point.dump_format if self.utc_mode: time_point = time_point.to_utc() return time_point, parse_format def date_shift(self, time_point=None, offset=None): """Return a date string with an offset. time_point -- A time point or time point string. Otherwise, use current time. offset -- If specified, it should be a string containing the offset that has the format "[+/-]nU[nU...]" where "n" is an integer, and U is a unit matching a key in self.UNITS. """ if time_point is None: time_point = self.date_parse()[0] # Offset if offset: sign = "+" if offset.startswith("-") or offset.startswith("+"): sign = offset[0] offset = offset[1:] if offset.startswith("P"): # Parse and apply. try: duration = self.duration_parser.parse(offset) except ValueError: raise OffsetValueError(offset) if sign == "-": time_point -= duration else: time_point += duration else: # Backwards compatibility for e.g. "-1h" if not self.is_offset(offset): raise OffsetValueError(offset) for num, unit in self.REC_OFFSET_FIND.findall(offset.lower()): num = int(num) if sign == "-": num = -num key = self.UNITS[unit] time_point += Duration(**{key: num}) return time_point def date_diff(self, time_point_1=None, time_point_2=None): """Return (duration, is_negative) between two TimePoint objects. duration -- is a Duration instance. is_negative -- is a RoseDateTimeOperator.NEGATIVE if time_point_2 is in the past of time_point_1. """ if time_point_2 < time_point_1: return (time_point_1 - time_point_2, self.NEGATIVE) else: return (time_point_2 - time_point_1, "") @classmethod def date_diff_format(cls, print_format, duration, sign): """Format a duration.""" if print_format: delta_lookup = { "y": duration.years, "m": duration.months, "d": duration.days, "h": duration.hours, "M": duration.minutes, "s": duration.seconds, } expression = "" for item in print_format: if item in delta_lookup: if float(delta_lookup[item]).is_integer(): expression += str(int(delta_lookup[item])) else: expression += str(delta_lookup[item]) else: expression += item return sign + expression else: return sign + str(duration) @staticmethod def get_calendar_mode(): """Get current calendar mode.""" return Calendar.default().mode def is_offset(self, offset): """Return True if the string offset can be parsed as an offset.""" return self.REC_OFFSET.match(offset) is not None @staticmethod def set_calendar_mode(calendar_mode=None): """Set calendar mode for subsequent operations. Raise KeyError if calendar_mode is invalid. """ if not calendar_mode: calendar_mode = os.getenv("ROSE_CYCLING_MODE") if calendar_mode and calendar_mode in Calendar.MODES: Calendar.default().set_mode(calendar_mode) def strftime(self, time_point, print_format): """Use either the metomi.isodatetime or datetime strftime time formatting. """ try: return time_point.strftime(print_format) except ValueError: return self.get_datetime_strftime(time_point, print_format) def strptime(self, time_point_str, parse_format): """Use either the isodatetime or datetime strptime time parsing.""" try: return self.time_point_parser.strptime(time_point_str, parse_format) except ValueError: return self.get_datetime_strptime(time_point_str, parse_format) @classmethod def get_datetime_strftime(cls, time_point, print_format): """Use the datetime library's strftime as a fallback.""" year, month, day = time_point.get_calendar_date() hour, minute, second = time_point.get_hour_minute_second() microsecond = int(1.0e6 * (second - int(second))) hour = int(hour) minute = int(minute) second = int(second) date_time = datetime(year, month, day, hour, minute, second, microsecond) return date_time.strftime(print_format) def get_datetime_strptime(self, time_point_str, parse_format): """Use the datetime library's strptime as a fallback.""" date_time = datetime.strptime(time_point_str, parse_format) return self.time_point_parser.parse(date_time.isoformat())
def strftime(iso8601_datetime, strftime_str, strptime_str=None): """Format an :term:`ISO8601 datetime` string using an strftime string. .. code-block:: cylc {{ '10661004T08+01' | strftime('%H') }} # 00 It is also possible to parse non-standard date-time strings by passing a strptime string as the second argument. Args: iso8601_datetime (str): Any valid ISO8601 datetime as a string. strftime_str (str): A valid strftime string to format the output datetime. strptime_str (str - optional): A valid strptime string defining the format of the provided iso8601_datetime. Return: The result of applying the strftime to the iso8601_datetime as parsed by the strptime string if provided. Raises: ISO8601SyntaxError: In the event of an invalid datetime string. StrftimeSyntaxError: In the event of an invalid strftime string. Python Examples: >>> # Basic usage. >>> strftime('2000-01-01T00Z', '%H') '00' >>> strftime('2000', '%H') '00' >>> strftime('2000', '%Y/%m/%d %H:%M:%S') '2000/01/01 00:00:00' >>> strftime('10661014T08+01', '%z') # Timezone offset. '+0100' >>> strftime('10661014T08+01', '%j') # Day of the year '287' >>> # Strptime. >>> strftime('12,30,2000', '%m', '%m,%d,%Y') '12' >>> strftime('1066/10/14 08:00:00', '%Y%m%dT%H', '%Y/%m/%d %H:%M:%S') '10661014T08' >>> # Exceptions. >>> strftime('invalid', '%H') # doctest: +NORMALIZE_WHITESPACE Traceback (most recent call last): <class 'metomi.isodatetime.exceptions.ISO8601SyntaxError'> metomi.isodatetime.exceptions.ISO8601SyntaxError: Invalid ISO 8601 \ date representation: invalid >>> strftime('2000', '%invalid') # doctest: +NORMALIZE_WHITESPACE Traceback (most recent call last): metomi.isodatetime.exceptions.StrftimeSyntaxError: Invalid \ strftime/strptime representation: %i >>> strftime('2000', '%Y', '%invalid') ... # doctest: +NORMALIZE_WHITESPACE Traceback (most recent call last): metomi.isodatetime.exceptions.StrftimeSyntaxError: Invalid \ strftime/strptime representation: %i Jinja2 Examples: .. code-block:: cylc {% set START_CYCLE = '10661004T08+01' %} {{START_CYCLE | strftime('%Y')}} # 1066 {{START_CYCLE | strftime('%m')}} # 10 {{START_CYCLE | strftime('%d')}} # 14 {{START_CYCLE | strftime('%H:%M:%S %z')}} # 08:00:00 +01 {{'12,30,2000' | strftime('%m', '%m,%d,%Y')}} # 12 """ if not strptime_str: return TimePointParser().parse(iso8601_datetime).strftime(strftime_str) return TimePointParser().strptime(iso8601_datetime, strptime_str).strftime( strftime_str)