def today(self):
if not self._processed_today:
self._today = datetimeutils.fill_in_datetime_strings(self._today)
self._today = datetimeutils.to_datetime(self._today)
self._processed_today = True
return self._today
def replace_config_wildcards(cls, val):
if isinstance(val, dict):
for k in val:
val[k] = cls.replace_config_wildcards(val[k])
elif isinstance(val, list):
val = [cls.replace_config_wildcards(v) for v in val]
elif hasattr(val, 'lower'): # i.e. it's a string
if val:
# first, fill in any datetime control codes or wildcards
val = datetimeutils.fill_in_datetime_strings(val,
today=cls._TODAY)
# then, see if the resulting string purely represents a datetime
try:
val = datetimeutils.to_datetime(val, limit_range=True)
except BlueSkyDatetimeValueError:
pass
if hasattr(val, 'capitalize'):
# This gets rid of datetime parsing busters embedded
# in strings to prevent conversion to datetime object
val = val.replace('{datetime-parse-buster}', '')
if cls._RUN_ID:
val = val.replace('{run_id}', cls._RUN_ID)
# TODO: any other replacements?
return val
def run_id(self, run_id):
if self._maually_set_run_id:
raise TypeError(self.RUN_ID_IS_IMMUTABLE_MSG)
self._maually_set_run_id = True
logging.debug('filling in run_id wildcards')
self._run_id = datetimeutils.fill_in_datetime_strings(run_id,
today=self.today)
Config.set_run_id(self._run_id)
def _stream(self, file_name, flag): #, do_strip_newlines):
if file_name:
file_name = datetimeutils.fill_in_datetime_strings(
file_name, today=self.today)
file_name = file_name.replace('{run_id}', self.run_id)
return open(file_name, flag)
else:
if flag == 'r':
return sys.stdin
else:
return sys.stdout
def _stream(self, file_name, flag): #, do_strip_newlines):
if file_name:
file_name = datetimeutils.fill_in_datetime_strings(
file_name, today=self.today)
file_name = file_name.replace('{run_id}', self.run_id)
if file_name.startswith('http'):
logging.debug("Loading input over http: %s", file_name)
r = requests.get(file_name)
return io.StringIO(r.text)
else:
logging.debug("Loading local file: %s", file_name)
return open(file_name, flag)
else:
if flag == 'r':
return sys.stdin
else:
return sys.stdout
def test_today_not_defined(self):
# None value - leave as is
assert datetimeutils.fill_in_datetime_strings(None) == None
# invalid value - leave as is
assert datetimeutils.fill_in_datetime_strings(123) == 123
# string with nothing to replace
assert datetimeutils.fill_in_datetime_strings("sljdfsd") == "sljdfsd"
# invalid and thus not-replaceable
assert datetimeutils.fill_in_datetime_strings("{today4}") == "{today4}"
assert datetimeutils.fill_in_datetime_strings(
"{yesterday++1}") == "{yesterday++1}"
assert datetimeutils.fill_in_datetime_strings(
"{timestamp++1}") == "{timestamp++1}"
assert datetimeutils.fill_in_datetime_strings(
"{today--1:%Y}") == "{today--1:%Y}"
# control codes outside of '{(today|tomorrow):' + '}' and thus not replaced
assert datetimeutils.fill_in_datetime_strings("%Y-%m-%d") == "%Y-%m-%d"
# strings with replacements
assert datetimeutils.fill_in_datetime_strings("{today}") == "20160114"
assert datetimeutils.fill_in_datetime_strings(
"{today}00") == "2016011400"
assert datetimeutils.fill_in_datetime_strings(
"{today+1}") == "20160115"
assert datetimeutils.fill_in_datetime_strings(
"{today-4}") == "20160110"
assert datetimeutils.fill_in_datetime_strings(
"{today-4}00") == "2016011000"
assert datetimeutils.fill_in_datetime_strings(
"{yesterday+3}") == "20160116"
assert datetimeutils.fill_in_datetime_strings(
"{yesterday-4}") == "20160109"
assert datetimeutils.fill_in_datetime_strings(
"{yesterday-4}12") == "2016010912"
assert datetimeutils.fill_in_datetime_strings(
"{timestamp+3}") == "20160117130111"
assert datetimeutils.fill_in_datetime_strings(
"{timestamp-4}") == "20160110130111"
assert datetimeutils.fill_in_datetime_strings(
"{timestamp-4}-0") == "20160110130111-0"
assert datetimeutils.fill_in_datetime_strings(
"{today:%Y-%m-%d}") == "2016-01-14"
assert datetimeutils.fill_in_datetime_strings(
"{today:%Y-%m-%d}T12Z") == "2016-01-14T12Z"
assert datetimeutils.fill_in_datetime_strings(
"{today+2:%Y-%m-%d}") == "2016-01-16"
assert datetimeutils.fill_in_datetime_strings(
"{today-2:%Y-%m-%d}") == "2016-01-12"
assert datetimeutils.fill_in_datetime_strings(
"{today-3:%Y-%m-%d}T12Z") == "2016-01-11T12Z"
assert datetimeutils.fill_in_datetime_strings(
"{yesterday:%Y-%m-%d}") == "2016-01-13"
assert datetimeutils.fill_in_datetime_strings(
"{yesterday:%Y-%m-%d}T12Z") == "2016-01-13T12Z"
assert datetimeutils.fill_in_datetime_strings(
"{yesterday+1:%Y-%m-%d}") == "2016-01-14"
assert datetimeutils.fill_in_datetime_strings(
"{yesterday-4:%Y-%m-%d}") == "2016-01-09"
assert datetimeutils.fill_in_datetime_strings(
"{yesterday-5:%Y-%m-%d}T12Z") == "2016-01-08T12Z"
assert datetimeutils.fill_in_datetime_strings(
"{timestamp:%Y-%m-%dT%H}") == "2016-01-14T13"
assert datetimeutils.fill_in_datetime_strings(
"{timestamp:%Y-%m-%d}T12Z") == "2016-01-14T12Z"
assert datetimeutils.fill_in_datetime_strings(
"{timestamp+1:%Y-%m-%d}") == "2016-01-15"
assert datetimeutils.fill_in_datetime_strings(
"{timestamp-4:%Y-%m-%d}") == "2016-01-10"
assert datetimeutils.fill_in_datetime_strings(
"{timestamp-5:%Y-%m-%d}T12Z") == "2016-01-09T12Z"
val = datetimeutils.fill_in_datetime_strings(
"{today}-sdf-{yesterday:%Y-%m-%d}T12Z-%Y__%m-%d-{today:%m_%d}-"
"{yesterday}-{sdf}-{today:%m%d%Y}-{yesterday:%m-%d-%Y}"
"{today-2}=={today-3:%Y-%m-%d}-{yesterday-2}-{yesterday-3:%Y-%m-%d}"
"{today+2}=={yesterday+2:%Y-%m-%d}"
"%Y%m%dT%H%M%S"
"{timestamp}")
assert val == ("20160114-sdf-2016-01-13T12Z-%Y__%m-%d-01_14-"
"20160113-{sdf}-01142016-01-13-2016"
"20160112==2016-01-11-20160111-2016-01-10"
"20160116==2016-01-15"
"%Y%m%dT%H%M%S"
"20160114130111")
def test_today_defined(self):
# None value - leave as is
assert datetimeutils.fill_in_datetime_strings(None,
today=datetime.date(
2016, 2, 15)) == None
assert datetimeutils.fill_in_datetime_strings(None,
today=datetime.datetime(
2016, 2, 15,
0)) == None
# invalid value - leave as is
assert datetimeutils.fill_in_datetime_strings(123,
today=datetime.date(
2016, 2, 15)) == 123
assert datetimeutils.fill_in_datetime_strings(123,
today=datetime.datetime(
2016, 2, 15,
1)) == 123
# string with nothing to replace
assert datetimeutils.fill_in_datetime_strings("sljdfsd",
today=datetime.date(
2016, 2,
15)) == "sljdfsd"
assert datetimeutils.fill_in_datetime_strings("sljdfsd",
today=datetime.datetime(
2016, 2, 15,
1)) == "sljdfsd"
# invalid and thus not-replaceable
assert datetimeutils.fill_in_datetime_strings("{today4}",
today=datetime.date(
2016, 2,
15)) == "{today4}"
assert datetimeutils.fill_in_datetime_strings(
"{yesterday++1}", today=datetime.date(2016, 2,
15)) == "{yesterday++1}"
assert datetimeutils.fill_in_datetime_strings(
"{timestamp++1}", today=datetime.date(2016, 2,
15)) == "{timestamp++1}"
assert datetimeutils.fill_in_datetime_strings(
"{today--1:%Y}", today=datetime.date(2016, 2,
15)) == "{today--1:%Y}"
# control codes outside of '{(today|tomorrow):' + '}' and thus not replaced
assert datetimeutils.fill_in_datetime_strings("%Y-%m-%d") == "%Y-%m-%d"
# strings with replacements
assert datetimeutils.fill_in_datetime_strings("{today}",
today=datetime.date(
2016, 2,
15)) == "20160215"
assert datetimeutils.fill_in_datetime_strings("{today}",
today=datetime.datetime(
2016, 2, 15,
1)) == "20160215"
assert datetimeutils.fill_in_datetime_strings("{today-1}",
today=datetime.date(
2016, 2,
15)) == "20160214"
assert datetimeutils.fill_in_datetime_strings("{today-2}",
today=datetime.datetime(
2016, 2, 15,
1)) == "20160213"
assert datetimeutils.fill_in_datetime_strings("{yesterday}",
today=datetime.date(
2016, 2,
15)) == "20160214"
assert datetimeutils.fill_in_datetime_strings("{yesterday}",
today=datetime.datetime(
2016, 2, 15,
1)) == "20160214"
assert datetimeutils.fill_in_datetime_strings("{yesterday-1}",
today=datetime.date(
2016, 2,
15)) == "20160213"
assert datetimeutils.fill_in_datetime_strings("{yesterday-2}",
today=datetime.datetime(
2016, 2, 15,
1)) == "20160212"
assert datetimeutils.fill_in_datetime_strings(
"{timestamp}", today=datetime.date(2016, 2,
15)) == "20160110031122"
assert datetimeutils.fill_in_datetime_strings(
"{timestamp}", today=datetime.datetime(2016, 2, 15,
1)) == "20160110031122"
assert datetimeutils.fill_in_datetime_strings(
"{timestamp-1}", today=datetime.date(2016, 2,
15)) == "20160109031122"
assert datetimeutils.fill_in_datetime_strings(
"{timestamp-2}", today=datetime.datetime(2016, 2, 15,
1)) == "20160108031122"
assert datetimeutils.fill_in_datetime_strings("{today:%Y-%m-%d}",
today=datetime.date(
2016, 2,
15)) == "2016-02-15"
assert datetimeutils.fill_in_datetime_strings("{today:%Y-%m-%d}",
today=datetime.datetime(
2016, 2, 15,
1)) == "2016-02-15"
assert datetimeutils.fill_in_datetime_strings("{today-1:%Y-%m-%d}",
today=datetime.date(
2016, 2,
15)) == "2016-02-14"
assert datetimeutils.fill_in_datetime_strings("{today-2:%Y-%m-%d}",
today=datetime.datetime(
2016, 2, 15,
1)) == "2016-02-13"
assert datetimeutils.fill_in_datetime_strings("{yesterday:%Y-%m-%d}",
today=datetime.date(
2016, 2,
15)) == "2016-02-14"
assert datetimeutils.fill_in_datetime_strings("{yesterday:%Y-%m-%d}",
today=datetime.datetime(
2016, 2, 15,
1)) == "2016-02-14"
assert datetimeutils.fill_in_datetime_strings("{yesterday-1:%Y-%m-%d}",
today=datetime.date(
2016, 2,
15)) == "2016-02-13"
assert datetimeutils.fill_in_datetime_strings("{yesterday-2:%Y-%m-%d}",
today=datetime.datetime(
2016, 2, 15,
1)) == "2016-02-12"
assert datetimeutils.fill_in_datetime_strings("{timestamp:%Y-%m-%d}",
today=datetime.date(
2016, 2,
15)) == "2016-01-10"
assert datetimeutils.fill_in_datetime_strings(
"{timestamp:%Y-%m-%dT%H:%M:%S}",
today=datetime.datetime(2016, 2, 15, 1)) == "2016-01-10T03:11:22"
assert datetimeutils.fill_in_datetime_strings("{timestamp-1:%Y-%m-%d}",
today=datetime.date(
2016, 2,
15)) == "2016-01-09"
assert datetimeutils.fill_in_datetime_strings(
"{timestamp-2:%Y-%m-%dT%H:%M:%S}",
today=datetime.datetime(2016, 2, 15, 1)) == "2016-01-08T03:11:22"
val = datetimeutils.fill_in_datetime_strings(
"{today}-sdf-{yesterday:%Y-%m-%d}T12Z-%Y__%m-%d-{today:%m_%d}-"
"{yesterday}-{sdf}-{today:%m%d%Y}-{yesterday:%m-%d-%Y}"
"{today-2}=={today-3:%Y-%m-%d}-{yesterday-2}-{yesterday-3:%Y-%m-%d}"
"{today+2}=={yesterday+2:%Y-%m-%d}"
"%Y%m%dT%H%M%S"
"{timestamp}",
today=datetime.date(2016, 1, 14))
assert val == ("20160114-sdf-2016-01-13T12Z-%Y__%m-%d-01_14-"
"20160113-{sdf}-01142016-01-13-2016"
"20160112==2016-01-11-20160111-2016-01-10"
"20160116==2016-01-15"
"%Y%m%dT%H%M%S"
"20160110031122")
def run_id(self, run_id):
logging.debug('filling in run_id wildcards')
self._run_id = datetimeutils.fill_in_datetime_strings(run_id,
today=self.today)
Config().set_run_id(self._run_id)
def today(self, today):
today = datetimeutils.fill_in_datetime_strings(today)
today = datetimeutils.to_datetime(today)
self._today = today
Config().set_today(self._today)
