def test_blankpattern(self):
# Make sure when tuple or something has no values no regex is generated.
# Fixes bug #661354
test_locale = _strptime.LocaleTime(timezone=('', ''))
self.failUnless(
_strptime.TimeRE(test_locale).pattern("%Z") == '',
"with timezone == ('',''), TimeRE().pattern('%Z') != ''")
def test_blankpattern(self):
# Make sure when tuple or something has no values no regex is generated.
# Fixes bug #661354
test_locale = _strptime.LocaleTime()
test_locale.timezone = (frozenset(), frozenset())
self.assertEqual(_strptime.TimeRE(test_locale).pattern("%Z"), '',
"with timezone == ('',''), TimeRE().pattern('%Z') != ''")
def test_locale_data_w_regex_metacharacters(self):
locale_time = _strptime.LocaleTime()
locale_time.timezone = frozenset(('utc', 'gmt',
'Tokyo (standard time)')), frozenset('Tokyo (daylight time)')
time_re = _strptime.TimeRE(locale_time)
self.assertTrue(time_re.compile('%Z').match('Tokyo (standard time)'
),
'locale data that contains regex metacharacters is not properly escaped'
)
def test_locale_data_w_regex_metacharacters(self):
# Check that if locale data contains regex metacharacters they are
# escaped properly.
# Discovered by bug #1039270 .
locale_time = _strptime.LocaleTime()
locale_time.timezone = (frozenset(("utc", "gmt",
"Tokyo (standard time)")),
frozenset("Tokyo (daylight time)"))
time_re = _strptime.TimeRE(locale_time)
self.assertTrue(time_re.compile("%Z").match("Tokyo (standard time)"),
"locale data that contains regex metacharacters is not"
" properly escaped")
def test_bad_timezone(self):
# Explicitly test possibility of bad timezone;
# when time.tzname[0] == time.tzname[1] and time.daylight
if sys.platform == "mac":
return # MacOS9 has severely broken timezone support.
try:
original_tzname = time.tzname
original_daylight = time.daylight
time.tzname = ("PDT", "PDT")
time.daylight = 1
# Need to make sure that timezone is not calculated since that
# calls time.tzset and overrides temporary changes to time .
_strptime._locale_cache = _strptime.TimeRE(
_strptime.LocaleTime(timezone=("PDT", "PDT")))
_strptime._regex_cache.clear()
tz_value = _strptime.strptime("PDT", "%Z")[8]
self.failUnlessEqual(tz_value, -1)
finally:
time.tzname = original_tzname
time.daylight = original_daylight
_strptime._locale_cache = _strptime.TimeRE()
_strptime._regex_cache.clear()
def main():
os.environ["TZ"] = 'EST5EDT'
time.tzset()
import _strptime
_strptime._cache_lock.acquire()
_strptime._TimeRE_cache = _strptime.TimeRE()
_strptime._regex_cache = {}
_strptime._cache_lock.release()
from java.lang import System
System.setProperty('user.name', 'GFETEST')
from com.raytheon.uf.viz.core.localization import LocalizationManager
from com.raytheon.uf.common.localization import LocalizationContext
LocalizationLevel = LocalizationContext.LocalizationLevel
LocalizationManager.registerContextName(LocalizationLevel.USER, 'GFETEST')
import loadConfig
loadConfig.loadPreferences("gfeConfig")
from com.raytheon.viz.gfe.core import DataManager
dm = DataManager.getInstance(None)
import IToolInterface
# have to add the user dir to the python path here since we just switched users
# TODO look into switching it from the java
from com.raytheon.uf.common.dataplugin.gfe.python import GfePyIncludeUtil
for s in sys.path:
if 'textUtilities' in s \
or 'textProducts' in s \
or 'combinations' in s:
sys.path.remove(s)
for s in str(GfePyIncludeUtil.getHeadlineIncludePath()).split(':'):
sys.path.append(s)
for s in str(GfePyIncludeUtil.getTextUtilitiesIncludePath()).split(':'):
sys.path.append(s)
for s in str(GfePyIncludeUtil.getTextProductsIncludePath()).split(':'):
sys.path.append(s)
for s in str(GfePyIncludeUtil.getCombinationsIncludePath()).split(':'):
sys.path.append(s)
# create output directory for products
try:
os.makedirs(OUTPUT_DIR)
except OSError, e:
if e.errno != errno.EEXIST:
self.output("%s: '%s'" % (e.strerror, e.filename))
def _string_template_to_date_formater(self):
## Convert string template to date formater
#ref = self.file_template.template.replace('$','%')
ref = self.changing_path.template.replace('$','%')
ref = ref.replace('{','')
ref = ref.replace('}','')
if 0:
# hack to add custom parsing in strptime
# https://stackoverflow.com/a/54451291/1840524
import _strptime
TimeRE = _strptime.TimeRE()
TimeRE.update({'E': '(.h5)?'}) # match 0 or 1 '.h5' pattern
_strptime._TimeRE_cache = TimeRE
ref = ref.replace('*','%E')
return ref
def strptime(data_string, format="%a %b %d %H:%M:%S %Y"):
"""Return a time struct based on the input string and the format string.
This has been taken from the Python2.5 distribution and slightly modified.
"""
time_re = _strptime.TimeRE()
locale_time = time_re.locale_time
format_regex = _regex_cache.get(format)
if not format_regex:
try:
format_regex = time_re.compile(format)
# KeyError raised when a bad format is found; can be specified as
# \\, in which case it was a stray % but with a space after it
except KeyError, err:
bad_directive = err.args[0]
if bad_directive == "\\":
bad_directive = "%"
del err
raise ValueError("'%s' is a bad directive in format '%s'" %
(bad_directive, format))
# IndexError only occurs when the format string is "%"
except IndexError:
raise ValueError("stray %% in format '%s'" % format)
def setUp(self):
"""Construct generic TimeRE object."""
self.time_re = _strptime.TimeRE()
self.locale_time = _strptime.LocaleTime()
def __init__(self):
# the child class must have initalized a few things already
assert isinstance(self.locale_time, (AbstractLocaleTime, _strptime_module.LocaleTime))
self._regex_cache = {}
self._time_re = _strptime_module.TimeRE(locale_time=self.locale_time)
def test_blankpattern(self):
test_locale = _strptime.LocaleTime()
test_locale.timezone = frozenset(), frozenset()
self.assertEqual(_strptime.TimeRE(test_locale).pattern('%Z'), '',
"with timezone == ('',''), TimeRE().pattern('%Z') != ''")
def get_continuous_intervals(self):
"""
Scan the provided directory to create a list of continuous intervals based on selected frequency:
[ [t0,t1] , [t2,t3] , ... ]
ti being datetime objects.
"""
## date template
dic_date = {'Y':'????', 'm':'??', 'd':'??', 'H':'??', 'M':'??'}
self.valid_year = {}
## INFO: timing for 337055 files to list:
# from timeit import default_timer as timer
# t0 = timer()
# out = list(self.fixed_path.glob(self.changing_path.substitute(**dic_date)))
# print(len(out), timer()-t0)
#
# t0 = timer()
# out = fnmatch.filter(os.listdir(self.fixed_path), self.changing_path.substitute(**dic_date))
# print(len(out), timer()-t0)
#
# Results:
# 337055 1.685467129573226
# 337055 0.328439112752676
## > 5x faster with fnmatch
## First select available (valid) years
## Use os.listdir() and fnmatch.filter() to perform quick selection
t0 = timer()
#flist = os.listdir(self.fixed_path)
flist = [ f for dp, dn, fn in os.walk(self.fixed_path) for f in fn]
print(timer()-t0)
#return
for y in range(self.init_year, self.last_year+1):
dic_date['Y'] = y
#print(y, 'sub:', self.file_template.substitute(**dic_date))
out = fnmatch.filter(flist, self.file_template.substitute(**dic_date))
if len(out) > 0:
self.valid_year[y] = []
print("valid year:", self.valid_year.keys())
#return
## Then iterate over months of valid years
t0 = timer()
for y in self.valid_year.keys():
dic_date['Y'] = y
for m in range(1,13):
dic_date['m'] = str(m).zfill(2)
#print('sub:', self.changing_path.substitute(**dic_date))
out = fnmatch.filter(flist, self.file_template.substitute(**dic_date))
out = [i for i in out if not i.endswith('.bz2')]
# if there is data in the month
if len(out)>0:
t00 = timer()
## Get available file by month
#convert string template to date formater
ref = self.file_template.template.replace('$','%')
ref = ref.replace('{','')
ref = ref.replace('}','')
# hack to add custom parsing in strptime
# https://stackoverflow.com/a/54451291/1840524
import _strptime
TimeRE = _strptime.TimeRE()
TimeRE.update({'x': '(.h5)?'}) # match 0 or 1 '.h5' pattern
_strptime._TimeRE_cache = TimeRE
ref = ref.replace('*','%x')
# process input list
# remove bz2 files
avail_date = np.array(sorted([datetime.datetime.strptime(f, ref) for f in out]))
## INFO: Get all theoretical dates, actually not useful here
## Note the use of MonthEnd to offset of one (or more) month precisely
#start = datetime.datetime(y, m, 1)
#all_date = pd.date_range(start, start+pd.tseries.offsets.MonthEnd(1), freq=self.freq)
print(y, m, len(out), '>', len(avail_date))
## Recorf all valid intervalls as [start_dat, end_date] of continuous range according to self.freq
inter = [avail_date[0]]
for d1,d2 in self.grouper(avail_date):
# check if there is more than one self.freq between two following date
if d2-d1!=pd.to_timedelta(self.freq):
inter.append(d1)
self.valid_year[y].append(inter)
inter = [d2]
inter.append(avail_date[-1])
self.valid_year[y].append(inter)
#print(timer()-t00)
print(timer()-t0)
self.show_tot_time()
## Merge continuous intervals when changing month.
for y,inter in self.valid_year.items():
merged = []
merged.append(inter[0])
for i1,i2 in self.grouper(inter):
if i2[0]-i1[1]==pd.to_timedelta(self.freq):
merged[-1][-1] = i2[1] # [d1,d2],[d3,d4] -> [d1,d4]
else:
merged.append(i2)
self.valid_year[y] = merged
self.show_tot_time()
