def test_blankpattern(self): # Make sure when tuple or something has no values no regex is generated. # Fixes bug #661354 test_locale = _strptime.LocaleTime(timezone=('', '')) self.failUnless( _strptime.TimeRE(test_locale).pattern("%Z") == '', "with timezone == ('',''), TimeRE().pattern('%Z') != ''")
def test_blankpattern(self): # Make sure when tuple or something has no values no regex is generated. # Fixes bug #661354 test_locale = _strptime.LocaleTime() test_locale.timezone = (frozenset(), frozenset()) self.assertEqual(_strptime.TimeRE(test_locale).pattern("%Z"), '', "with timezone == ('',''), TimeRE().pattern('%Z') != ''")
def test_locale_data_w_regex_metacharacters(self): locale_time = _strptime.LocaleTime() locale_time.timezone = frozenset(('utc', 'gmt', 'Tokyo (standard time)')), frozenset('Tokyo (daylight time)') time_re = _strptime.TimeRE(locale_time) self.assertTrue(time_re.compile('%Z').match('Tokyo (standard time)' ), 'locale data that contains regex metacharacters is not properly escaped' )
def test_locale_data_w_regex_metacharacters(self): # Check that if locale data contains regex metacharacters they are # escaped properly. # Discovered by bug #1039270 . locale_time = _strptime.LocaleTime() locale_time.timezone = (frozenset(("utc", "gmt", "Tokyo (standard time)")), frozenset("Tokyo (daylight time)")) time_re = _strptime.TimeRE(locale_time) self.assertTrue(time_re.compile("%Z").match("Tokyo (standard time)"), "locale data that contains regex metacharacters is not" " properly escaped")
def test_bad_timezone(self): # Explicitly test possibility of bad timezone; # when time.tzname[0] == time.tzname[1] and time.daylight if sys.platform == "mac": return # MacOS9 has severely broken timezone support. try: original_tzname = time.tzname original_daylight = time.daylight time.tzname = ("PDT", "PDT") time.daylight = 1 # Need to make sure that timezone is not calculated since that # calls time.tzset and overrides temporary changes to time . _strptime._locale_cache = _strptime.TimeRE( _strptime.LocaleTime(timezone=("PDT", "PDT"))) _strptime._regex_cache.clear() tz_value = _strptime.strptime("PDT", "%Z")[8] self.failUnlessEqual(tz_value, -1) finally: time.tzname = original_tzname time.daylight = original_daylight _strptime._locale_cache = _strptime.TimeRE() _strptime._regex_cache.clear()
def main(): os.environ["TZ"] = 'EST5EDT' time.tzset() import _strptime _strptime._cache_lock.acquire() _strptime._TimeRE_cache = _strptime.TimeRE() _strptime._regex_cache = {} _strptime._cache_lock.release() from java.lang import System System.setProperty('user.name', 'GFETEST') from com.raytheon.uf.viz.core.localization import LocalizationManager from com.raytheon.uf.common.localization import LocalizationContext LocalizationLevel = LocalizationContext.LocalizationLevel LocalizationManager.registerContextName(LocalizationLevel.USER, 'GFETEST') import loadConfig loadConfig.loadPreferences("gfeConfig") from com.raytheon.viz.gfe.core import DataManager dm = DataManager.getInstance(None) import IToolInterface # have to add the user dir to the python path here since we just switched users # TODO look into switching it from the java from com.raytheon.uf.common.dataplugin.gfe.python import GfePyIncludeUtil for s in sys.path: if 'textUtilities' in s \ or 'textProducts' in s \ or 'combinations' in s: sys.path.remove(s) for s in str(GfePyIncludeUtil.getHeadlineIncludePath()).split(':'): sys.path.append(s) for s in str(GfePyIncludeUtil.getTextUtilitiesIncludePath()).split(':'): sys.path.append(s) for s in str(GfePyIncludeUtil.getTextProductsIncludePath()).split(':'): sys.path.append(s) for s in str(GfePyIncludeUtil.getCombinationsIncludePath()).split(':'): sys.path.append(s) # create output directory for products try: os.makedirs(OUTPUT_DIR) except OSError, e: if e.errno != errno.EEXIST: self.output("%s: '%s'" % (e.strerror, e.filename))
def _string_template_to_date_formater(self): ## Convert string template to date formater #ref = self.file_template.template.replace('$','%') ref = self.changing_path.template.replace('$','%') ref = ref.replace('{','') ref = ref.replace('}','') if 0: # hack to add custom parsing in strptime # https://stackoverflow.com/a/54451291/1840524 import _strptime TimeRE = _strptime.TimeRE() TimeRE.update({'E': '(.h5)?'}) # match 0 or 1 '.h5' pattern _strptime._TimeRE_cache = TimeRE ref = ref.replace('*','%E') return ref
def strptime(data_string, format="%a %b %d %H:%M:%S %Y"): """Return a time struct based on the input string and the format string. This has been taken from the Python2.5 distribution and slightly modified. """ time_re = _strptime.TimeRE() locale_time = time_re.locale_time format_regex = _regex_cache.get(format) if not format_regex: try: format_regex = time_re.compile(format) # KeyError raised when a bad format is found; can be specified as # \\, in which case it was a stray % but with a space after it except KeyError, err: bad_directive = err.args[0] if bad_directive == "\\": bad_directive = "%" del err raise ValueError("'%s' is a bad directive in format '%s'" % (bad_directive, format)) # IndexError only occurs when the format string is "%" except IndexError: raise ValueError("stray %% in format '%s'" % format)
def setUp(self): """Construct generic TimeRE object.""" self.time_re = _strptime.TimeRE() self.locale_time = _strptime.LocaleTime()
def __init__(self): # the child class must have initalized a few things already assert isinstance(self.locale_time, (AbstractLocaleTime, _strptime_module.LocaleTime)) self._regex_cache = {} self._time_re = _strptime_module.TimeRE(locale_time=self.locale_time)
def test_blankpattern(self): test_locale = _strptime.LocaleTime() test_locale.timezone = frozenset(), frozenset() self.assertEqual(_strptime.TimeRE(test_locale).pattern('%Z'), '', "with timezone == ('',''), TimeRE().pattern('%Z') != ''")
def get_continuous_intervals(self): """ Scan the provided directory to create a list of continuous intervals based on selected frequency: [ [t0,t1] , [t2,t3] , ... ] ti being datetime objects. """ ## date template dic_date = {'Y':'????', 'm':'??', 'd':'??', 'H':'??', 'M':'??'} self.valid_year = {} ## INFO: timing for 337055 files to list: # from timeit import default_timer as timer # t0 = timer() # out = list(self.fixed_path.glob(self.changing_path.substitute(**dic_date))) # print(len(out), timer()-t0) # # t0 = timer() # out = fnmatch.filter(os.listdir(self.fixed_path), self.changing_path.substitute(**dic_date)) # print(len(out), timer()-t0) # # Results: # 337055 1.685467129573226 # 337055 0.328439112752676 ## > 5x faster with fnmatch ## First select available (valid) years ## Use os.listdir() and fnmatch.filter() to perform quick selection t0 = timer() #flist = os.listdir(self.fixed_path) flist = [ f for dp, dn, fn in os.walk(self.fixed_path) for f in fn] print(timer()-t0) #return for y in range(self.init_year, self.last_year+1): dic_date['Y'] = y #print(y, 'sub:', self.file_template.substitute(**dic_date)) out = fnmatch.filter(flist, self.file_template.substitute(**dic_date)) if len(out) > 0: self.valid_year[y] = [] print("valid year:", self.valid_year.keys()) #return ## Then iterate over months of valid years t0 = timer() for y in self.valid_year.keys(): dic_date['Y'] = y for m in range(1,13): dic_date['m'] = str(m).zfill(2) #print('sub:', self.changing_path.substitute(**dic_date)) out = fnmatch.filter(flist, self.file_template.substitute(**dic_date)) out = [i for i in out if not i.endswith('.bz2')] # if there is data in the month if len(out)>0: t00 = timer() ## Get available file by month #convert string template to date formater ref = self.file_template.template.replace('$','%') ref = ref.replace('{','') ref = ref.replace('}','') # hack to add custom parsing in strptime # https://stackoverflow.com/a/54451291/1840524 import _strptime TimeRE = _strptime.TimeRE() TimeRE.update({'x': '(.h5)?'}) # match 0 or 1 '.h5' pattern _strptime._TimeRE_cache = TimeRE ref = ref.replace('*','%x') # process input list # remove bz2 files avail_date = np.array(sorted([datetime.datetime.strptime(f, ref) for f in out])) ## INFO: Get all theoretical dates, actually not useful here ## Note the use of MonthEnd to offset of one (or more) month precisely #start = datetime.datetime(y, m, 1) #all_date = pd.date_range(start, start+pd.tseries.offsets.MonthEnd(1), freq=self.freq) print(y, m, len(out), '>', len(avail_date)) ## Recorf all valid intervalls as [start_dat, end_date] of continuous range according to self.freq inter = [avail_date[0]] for d1,d2 in self.grouper(avail_date): # check if there is more than one self.freq between two following date if d2-d1!=pd.to_timedelta(self.freq): inter.append(d1) self.valid_year[y].append(inter) inter = [d2] inter.append(avail_date[-1]) self.valid_year[y].append(inter) #print(timer()-t00) print(timer()-t0) self.show_tot_time() ## Merge continuous intervals when changing month. for y,inter in self.valid_year.items(): merged = [] merged.append(inter[0]) for i1,i2 in self.grouper(inter): if i2[0]-i1[1]==pd.to_timedelta(self.freq): merged[-1][-1] = i2[1] # [d1,d2],[d3,d4] -> [d1,d4] else: merged.append(i2) self.valid_year[y] = merged self.show_tot_time()