def testDirectoryRange_Month():
s = Scraper('%Y%m/%d/%j_%H.txt')
startdate = parse_time((2008, 2, 20, 10))
enddate = parse_time((2008, 3, 2, 5))
timerange = TimeRange(startdate, enddate)
assert len(s.range(timerange)) == 12
startdate = parse_time((2009, 2, 20, 10))
enddate = parse_time((2009, 3, 2, 5))
timerange = TimeRange(startdate, enddate)
assert len(s.range(timerange)) == 11
def testDirectoryRangeFalse():
s = Scraper('%Y%m%d/%Y%m%d_%H.fit.gz')
directory_list = [
'20091230/', '20091231/', '20100101/', '20090102/', '20090103/'
]
timerange = TimeRange('2009/12/30', '2010/01/03')
assert s.range(timerange) != directory_list
def testDirectoryRegex():
# Test for Windows where '\' is a path separator and not part of the regex
s = Scraper(
'scheme://a.url.with/a/few/forward/slashes/andbacklash\\inthename.ext',
regex=True)
timerange = TimeRange('2019-02-01', '2019-02-03')
directory = s.range(timerange)
assert directory == ['scheme://a.url.with/a/few/forward/slashes/']
def testDirectoryRange():
s = Scraper('%Y/%m/%d/%Y%m%d_%H.fit.gz')
directory_list = [
'2009/12/30/', '2009/12/31/', '2010/01/01/', '2010/01/02/',
'2010/01/03/'
]
timerange = TimeRange('2009-12-30', '2010-01-03')
assert s.range(timerange) == directory_list
def testDirectoryRange_single():
s = Scraper('%Y%m%d/%H_%M.csv')
startdate = parse_time((2010, 10, 10, 5, 0))
enddate = parse_time((2010, 10, 10, 7, 0))
timerange = TimeRange(startdate, enddate)
assert len(s.range(timerange)) == 1
def testDirectoryRangeHours():
s = Scraper('%Y%m%d_%H/%H%M.csv')
timerange = TimeRange('2009-12-31T23:40:00', '2010-01-01T01:15:00')
assert len(s.range(timerange)) == 3 # 3 directories (1 per hour)
def testNoDateDirectory():
s = Scraper('mySpacecraft/myInstrument/xMinutes/aaa%y%b.ext')
directory_list = ['mySpacecraft/myInstrument/xMinutes/']
timerange = TimeRange('2009/11/20', '2010/01/03')
assert s.range(timerange) == directory_list
def testNoDirectory():
s = Scraper('files/%Y%m%d_%H%M.dat')
startdate = parse_time((2010, 1, 10, 20, 30))
enddate = parse_time((2010, 1, 20, 20, 30))
timerange = TimeRange(startdate, enddate)
assert len(s.range(timerange)) == 1