def test_datetimeToDate(self):
"""test datetimeToDate"""
self.assertEqual(Utils.datetimeToDate(datetime.date(2016, 12, 10)),
datetime.date(2016, 12, 10))
self.assertEqual(
Utils.datetimeToDate(datetime.datetime(2016, 12, 10, 11, 5)),
datetime.date(2016, 12, 10))
def _processBool(conf):
for k in conf:
if 'link' in conf[k]:
conf[k]['link'] = Utils.toBool(conf[k]['link'])
if 'error' in conf[k]:
conf[k]['error'] = Utils.toBool(conf[k]['error'])
return conf
def test_dateForPrinting(self):
"""dateForPrinting"""
dt = datetime.datetime(2012, 8, 30, 8, 5)
ans1 = '[2012-08-30T08:05:00]'
self.assertEqual(ans1, Utils.dateForPrinting(dt))
self.assertEqual(
'[' + datetime.datetime.now().replace(microsecond=0).isoformat() +
']', Utils.dateForPrinting())
def test_toNone(self):
"""toNone"""
invals = ['', 'None', 'none', 'NONE']
for v in invals:
self.assertTrue(Utils.toNone(v) is None)
invals = ['sdg', 'false', False, 'sagdfa']
for v in invals:
self.assertFalse(Utils.toNone(v) is None)
def test_toBool(self):
"""toBool"""
invals = ['True', 'true', True, 1, 'Yes', 'yes']
for v in invals:
self.assertTrue(Utils.toBool(v))
invals = ['sdg', 'false', False, 'sagdfa']
for v in invals:
self.assertFalse(Utils.toBool(v))
def test_split_code_args(self):
"""split_code_args"""
self.assertEqual(["code", "hello", "outfile"],
Utils.split_code_args("code hello outfile"))
self.assertEqual(["code", "-n hello", "outfile"],
Utils.split_code_args("code -n hello outfile"))
self.assertEqual(
["code", "infile", "--flag hello", "outfile"],
Utils.split_code_args("code infile --flag hello outfile"))
def test_expandDates(self):
"""expandDates"""
d1 = datetime.datetime(2013, 1, 1)
d2 = datetime.datetime(2013, 1, 2)
d3 = datetime.datetime(2013, 1, 3)
self.assertEqual(list(Utils.expandDates(d1, d1)), [d1])
self.assertEqual(list(Utils.expandDates(d1, d2)), [d1, d2])
self.assertEqual(list(Utils.expandDates(d1, d3)), [d1, d2, d3])
def test_chunker(self):
"""chunker()"""
self.assertEqual(list(Utils.chunker(range(10), 3)),
[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]])
self.assertEqual(list(Utils.chunker(range(10), 4)),
[[0, 1, 2, 3], [4, 5, 6, 7], [8, 9]])
self.assertEqual(list(Utils.chunker(range(10), 4)),
[[0, 1, 2, 3], [4, 5, 6, 7], [8, 9]])
self.assertEqual(list(Utils.chunker(range(10), 10)),
[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]])
self.assertEqual(list(Utils.chunker(range(10), 20)),
[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]])
def test_strargs_to_args3(self):
"""strargs_to_args"""
self.assertEqual(Utils.strargs_to_args(['--arg1=arg', '--arg2=arg2']),
{
'--arg1': 'arg',
'--arg2': 'arg2'
})
def test_progressbar(self):
"""progressbar shouldhave a known output"""
realstdout = sys.stdout
output = StringIO.StringIO()
sys.stdout = output
self.assertEqual(Utils.progressbar(0, 1, 100), None)
result = output.getvalue()
output.close()
self.assertEqual(result, "\rDownload Progress ...0%")
sys.stdout = realstdout
def test_readconfig(self):
"""test readconfig"""
self.assertEqual(
{
'section2': {
'sect2a': 'sect2_value1'
},
'section1': {
'sect1a': 'sect1_value1',
'sect1b': 'sect1_value2'
}
}, Utils.readconfig(os.path.dirname(__file__) + '/testconfig.txt'))
def test_daterange_to_dates(self):
"""daterange_to_dates"""
daterange = [
datetime.datetime(2000, 1, 4),
datetime.datetime(2000, 1, 6)
]
expected = [
datetime.datetime(2000, 1, 4),
datetime.datetime(2000, 1, 5),
datetime.datetime(2000, 1, 6)
]
self.assertEqual(expected, Utils.daterange_to_dates(daterange))
daterange = [
datetime.datetime(2000, 1, 4),
datetime.datetime(2000, 1, 5, 23)
]
expected = [
datetime.datetime(2000, 1, 4),
datetime.datetime(2000, 1, 5)
]
self.assertEqual(expected, Utils.daterange_to_dates(daterange))
def test_toDatetime(self):
"""Test toDatetime"""
self.assertEqual(datetime.datetime(2010, 1, 1),
Utils.toDatetime('2010-1-1'))
self.assertEqual(datetime.datetime(2010, 1, 1),
Utils.toDatetime(datetime.datetime(2010, 1, 1)))
self.assertEqual(datetime.datetime(2010, 1, 1, 5),
Utils.toDatetime(datetime.datetime(2010, 1, 1, 5)))
self.assertEqual(datetime.datetime(2010, 1, 1),
Utils.toDatetime(datetime.date(2010, 1, 1)))
self.assertEqual(datetime.datetime(2010, 1, 1, 23, 59, 59, 999999),
Utils.toDatetime('2010-1-1', end=True))
self.assertEqual(
datetime.datetime(2010, 1, 1),
Utils.toDatetime(datetime.datetime(2010, 1, 1), end=True))
self.assertEqual(
datetime.datetime(2010, 1, 1, 5),
Utils.toDatetime(datetime.datetime(2010, 1, 1, 5), end=True))
self.assertEqual(datetime.datetime(2010, 1, 1, 23, 59, 59, 999999),
Utils.toDatetime(datetime.date(2010, 1, 1), end=True))
def test_flatten(self):
"""flatten"""
self.assertEqual([1, 2, 3, 4, 5, 6, 7, 8, 9],
list(
Utils.flatten([[1, 2], [3, 4, 5], [6, 7], [8],
[9]])))
def _processNone(conf):
for k in conf:
if 'ignore' in conf[k]:
conf[k]['ignore'] = Utils.toNone(conf[k]['ignore'])
return conf
def test_processRunnin1(self):
"""processRunning"""
self.assertTrue(Utils.processRunning(os.getpid()))
self.assertFalse(Utils.processRunning(44565))
if len(args) != 0:
parser.error("incorrect number of arguments")
if options.mission is None:
parser.error("-m must be specified")
dbu = DButils.DButils(options.mission, echo=options.echo)
# If we will be editing the DB we have to have lock
if options.fix:
# check currently processing
curr_proc = dbu.currentlyProcessing()
if curr_proc: # returns False or the PID
# check if the PID is running
if Utils.processRunning(curr_proc):
# we still have an instance processing, don't start another
dbu.closeDB()
DBlogging.dblogger.error(
"There is a process running, can't start another: PID: %d"
% (curr_proc))
raise (ProcessException(
"There is a process running, can't start another: PID: %d"
% (curr_proc)))
else:
# There is a processing flag set but it died, don't start another
dbu.closeDB()
DBlogging.dblogger.error(
"There is a processing flag set but it died, don't start another"
)
raise (ProcessException(
(options, args) = parser.parse_args()
if len(args) != 1:
parser.error("incorrect number of arguments")
conffile = os.path.expanduser(os.path.expandvars(os.path.abspath(args[0])))
if not os.path.isfile(conffile):
parser.error("could not read config file: {0}".format(conffile))
conf = readconfig(conffile)
conf = _processSubs(conf)
conf = _processDates(conf)
print('Read and parsed config file: {0}'.format(conffile))
# figure out the dates we are going to use
# make a range of dates
dates = list(Utils.expandDates(conf['settings']['startdate'], conf['settings']['enddate']))
dates = [v.date() for v in dates]
dates = list(Utils.chunker(dates, int(conf['panel']['daysperplot'])))
# make the arrays that hold the files
dbu = DButils.DButils(conf['settings']['mission'], echo=echo)
# go through the panel and see how many plots there are
nplots = _get_nplots(conf)
ans = []
for pnum, ind_pnum in enumerate(range(nplots), 1):
ans.append([])
n_prods = _get_nproducts(conf, pnum)
prods = []
def test_strargs_to_args5(self):
"""strargs_to_args"""
self.assertEqual(Utils.strargs_to_args('--arg'), {})
def test_strargs_to_args2(self):
"""strargs_to_args"""
self.assertEqual(Utils.strargs_to_args('--arg1=arg'),
{'--arg1': 'arg'})
def test_strargs_to_args1(self):
"""strargs_to_args"""
self.assertTrue(Utils.strargs_to_args(None) is None)
raise (ValueError("No mission in DB, this must be an empty db"))
if len(missions) > 1:
raise (NotImplementedError("Can't yet handle multi mission db"))
mission = missions[0]
mission = dbu.getEntry('Mission', mission)
for d in dir(mission):
if d[0] == '_': continue
out.set('mission', d, str(getattr(mission, d)))
# get satellite
out.add_section('satellite')
sats = dbu.getAllSatellites()
for v in sats:
if v['mission'].mission_id != mission.mission_id:
raise (NotImplementedError("Can't yet handle multi mission db"))
nsats = len(Utils.unique([v['satellite'].satellite_id for v in sats]))
if nsats > 1 and options.satellite is None:
raise (ValueError("More than one sat in db and no --satellite set\n {0}".format(
[v['satellite'].satellite_name for v in sats])))
elif nsats == 1 and options.satellite is None:
options.satellite = sats['satellite'].satellite_name
for s in sats:
if s['satellite'].satellite_name != options.satellite:
continue
for d in dir(s['satellite']):
if d[0] == '_': continue
out.set('satellite', d, str(getattr(s['satellite'], d)))
# get instrument
out.add_section('instrument')
insts = dbu.getAllInstruments()
DBlogging.change_logfile(logname)
if options.loglevel not in DBlogging.LEVELS:
parser.error("invalid --log-level specified")
DBlogging.dblogger.setLevel(DBlogging.LEVELS[options.loglevel])
pq = dbprocessing.ProcessQueue(options.mission,
dryrun=options.dryrun,
echo=options.echo)
# check currently processing
curr_proc = pq.dbu.currentlyProcessing()
if curr_proc: # returns False or the PID
# check if the PID is running
if Utils.processRunning(curr_proc):
# we still have an instance processing, don't start another
pq.dbu.closeDB()
DBlogging.dblogger.error(
"There is a process running, can't start another: PID: %d" %
(curr_proc))
raise (ProcessException(
"There is a process running, can't start another: PID: %d" %
(curr_proc)))
else:
# There is a processing flag set but it died, don't start another
pq.dbu.closeDB()
DBlogging.dblogger.error(
"There is a processing flag set but it died, don't start another"
)
raise (ProcessException(
def test_unique(self):
"""unique"""
self.assertEqual(Utils.unique(range(5)), range(5))
self.assertEqual(Utils.unique([1, 1, 2, 2, 3]), [1, 2, 3])
self.assertEqual(Utils.unique([1, 1, 3, 2, 2, 3]), [1, 3, 2])
def test_parseVersion(self):
"""parseVersion"""
self.assertEqual(Version.Version(1, 2, 3), Utils.parseVersion('1.2.3'))
self.assertRaises(TypeError, Utils.parseVersion, '1.2')
def test_parseDate(self):
"""parseDate"""
self.assertEqual(datetime.datetime(2013, 1, 1),
Utils.parseDate('2013-01-01'))
self.assertRaises(ValueError, Utils.parseDate, '2013-13-01')
def test_dirSubs(self):
"""dirSubs substitutions should work"""
path = '{Y}{m}{d}'
filename = 'test_filename'
utc_file_date = datetime.date(2012, 4, 12)
utc_start_time = datetime.datetime(2012, 4, 12, 1, 2, 3)
version = '1.2.3'
version2 = Version.Version(3, 2, 1)
self.assertEqual(
'20120412',
Utils.dirSubs(path, filename, utc_file_date, utc_start_time,
version))
path = '{DATE}'
self.assertEqual(
'20120412',
Utils.dirSubs(path, filename, utc_file_date, utc_start_time,
version))
path = '{Y}{b}{d}'
self.assertEqual(
'2012Apr12',
Utils.dirSubs(path, filename, utc_file_date, utc_start_time,
version))
path = '{y}{j}'
self.assertEqual(
'12103',
Utils.dirSubs(path, filename, utc_file_date, utc_start_time,
version))
path = '{VERSION}'
self.assertEqual(
'1.2.3',
Utils.dirSubs(path, filename, utc_file_date, utc_start_time,
version))
self.assertEqual(
'3.2.1',
Utils.dirSubs(path, filename, utc_file_date, utc_start_time,
version2))
path = '{H}{M}{S}'
self.assertEqual(
'010203',
Utils.dirSubs(path, filename, utc_file_date, utc_start_time,
version))
# Substitutions that require referring to the DB...
filename = 'testDB_000_000.raw'
path = '{INSTRUMENT}'
self.assertEqual(
'rot13',
Utils.dirSubs(path,
filename,
utc_file_date,
utc_start_time,
version,
dbu=self.dbu))
path = '{SATELLITE}'
self.assertEqual(
'testDB-a',
Utils.dirSubs(path,
filename,
utc_file_date,
utc_start_time,
version,
dbu=self.dbu))
path = '{SPACECRAFT}'
self.assertEqual(
'testDB-a',
Utils.dirSubs(path,
filename,
utc_file_date,
utc_start_time,
version,
dbu=self.dbu))
path = '{MISSION}'
self.assertEqual(
'testDB',
Utils.dirSubs(path,
filename,
utc_file_date,
utc_start_time,
version,
dbu=self.dbu))
path = '{PRODUCT}'
self.assertEqual(
'testDB_rot13_L0_first',
Utils.dirSubs(path,
filename,
utc_file_date,
utc_start_time,
version,
dbu=self.dbu))
# Verify that unknown values are ignored
path = '{xxx}'
self.assertEqual(
'{xxx}',
Utils.dirSubs(path,
filename,
utc_file_date,
utc_start_time,
version,
dbu=self.dbu))
startDate = dup.parse(options.startDate)
else:
startDate = datetime.datetime(2012, 8, 30)
if options.endDate is not None:
endDate = dup.parse(options.endDate)
else:
endDate = datetime.datetime.now()
if endDate < startDate:
parser.error("endDate must be >= to startDate")
pq = dbprocessing.ProcessQueue(options.mission,
dryrun=options.dryrun,
echo=options.echo)
dates = Utils.expandDates(startDate, endDate)
print('dates', list(dates))
inproc = args[0]
# get the input products for a process
products = pq.dbu.getInputProductID(inproc)
print('products', products)
runme = []
for d in dates:
print("Processing date: {0}".format(d))
# we need a file_id that goes into the process
input_files = []
for p, opt in products:
prod_ = pq.dbu.getEntry('Product', p)