def test_download_files_with_wait_all_successful(self):
temp_dir = tempfile.mkdtemp()
try:
prev_friday = util.get_previous_friday_from_date(
datetime.now(tzlocal()))
sat = prev_friday + timedelta(days=1)
dse = sat - datetime(1970, 01, 01, tzinfo=tzutc())
secs_since_epoch = self.get_total_seconds(dse)
nonpoly = os.path.join(temp_dir, DataImportTask.NONPOLYMER_TSV)
f = open(nonpoly, 'w')
f.write('nonpoly\n')
f.flush()
f.close()
os.utime(nonpoly, (secs_since_epoch, secs_since_epoch))
seq = os.path.join(temp_dir, DataImportTask.SEQUENCE_TSV)
f = open(seq, 'w')
f.write('seq\n')
f.flush()
f.close()
os.utime(seq, (secs_since_epoch, secs_since_epoch))
oldseq = os.path.join(temp_dir, DataImportTask.OLDSEQUENCE_TSV)
f = open(oldseq, 'w')
f.write('oldseq\n')
f.flush()
f.close()
os.utime(oldseq, (secs_since_epoch, secs_since_epoch))
crystal = os.path.join(temp_dir, DataImportTask.CRYSTALPH_TSV)
f = open(crystal, 'w')
f.write('crystal\n')
f.flush()
f.close()
os.utime(crystal, (secs_since_epoch, secs_since_epoch))
comp = os.path.join(temp_dir, DataImportTask.COMPINCHI_ICH)
f = open(comp, 'w')
f.write('comp\n')
f.flush()
f.close()
params = D3RParameters()
params.skipimportwait = False
params.importretry = 2
params.importsleep = 0
params.compinchi = 'file://' + temp_dir
task = DataImportTask(temp_dir, params)
task.create_dir()
val = task._download_files('file://' + temp_dir)
self.assertEqual(task.get_error(), None)
self.assertTrue(val)
finally:
shutil.rmtree(temp_dir)
def test_has_url_been_updated_since_start_of_celpp_week(self):
temp_dir = tempfile.mkdtemp()
try:
try:
util.has_url_been_updated_since_start_of_celpp_week(None)
self.fail('Expected exception')
except Exception:
pass
fakefile = os.path.join(temp_dir, 'foo')
try:
util.has_url_been_updated_since_start_of_celpp_week('file://' +
fakefile)
self.fail('Expected exception')
except URLError:
pass
f = open(fakefile, 'w')
f.write('hi\n')
f.flush()
f.close()
prev_friday = util.get_previous_friday_from_date(
datetime.now(tzlocal()))
thurs = prev_friday - timedelta(days=1)
dse = thurs - datetime(1970, 01, 01, tzinfo=tzutc())
secs_since_epoch = self.get_total_seconds(dse)
os.utime(fakefile, (secs_since_epoch, secs_since_epoch))
val = util.has_url_been_updated_since_start_of_celpp_week(
'file://' + fakefile)
self.assertEqual(val, False)
sat = prev_friday + timedelta(days=1)
dse = sat - datetime(1970, 01, 01, tzinfo=tzutc())
secs_since_epoch = self.get_total_seconds(dse)
os.utime(fakefile, (secs_since_epoch, secs_since_epoch))
val = util.has_url_been_updated_since_start_of_celpp_week(
'file://' + fakefile)
self.assertEqual(val, True)
finally:
shutil.rmtree(temp_dir)
def test_is_datetime_after_celpp_week_start(self):
try:
util.is_datetime_after_celpp_week_start(None)
self.fail('Expected exception')
except Exception:
pass
try:
util.is_datetime_after_celpp_week_start('hi')
self.fail('Expected exception')
except AssertionError:
pass
prev_friday = util.get_previous_friday_from_date(
datetime.now(tzlocal()))
oneday = timedelta(days=1)
sat = prev_friday + oneday
self.assertTrue(util.is_datetime_after_celpp_week_start(sat))
thurs = prev_friday - oneday
self.assertFalse(util.is_datetime_after_celpp_week_start(thurs))
def test_wait_for_url_to_be_updated(self):
temp_dir = tempfile.mkdtemp()
try:
# skipimportwait is None
params = D3RParameters()
task = DataImportTask(temp_dir, params)
task._wait_for_url_to_be_updated('foo')
# skipimportwait is True
params = D3RParameters()
params.skipimportwait = True
task = DataImportTask(temp_dir, params)
task._wait_for_url_to_be_updated('foo')
params = D3RParameters()
params.skipimportwait = False
params.importretry = 2
params.importsleep = 0
task = DataImportTask(temp_dir, params)
fakefile = os.path.join(temp_dir, 'foo')
f = open(fakefile, 'w')
f.write('hi\n')
f.flush()
f.close()
prev_friday = util.get_previous_friday_from_date(
datetime.now(tzlocal()))
thurs = prev_friday - timedelta(days=1)
dse = thurs - datetime(1970, 01, 01, tzinfo=tzutc())
secs_since_epoch = self.get_total_seconds(dse)
os.utime(fakefile, (secs_since_epoch, secs_since_epoch))
# test retry exceeded
try:
task._wait_for_url_to_be_updated('file://' + fakefile)
self.fail('Expected ImportRetryCountExceededError')
except ImportRetryCountExceededError:
pass
# test importsleep not defined
params = D3RParameters()
params.skipimportwait = False
params.importretry = 2
task.set_args(params)
try:
task._wait_for_url_to_be_updated('file://' + fakefile)
self.fail('Expected ImportRetryCountExceededError')
except ImportRetryCountExceededError:
pass
sat = prev_friday + timedelta(days=1)
dse = sat - datetime(1970, 01, 01, tzinfo=tzutc())
secs_since_epoch = self.get_total_seconds(dse)
os.utime(fakefile, (secs_since_epoch, secs_since_epoch))
task._wait_for_url_to_be_updated('file://' + fakefile)
finally:
shutil.rmtree(temp_dir)
def test_get_previous_date_of_previous_friday_from_date(self):
# test None passed in
try:
util.get_previous_friday_from_date(None)
self.fail('Expected exception')
except Exception:
pass
# test non date passed in
try:
util.get_previous_friday_from_date('hello')
self.fail('Expected exception')
except Exception:
pass
# test date on Friday
dt = datetime(2016, 6, 17, 8, 0, 5)
newdt = util.get_previous_friday_from_date(dt)
self.assertTrue(newdt.year == 2016)
self.assertTrue(newdt.day == 17)
self.assertTrue(newdt.month == 6)
self.assertTrue(newdt.hour == 8)
self.assertTrue(newdt.minute == 0)
self.assertTrue(newdt.second == 5)
# test date on Saturday
dt = datetime(2015, 8, 1, 15, 20, 2)
newdt = util.get_previous_friday_from_date(dt)
self.assertTrue(newdt.year == 2015)
self.assertTrue(newdt.day == 31)
self.assertTrue(newdt.month == 7)
self.assertTrue(newdt.hour == 15)
self.assertTrue(newdt.minute == 20)
self.assertTrue(newdt.second == 2)
# test date on Sunday
dt = datetime(2016, 6, 19, 8, 0, 5)
newdt = util.get_previous_friday_from_date(dt)
self.assertTrue(newdt.year == 2016)
self.assertTrue(newdt.day == 17)
self.assertTrue(newdt.month == 6)
# test date on Monday
dt = datetime(2016, 6, 20, 8, 0, 5)
newdt = util.get_previous_friday_from_date(dt)
self.assertTrue(newdt.year == 2016)
self.assertTrue(newdt.day == 17)
self.assertTrue(newdt.month == 6)
# test date on Tuesday
dt = datetime(2016, 6, 21, 8, 0, 5)
newdt = util.get_previous_friday_from_date(dt)
self.assertTrue(newdt.year == 2016)
self.assertTrue(newdt.day == 17)
self.assertTrue(newdt.month == 6)
# test date on Wednesday
dt = datetime(2016, 6, 22, 8, 0, 5)
newdt = util.get_previous_friday_from_date(dt)
self.assertTrue(newdt.year == 2016)
self.assertTrue(newdt.day == 17)
self.assertTrue(newdt.month == 6)
# test date on Thursday
dt = datetime(2016, 6, 23, 8, 0, 5)
newdt = util.get_previous_friday_from_date(dt)
self.assertTrue(newdt.year == 2016)
self.assertTrue(newdt.day == 17)
self.assertTrue(newdt.month == 6)