def does_period_have_data(inst, db, ssl, table, start_dt, stop_dt):
assert start_dt is None or isinstance(start_dt, datetime.datetime)
assert stop_dt is None or isinstance(stop_dt, datetime.datetime)
if start_dt is None or stop_dt is None:
ymd_filter = ''
else:
ymd_filter = tools.get_ymd_filter(tools.dt_to_iso8601(start_dt), tools.dt_to_iso8601(stop_dt))
ssl_opt = tools.format_ssl(ssl)
sql = """ SELECT 'found-data'
FROM {tab} c
WHERE 1 = 1
{filter}
LIMIT 1
""".format(tab=table, filter=ymd_filter)
sql = ' '.join(sql.split())
cmd = """ impala-shell -i {inst} -d {db} --quiet -B {ssl} -q "{sql}" | columns | cut -f 1
""".format(inst=inst, db=db, ssl=ssl_opt, sql=sql)
r = envoy.run(cmd)
if r.status_code != 0:
print(cmd)
print(r.std_err)
print(r.std_out)
tools.abort("Error: does_period_have_data() failed!")
else:
if 'found-data' in r.std_out.strip():
return True
else:
return False
def main():
next_start_dt = None
next_stop_dt = None
next_period_has_data = False
args = get_args()
validate_args(args)
if args.mode == 'incremental':
if args.data_start_ts_prior == '':
first_dt = get_first_dt_by_ymd(args.inst, args.db, args.table, args.ssl)
if first_dt:
next_period_has_data = True
next_start_dt = tools.dt_override(first_dt, hour=0, minute=0, second=0, ms=0)
next_stop_dt = tools.dt_override(first_dt, hour=23, minute=59, second=59, ms=999999)
else:
try:
next_start_dt, next_stop_dt = tools.get_next_dt(tools.iso8601_to_dt(args.data_start_ts_prior),
tools.iso8601_to_dt(args.data_stop_ts_prior))
except ValueError:
print('invalid data_start_ts_prior or data_stop_ts_prior: %s and %s' % (args.data_start_ts_prior, args.data_stop_ts_prior))
next_period_has_data = does_period_have_data(args.inst, args.db, args.ssl, args.table, next_start_dt, next_stop_dt)
else:
next_period_has_data = does_period_have_data(args.inst, args.db, args.ssl, args.table, start_dt=None, stop_dt=None)
results = {}
if next_period_has_data:
results['table_status'] = 'active'
results['mode'] = args.mode
if args.mode == 'incremental':
results['data_start_ts'] = tools.dt_to_iso8601(next_start_dt, precision='second')
results['data_stop_ts'] = tools.dt_to_iso8601(next_stop_dt, precision='second')
else:
results['data_start_ts'] = ''
results['data_stop_ts'] = ''
else:
results['table_status'] = 'inactive'
results['mode'] = args.mode
results['data_start_ts'] = args.data_start_ts_prior
results['data_stop_ts'] = args.data_stop_ts_prior
results['rc'] = INTERNAL_STATUS_CD
results['log'] = 'start_ts_prior: %s, stop_ts_prior: %s, next_period_has_data: %s'\
% (args.data_start_ts_prior, args.data_stop_ts_prior, next_period_has_data)
for (key, val) in os.environ.items():
if 'hapinsp' in key:
results['log'] += ', %s:%s' % (key, val)
print(hapinsp_formatter.transform_args(results))
def test_invalid_args(self):
with pytest.raises(ValueError):
mod.dt_to_iso8601(dtdt(2016, 1, 2, 3, 4, 5, 123456), 'foo', 'ms')
with pytest.raises(ValueError):
mod.dt_to_iso8601(dtdt(2016, 1, 2, 3, 4, 5, 123456), 'ext', 'foo')
with pytest.raises(ValueError):
mod.dt_to_iso8601('foo', 'ext', 'ms')
def test_ext_ms(self):
assert mod.dt_to_iso8601(dtdt(2016, 1, 2, 3, 4, 5, 123456), 'ext', 'ms') == '2016-01-02T03:04:05.123456'
def test_ext_second(self):
assert mod.dt_to_iso8601(dtdt(2016, 1, 2, 3, 4, 5), 'ext', 'second') == '2016-01-02T03:04:05'
def test_none(self):
with pytest.raises(ValueError):
mod.dt_to_iso8601(None, 'basic', 'ms')
def test_basic_ms(self):
assert mod.dt_to_iso8601(dtdt(2016, 1, 2, 3, 4, 5, 123456), 'basic', 'ms') == '20160102T030405123456'
def test_basic_second(self):
assert mod.dt_to_iso8601(dtdt(2016, 1, 2, 3, 4, 5), 'basic', 'second') == '20160102T030405'