def test_timesince(self):
now = utc_now()
then = now - datetime.timedelta(days=365)
eq_(timesince(then, now), '1 year')
then = now - datetime.timedelta(days=7)
eq_(timesince(then, now), '1 week')
then = now - datetime.timedelta(days=1)
eq_(timesince(then, now), '1 day')
then = now - datetime.timedelta(hours=1)
eq_(timesince(then, now), '1 hour')
then = now - datetime.timedelta(minutes=1)
eq_(timesince(then, now), '1 minute')
then = now - datetime.timedelta(seconds=1)
eq_(timesince(then, now), '1 second')
# more than one things
then = now - datetime.timedelta(days=365 + 7)
eq_(timesince(then, now), '1 year')
then = now - datetime.timedelta(days=40)
eq_(timesince(then, now), '1 month, 1 week')
then = now - datetime.timedelta(days=2, seconds=60 * 60)
eq_(timesince(then, now), '2 days, 1 hour')
then = now - datetime.timedelta(days=2, seconds=60 * 60 * 2)
eq_(timesince(then, now), '2 days, 2 hours')
then = now - datetime.timedelta(hours=1, seconds=60)
eq_(timesince(then, now), '1 hour, 1 minute')
then = now - datetime.timedelta(hours=2, seconds=60 * 2)
eq_(timesince(then, now), '2 hours, 2 minutes')
then = now - datetime.timedelta(minutes=3, seconds=10)
eq_(timesince(then, now), '3 minutes, 10 seconds')
then = now - datetime.timedelta(seconds=1)
eq_(timesince(then, now), '1 second')
then = now - datetime.timedelta(seconds=0)
eq_(timesince(then, now), '0 seconds')
def test_utc_now(self):
now = utc_now()
ok_(now.tzinfo)
dt = datetime.datetime.utcnow()
eq_(now.tzinfo.tzname(dt), 'UTC')
eq_(now.tzinfo.utcoffset(dt), datetime.timedelta(0))
eq_(now.tzinfo.dst(dt), datetime.timedelta(0))
def run(self, date=None):
# NOTE(willkg): This lets us have a dry-run app that doesn't run as
# a backfill app. In the normal case, this will get passed a date.
date = date or utc_now()
db_class = self.config.primary_destination.database_class
primary_database = db_class(self.config.primary_destination)
tx_class = self.config.primary_destination.transaction_executor_class
primary_transaction = tx_class(
self.config,
primary_database,
)
transactions = [primary_transaction]
db_class = self.config.secondary_destination.database_class
# The reason for checking if this is anything at all is
# because one way of disabling the secondary destination
# is to set the database_class to an empty string.
if db_class:
secondary_database = db_class(self.config.secondary_destination)
if secondary_database.config != primary_database.config:
# The secondary really is different from the first one.
# By default, if not explicitly set, it'll pick up the same
# resource values as the first one.
tx_class = (self.config.secondary_destination.
transaction_executor_class)
secondary_transaction = tx_class(
self.config,
secondary_database,
)
transactions.append(secondary_transaction)
target_date = (date - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
raw_adi_logs_pathname = os.path.join(
tempfile.gettempdir(),
"%s.raw_adi_logs.TEMPORARY%s" % (target_date, '.txt'))
try:
with codecs.open(raw_adi_logs_pathname, 'w', 'utf-8') as f:
hive = pyhs2.connect(
host=self.config.hive_host,
port=self.config.hive_port,
authMechanism=self.config.hive_auth_mechanism,
user=self.config.hive_user,
password=self.config.hive_password,
database=self.config.hive_database,
# the underlying TSocket setTimeout() wants milliseconds
timeout=self.config.timeout * 1000)
cur = hive.cursor()
query = self.config.query % target_date
cur.execute(query)
rows_written = 0
for row in cur:
if None in row:
continue
f.write("\t".join(
self.remove_control_characters(urllib2.unquote(v)).
replace('\\', '\\\\') if isinstance(v, basestring
) else str(v)
for v in row))
f.write("\n")
rows_written += 1
if not rows_written:
raise NoRowsWritten('hive yielded no rows to write')
self.config.logger.info('Wrote %d rows from doing hive query' %
rows_written)
self.persist_data(transactions, raw_adi_logs_pathname, target_date)
finally:
if os.path.isfile(raw_adi_logs_pathname):
os.remove(raw_adi_logs_pathname)
def test_timesince_oddballs(self):
now = utc_now()
then = now - datetime.timedelta(days=7)
# compare two dates
eq_(timesince(then.date(), now.date()), '1 week')
def run(self, date=None):
# NOTE(willkg): This lets us have a dry-run app that doesn't run as
# a backfill app. In the normal case, this will get passed a date.
date = date or utc_now()
db_class = self.config.primary_destination.database_class
primary_database = db_class(self.config.primary_destination)
tx_class = self.config.primary_destination.transaction_executor_class
primary_transaction = tx_class(
self.config,
primary_database,
)
transactions = [primary_transaction]
db_class = self.config.secondary_destination.database_class
# The reason for checking if this is anything at all is
# because one way of disabling the secondary destination
# is to set the database_class to an empty string.
if db_class:
secondary_database = db_class(self.config.secondary_destination)
if secondary_database.config != primary_database.config:
# The secondary really is different from the first one.
# By default, if not explicitly set, it'll pick up the same
# resource values as the first one.
tx_class = (
self.config.secondary_destination
.transaction_executor_class
)
secondary_transaction = tx_class(
self.config,
secondary_database,
)
transactions.append(secondary_transaction)
target_date = (date - datetime.timedelta(days=1)).strftime('%Y-%m-%d')
raw_adi_logs_pathname = os.path.join(
tempfile.gettempdir(),
"%s.raw_adi_logs.TEMPORARY%s" % (
target_date,
'.txt'
)
)
try:
with codecs.open(raw_adi_logs_pathname, 'w', 'utf-8') as f:
hive = pyhs2.connect(
host=self.config.hive_host,
port=self.config.hive_port,
authMechanism=self.config.hive_auth_mechanism,
user=self.config.hive_user,
password=self.config.hive_password,
database=self.config.hive_database,
# the underlying TSocket setTimeout() wants milliseconds
timeout=self.config.timeout * 1000
)
cur = hive.cursor()
query = self.config.query % target_date
cur.execute(query)
rows_written = 0
for row in cur:
if None in row:
continue
f.write(
"\t"
.join(
self.remove_control_characters(
urllib2.unquote(v)
).replace('\\', '\\\\')
if isinstance(v, basestring) else str(v)
for v in row
)
)
f.write("\n")
rows_written += 1
if not rows_written:
raise NoRowsWritten('hive yielded no rows to write')
self.config.logger.info(
'Wrote %d rows from doing hive query' % rows_written
)
self.persist_data(transactions, raw_adi_logs_pathname, target_date)
finally:
if os.path.isfile(raw_adi_logs_pathname):
os.remove(raw_adi_logs_pathname)
