def tearDown(self):
#CLOSE THE CACHED DB CONNECTIONS
bz_etl.close_db_connections()
if all_db:
Log.error("not all db connections are closed")
Log.stop()
def tearDown(self):
#CLOSE THE CACHED DB CONNECTIONS
bz_etl.close_db_connections()
if all_db:
Log.error("not all db connections are closed")
Log.stop()
def test_replication():
try:
settings=startup.read_settings(filename="replication_settings.json")
Log.start(settings.debug)
source=ElasticSearch(settings.source)
destination=replicate.get_or_create_index(settings["destination"], source)
replicate.replicate(source, destination, [537285], CNV.string2datetime("19900101", "%Y%m%d"))
finally:
Log.stop()
def test_replication():
try:
settings = startup.read_settings(filename="replication_settings.json")
Log.start(settings.debug)
source = ElasticSearch(settings.source)
destination = replicate.get_or_create_index(settings["destination"],
source)
replicate.replicate(source, destination, [537285],
CNV.string2datetime("19900101", "%Y%m%d"))
finally:
Log.stop()
def main():
"""
MEANT TO BE RUN JUST ONCE IN DEVELOPMENT TO CONVERT A BIG PUBLIC
DATABASE (8G+) INTO A TINY TESTING DB (FOR ADDING TO REPOSITORY)
"""
try:
settings=startup.read_settings()
Log.start(settings.debug)
input=raw_input("We are going to totally wipe out the "+settings.bugzilla.schema.upper()+" schema at "+settings.bugzilla.host.upper()+"! Type \"YES\" to continue: ")
if input!="YES":
Log.note("Aborted. No Changes made.")
return
Log.note("Scrubbing db of those pesky records.")
Log.note("This is going to take hours ...")
DB.execute_file(settings.bugzilla, "./tests/resources/sql/scrub_db.sql", {
"schema":settings.bugzilla.schema,
"bug_list":SQL(settings.param.bugs)
})
Log.note("... Done!")
finally:
Log.stop()
# GET LAST UPDATED
time_file = File(settings.param.last_replication_time)
from_file = None
if time_file.exists:
from_file = CNV.milli2datetime(CNV.value2int(time_file.read()))
from_es = get_last_updated(destination)
last_updated = nvl(MIN(from_file, from_es), CNV.milli2datetime(0))
current_time = datetime.utcnow()
pending = get_pending(source, last_updated)
with ThreadedQueue(destination, size=1000) as data_sink:
replicate(source, data_sink, pending, last_updated)
# RECORD LAST UPDATED
time_file.write(unicode(CNV.datetime2milli(current_time)))
def start():
try:
settings=startup.read_settings()
Log.start(settings.debug)
main(settings)
except Exception, e:
Log.error("Problems exist", e)
finally:
Log.stop()
if __name__=="__main__":
start()
def tearDown(self):
Log.stop()
def tearDown(self):
Log.stop()