def test_specific_bugs(self): """ USE A MYSQL DATABASE TO FILL AN ES INSTANCE (USE Fake_ES() INSTANCES TO KEEP THIS TEST LOCAL) WITH VERSIONS OF BUGS FROM settings.param.bugs. COMPARE THOSE VERSIONS TO A REFERENCE ES (ALSO CHECKED INTO REPOSITORY) """ # settings.param.allow_private_bugs = True database.make_test_instance(self.settings.bugzilla) with DB(self.settings.bugzilla) as db: candidate = elasticsearch.make_test_instance("candidate", self.settings.candidate) reference = elasticsearch.open_test_instance("reference", self.settings.private_bugs_reference) #SETUP RUN PARAMETERS param = Struct() param.end_time = CNV.datetime2milli(get_current_time(db)) param.start_time = 0 param.start_time_str = extract_bugzilla.milli2string(db, 0) param.alias_file = self.settings.param.alias_file param.bug_list = self.settings.param.bugs param.allow_private_bugs = self.settings.param.allow_private_bugs with ThreadedQueue(candidate, size=1000) as output: etl(db, output, param, please_stop=None) #COMPARE ALL BUGS Thread.sleep(2) # MUST SLEEP WHILE ES DOES ITS INDEXING compare_both(candidate, reference, self.settings, self.settings.param.bugs)
def test_public_etl(self): """ ENSURE ETL GENERATES WHAT'S IN THE REFERENCE FILE """ File(self.settings.param.first_run_time).delete() File(self.settings.param.last_run_time).delete() self.settings.param.allow_private_bugs = Null database.make_test_instance(self.settings.bugzilla) es = elasticsearch.make_test_instance("candidate", self.settings.fake.bugs) es_comments = elasticsearch.make_test_instance("candidate_comments", self.settings.fake.comments) bz_etl.main(self.settings, es, es_comments) ref = elasticsearch.open_test_instance("reference", self.settings.public_bugs_reference) compare_both(es, ref, self.settings, self.settings.param.bugs) #DIRECT COMPARE THE FILE JSON can = File(self.settings.fake.comments.filename).read() ref = File(self.settings.public_comments_reference.filename).read() if can != ref: found = -1 for i, c in enumerate(can): if can[i] != ref[i]: found = i break Log.error("Comments do not match reference\n{{sample}}", {"sample": can[MIN(0, found - 100):found + 100:]})
def test_private_etl(self): """ ENSURE IDENTIFIABLE INFORMATION DOES NOT EXIST ON ANY BUGS """ File(self.settings.param.first_run_time).delete() File(self.settings.param.last_run_time).delete() self.settings.param.allow_private_bugs = True database.make_test_instance(self.settings.bugzilla) es = elasticsearch.make_test_instance("candidate", self.settings.fake.bugs) es_comments = elasticsearch.make_test_instance("candidate_comments", self.settings.fake.comments) bz_etl.main(self.settings, es, es_comments) ref = elasticsearch.open_test_instance("reference", self.settings.private_bugs_reference) compare_both(es, ref, self.settings, self.settings.param.bugs) #DIRECT COMPARE THE FILE JSON can = File(self.settings.fake.comments.filename).read() ref = File(self.settings.private_comments_reference.filename).read() if can != ref: for i, c in enumerate(can): found = -1 if can[i] != ref[i]: found = i break Log.error("Comments do not match reference\n{{sample}}", {"sample": can[MIN([0, found - 100]):found + 100]})
def test_specific_bugs(self): """ USE A MYSQL DATABASE TO FILL AN ES INSTANCE (USE Fake_ES() INSTANCES TO KEEP THIS TEST LOCAL) WITH VERSIONS OF BUGS FROM settings.param.bugs. COMPARE THOSE VERSIONS TO A REFERENCE ES (ALSO CHECKED INTO REPOSITORY) """ # settings.param.allow_private_bugs = True database.make_test_instance(self.settings.bugzilla) with DB(self.settings.bugzilla) as db: candidate = elasticsearch.make_test_instance( "candidate", self.settings.candidate) reference = elasticsearch.open_test_instance( "reference", self.settings.private_bugs_reference) #SETUP RUN PARAMETERS param = Struct() param.end_time = CNV.datetime2milli(get_current_time(db)) param.start_time = 0 param.start_time_str = extract_bugzilla.milli2string(db, 0) param.alias_file = self.settings.param.alias_file param.bug_list = self.settings.param.bugs param.allow_private_bugs = self.settings.param.allow_private_bugs with ThreadedQueue(candidate, size=1000) as output: etl(db, output, param, please_stop=None) #COMPARE ALL BUGS Thread.sleep(2) # MUST SLEEP WHILE ES DOES ITS INDEXING compare_both(candidate, reference, self.settings, self.settings.param.bugs)
def test_public_etl(self): """ ENSURE ETL GENERATES WHAT'S IN THE REFERENCE FILE """ File(self.settings.param.first_run_time).delete() File(self.settings.param.last_run_time).delete() self.settings.param.allow_private_bugs = Null database.make_test_instance(self.settings.bugzilla) es = elasticsearch.make_test_instance("candidate", self.settings.fake.bugs) es_comments = elasticsearch.make_test_instance( "candidate_comments", self.settings.fake.comments) bz_etl.main(self.settings, es, es_comments) ref = elasticsearch.open_test_instance( "reference", self.settings.public_bugs_reference) compare_both(es, ref, self.settings, self.settings.param.bugs) #DIRECT COMPARE THE FILE JSON can = File(self.settings.fake.comments.filename).read() ref = File(self.settings.public_comments_reference.filename).read() if can != ref: found = -1 for i, c in enumerate(can): if can[i] != ref[i]: found = i break Log.error("Comments do not match reference\n{{sample}}", {"sample": can[MIN(0, found - 100):found + 100:]})
def test_private_etl(self): """ ENSURE IDENTIFIABLE INFORMATION DOES NOT EXIST ON ANY BUGS """ File(self.settings.param.first_run_time).delete() File(self.settings.param.last_run_time).delete() self.settings.param.allow_private_bugs = True database.make_test_instance(self.settings.bugzilla) es = elasticsearch.make_test_instance("candidate", self.settings.fake.bugs) es_comments = elasticsearch.make_test_instance( "candidate_comments", self.settings.fake.comments) bz_etl.main(self.settings, es, es_comments) ref = elasticsearch.open_test_instance( "reference", self.settings.private_bugs_reference) compare_both(es, ref, self.settings, self.settings.param.bugs) #DIRECT COMPARE THE FILE JSON can = File(self.settings.fake.comments.filename).read() ref = File(self.settings.private_comments_reference.filename).read() if can != ref: for i, c in enumerate(can): found = -1 if can[i] != ref[i]: found = i break Log.error("Comments do not match reference\n{{sample}}", {"sample": can[MIN([0, found - 100]):found + 100]})