def setUp(self): DatabaseTest.setUp(self) self.common_cohort_1() # ******* EXPERIMENT HERE # Before the fix in schedules/daily.py to increase max recursion: # NO Max Recursion error thrown: # total_runs = 115 and parallelism = 3 # does NOT cause maximum recursion depth exceeded on Dan's machine # # Max Recursion error is thrown: # total_runs = 200 and parallelism = 2 # causes maximum recursion depth exceeded on Dan's machine # # After the fix: # total_runs = 200 and parallelism = 2 # no longer causes maximum recursion # self.total_runs = 200 self.sleep = 10 # crank up the queue parallel report configuration self.save_instances = queue.conf['MAX_INSTANCES_PER_RECURRENT_REPORT'] queue.conf['MAX_INSTANCES_PER_RECURRENT_REPORT'] = self.total_runs
def setUp(self): DatabaseTest.setUp(self) # registration for all the editors below self.r = r = 20140101000000 # exactly 30 days after registration self.m = m = 20140131000000 self.r_plus_30 = s(d(self.m)) self.create_test_cohort( editor_count=5, revisions_per_editor=8, revision_timestamps=[ # this one will make 5 edits within 30 days of self.r_plus_30 [r + 1, r + 2, r + 3, r + 4, r + 5, m + 6, m + 7, m + 8], # this one will make 3 edits within 30 days of self.r_plus_30 [r + 1, r + 2, r + 3, m + 4, m + 5, m + 6, m + 7, m + 8], # this one will make 8 edits within 30 days of self.r_plus_30 [r + 1, r + 2, r + 3, r + 4, r + 5, r + 6, r + 7, r + 8], # this one will make 0 edits within 30 days of self.r_plus_30 [m + 1, m + 2, m + 3, m + 4, m + 5, m + 6, m + 7, m + 8], # this one will make the 5th edit right on self.r_plus_30 [r + 1, r + 2, r + 3, r + 4, m + 0, m + 6, m + 7, m + 8], ], user_registrations=r, revision_lengths=10 ) self.create_non_editors([ (r, 'newusers', 'create'), (r, 'newusers', 'create'), (m, 'newusers', 'create'), ])
def setUp(self): one_hour = timedelta(hours=1) reg = datetime.now() - one_hour * 48 DatabaseTest.setUp(self) self.create_test_cohort( editor_count=3, revisions_per_editor=2, user_registrations=i(reg), revision_timestamps=[ [i(reg + one_hour), i(reg + one_hour * 25)], [i(reg + one_hour * 20), i(reg + one_hour * 40)], [i(reg + one_hour * 30), i(reg + one_hour * 73)], ], revision_lengths=10) self.helper_insert_editors( editor_count=3, revisions_per_editor=2, user_registrations=i(reg), revision_timestamps=[ [i(reg + one_hour), i(reg + one_hour * 25)], [i(reg + one_hour * 20), i(reg + one_hour * 40)], [i(reg + one_hour * 30), i(reg + one_hour * 73)], ], revision_lengths=10) self.e1 = self.editors[0].user_id self.e2 = self.editors[1].user_id self.e3 = self.editors[2].user_id
def setUp(self): one_hour = timedelta(hours=1) reg = datetime.now() - one_hour * 48 DatabaseTest.setUp(self) self.create_test_cohort( editor_count=2, revisions_per_editor=2, user_registrations=i(reg), revision_timestamps=[ [i(reg + one_hour) , i(reg + one_hour * 25)], [i(reg + one_hour * 30), i(reg + one_hour * 40)], ], revision_lengths=10 ) self.helper_insert_editors( editor_count=2, revisions_per_editor=2, user_registrations=i(reg), revision_timestamps=[ [i(reg + one_hour) , i(reg + one_hour * 25)], [i(reg + one_hour * 30), i(reg + one_hour * 40)], ], revision_lengths=10 ) self.e1 = self.editors[0].user_id self.e2 = self.editors[1].user_id
def setUp(self): DatabaseTest.setUp(self) one_hour = timedelta(hours=1) first_reg = datetime.now() - one_hour * 100 self.create_test_cohort( editor_count=3, revisions_per_editor=2, revision_timestamps=i(first_reg + one_hour * 100), user_registrations=[ i(first_reg), i(first_reg + one_hour * 20), i(first_reg + one_hour * 30), ], revision_lengths=10 ) self.create_non_editors([ (first_reg + one_hour , 'proxy' , 'proxy'), (first_reg + one_hour , 'proxy' , 'attached'), (first_reg + one_hour , 'proxy' , 'create'), (first_reg + one_hour , 'newusers', 'proxy'), (first_reg + one_hour , 'newusers', 'attached'), (first_reg + one_hour , 'newusers', 'create'), (first_reg - one_hour * 1000, 'newusers', 'proxy'), (first_reg - one_hour * 1000, 'proxy' , 'proxy'), (first_reg - one_hour * 1000, 'proxy' , 'create'), (first_reg - one_hour * 1000, 'newusers', 'create'), ]) # these are useful to set up the tests self.first_reg = first_reg self.one_hour = one_hour
def setUp(self): DatabaseTest.setUp(self) # registration for all the editors below self.r = r = 20140101000000 # exactly 30 days after registration self.m = m = 20140131000000 self.r_plus_30 = s(d(self.m)) self.create_test_cohort( editor_count=5, revisions_per_editor=8, revision_timestamps=[ # this one will make 5 edits within 30 days of self.r_plus_30 [r + 1, r + 2, r + 3, r + 4, r + 5, m + 6, m + 7, m + 8], # this one will make 3 edits within 30 days of self.r_plus_30 [r + 1, r + 2, r + 3, m + 4, m + 5, m + 6, m + 7, m + 8], # this one will make 8 edits within 30 days of self.r_plus_30 [r + 1, r + 2, r + 3, r + 4, r + 5, r + 6, r + 7, r + 8], # this one will make 0 edits within 30 days of self.r_plus_30 [m + 1, m + 2, m + 3, m + 4, m + 5, m + 6, m + 7, m + 8], # this one will make the 5th edit right on self.r_plus_30 [r + 1, r + 2, r + 3, r + 4, m + 0, m + 6, m + 7, m + 8], ], user_registrations=r, revision_lengths=10) self.create_non_editors([ (r, 'newusers', 'create'), (r, 'newusers', 'create'), (m, 'newusers', 'create'), ])
def setUp(self): DatabaseTest.setUp(self) # registration for all the editors below self.before_r = before_r = 20131201000000 self.r = r = 20140101000000 # exactly 30 days after registration self.m = m = 20140131000000 # exactly 60 days after registration self.m2 = m2 = 20140302000000 self.r_plus_60 = s(d(self.m2)) self.editor_count = 14 self.create_test_cohort( # 5 editors will have registered on time, 5 will not editor_count=self.editor_count, revisions_per_editor=10, revision_timestamps=[ # NOTE: these first 6 are registered BEFORE r # this one will make 7 edits within 30 days of m, 3 after [r] * 7 + [m + 1] * 3, # this one will make 3 edits within 30 days of m, 7 after [r] * 3 + [m + 1] * 7, # this one will make 10 edits within 30 days of m, 0 after [r] * 10, # this one will make 0 edits within 30 days of m, 10 after [m + 1] * 10, # this one will make 5 edits within 30 days of m, 5 after [r] * 5 + [m + 1] * 5, # this one will make the 5th edit right on m, 5 after [r] * 4 + [m] + [m + 1] * 5, # this one will make no edits within r -> r + 60 days [m2 + 1] * 10, # NOTE: these next 6 are registered AFTER r # this one will make 7 edits within 30 days of m, 3 after [r] * 7 + [m + 1] * 3, # this one will make 3 edits within 30 days of m, 7 after [r] * 3 + [m + 1] * 7, # this one will make 10 edits within 30 days of m, 0 after [r] * 10, # this one will make 0 edits within 30 days of m, 10 after [m + 1] * 10, # this one will make 5 edits within 30 days of m, 5 after [r] * 5 + [m + 1] * 5, # this one will make the 5th edit right on m, 5 after [r] * 4 + [m] + [m + 1] * 5, # this one will make no edits within r -> r + 60 days [m2 + 1] * 10, ], user_registrations=([before_r] * 7) + ([r] * 7), revision_lengths=10 ) self.create_non_editors([ (r, 'newusers', 'create'), (r, 'newusers', 'create'), (m, 'newusers', 'create'), ])
def setUp(self): DatabaseTest.setUp(self) # registration for all the editors below self.before_r = before_r = 20131201000000 self.r = r = 20140101000000 # exactly 30 days after registration self.m = m = 20140131000000 # exactly 60 days after registration self.m2 = m2 = 20140302000000 self.r_plus_60 = s(d(self.m2)) self.editor_count = 14 self.create_test_cohort( # 5 editors will have registered on time, 5 will not editor_count=self.editor_count, revisions_per_editor=10, revision_timestamps=[ # NOTE: these first 6 are registered BEFORE r # this one will make 7 edits within 30 days of m, 3 after [r] * 7 + [m + 1] * 3, # this one will make 3 edits within 30 days of m, 7 after [r] * 3 + [m + 1] * 7, # this one will make 10 edits within 30 days of m, 0 after [r] * 10, # this one will make 0 edits within 30 days of m, 10 after [m + 1] * 10, # this one will make 5 edits within 30 days of m, 5 after [r] * 5 + [m + 1] * 5, # this one will make the 5th edit right on m, 5 after [r] * 4 + [m] + [m + 1] * 5, # this one will make no edits within r -> r + 60 days [m2 + 1] * 10, # NOTE: these next 6 are registered AFTER r # this one will make 7 edits within 30 days of m, 3 after [r] * 7 + [m + 1] * 3, # this one will make 3 edits within 30 days of m, 7 after [r] * 3 + [m + 1] * 7, # this one will make 10 edits within 30 days of m, 0 after [r] * 10, # this one will make 0 edits within 30 days of m, 10 after [m + 1] * 10, # this one will make 5 edits within 30 days of m, 5 after [r] * 5 + [m + 1] * 5, # this one will make the 5th edit right on m, 5 after [r] * 4 + [m] + [m + 1] * 5, # this one will make no edits within r -> r + 60 days [m2 + 1] * 10, ], user_registrations=([before_r] * 7) + ([r] * 7), revision_lengths=10) self.create_non_editors([ (r, 'newusers', 'create'), (r, 'newusers', 'create'), (m, 'newusers', 'create'), ])
def setUp(self): DatabaseTest.setUp(self) self.create_test_cohort(editor_count=1, revisions_per_editor=5, revision_timestamps=[[ 20130101000000, 20130101010000, 20130101010100, 20130101020100, 20130101020101, ]], revision_lengths=10)
def setUp(self): DatabaseTest.setUp(self) self.create_test_cohort( editor_count=1, revisions_per_editor=5, revision_timestamps=[[ 20130101000000, 20130101010000, 20130101010100, 20130101020100, 20130101020101, ]], revision_lengths=10 )
def setUp(self): DatabaseTest.setUp(self) self.create_test_cohort( editor_count=2, revisions_per_editor=3, revision_timestamps=[ [20121231230000, 20130101003000, 20130101010000], [20130101120000, 20130102000000, 20130102120000], ], revision_lengths=[ [1, 2, 3], # User A makes some edits. [2, 4, 5], # User B reverts user A's edit #3 back to edit #2. ], )
def setUp(self): DatabaseTest.setUp(self) # registration for all the editors below self.before_r = before_r = 20131201000000 self.r = r = 20140101000000 # exactly 30 days after registration self.m = m = 20140131000000 self.r_plus_30 = s(d(self.m)) self.editor_count = 10 self.create_test_cohort( # 5 editors will have registered on time, 5 will not editor_count=self.editor_count, revisions_per_editor=8, revision_timestamps=[ # NOTE: these first 5 are registered BEFORE r # this one will make 5 edits within 30 days of self.r_plus_30 [r + 1, r + 2, r + 3, r + 4, r + 5, m + 6, m + 7, m + 8], # this one will make 3 edits within 30 days of self.r_plus_30 [r + 1, r + 2, r + 3, m + 4, m + 5, m + 6, m + 7, m + 8], # this one will make 8 edits within 30 days of self.r_plus_30 [r + 1, r + 2, r + 3, r + 4, r + 5, r + 6, r + 7, r + 8], # this one will make 0 edits within 30 days of self.r_plus_30 [m + 1, m + 2, m + 3, m + 4, m + 5, m + 6, m + 7, m + 8], # this one will make the 5th edit right on self.r_plus_30 [r + 1, r + 2, r + 3, r + 4, m + 0, m + 6, m + 7, m + 8], # NOTE: these next 5 are registered AFTER r # this one will make 5 edits within 30 days of self.r_plus_30 [r + 1, r + 2, r + 3, r + 4, r + 5, m + 6, m + 7, m + 8], # this one will make 3 edits within 30 days of self.r_plus_30 [r + 1, r + 2, r + 3, m + 4, m + 5, m + 6, m + 7, m + 8], # this one will make 8 edits within 30 days of self.r_plus_30 [r + 1, r + 2, r + 3, r + 4, r + 5, r + 6, r + 7, r + 8], # this one will make 0 edits within 30 days of self.r_plus_30 [m + 1, m + 2, m + 3, m + 4, m + 5, m + 6, m + 7, m + 8], # this one will make the 5th edit right on self.r_plus_30 [r + 1, r + 2, r + 3, r + 4, m + 0, m + 6, m + 7, m + 8], ], user_registrations=([before_r] * 5) + ([r] * 5), revision_lengths=10 ) self.create_non_editors([ (r, 'newusers', 'create'), (r, 'newusers', 'create'), (m, 'newusers', 'create'), ])
def setUp(self): DatabaseTest.setUp(self) # registration for all the editors below self.before_r = before_r = 20131201000000 self.r = r = 20140101000000 # exactly 30 days after registration self.m = m = 20140131000000 self.r_plus_30 = s(d(self.m)) self.editor_count = 10 self.create_test_cohort( # 5 editors will have registered on time, 5 will not editor_count=self.editor_count, revisions_per_editor=8, revision_timestamps=[ # NOTE: these first 5 are registered BEFORE r # this one will make 5 edits within 30 days of self.r_plus_30 [r + 1, r + 2, r + 3, r + 4, r + 5, m + 6, m + 7, m + 8], # this one will make 3 edits within 30 days of self.r_plus_30 [r + 1, r + 2, r + 3, m + 4, m + 5, m + 6, m + 7, m + 8], # this one will make 8 edits within 30 days of self.r_plus_30 [r + 1, r + 2, r + 3, r + 4, r + 5, r + 6, r + 7, r + 8], # this one will make 0 edits within 30 days of self.r_plus_30 [m + 1, m + 2, m + 3, m + 4, m + 5, m + 6, m + 7, m + 8], # this one will make the 5th edit right on self.r_plus_30 [r + 1, r + 2, r + 3, r + 4, m + 0, m + 6, m + 7, m + 8], # NOTE: these next 5 are registered AFTER r # this one will make 5 edits within 30 days of self.r_plus_30 [r + 1, r + 2, r + 3, r + 4, r + 5, m + 6, m + 7, m + 8], # this one will make 3 edits within 30 days of self.r_plus_30 [r + 1, r + 2, r + 3, m + 4, m + 5, m + 6, m + 7, m + 8], # this one will make 8 edits within 30 days of self.r_plus_30 [r + 1, r + 2, r + 3, r + 4, r + 5, r + 6, r + 7, r + 8], # this one will make 0 edits within 30 days of self.r_plus_30 [m + 1, m + 2, m + 3, m + 4, m + 5, m + 6, m + 7, m + 8], # this one will make the 5th edit right on self.r_plus_30 [r + 1, r + 2, r + 3, r + 4, m + 0, m + 6, m + 7, m + 8], ], user_registrations=([before_r] * 5) + ([r] * 5), revision_lengths=10) self.create_non_editors([ (r, 'newusers', 'create'), (r, 'newusers', 'create'), (m, 'newusers', 'create'), ])
def setUp(self): self.editor_count = 100000 self.revision_count = 2 one_hour = timedelta(hours=1) start = datetime.now() - one_hour * self.editor_count * 2 timestamps = [[ i(start + one_hour * (rev + edi)) for rev in range(self.revision_count) ] for edi in range(self.editor_count)] DatabaseTest.setUp(self) self.create_test_cohort( editor_count=self.editor_count, revisions_per_editor=self.revision_count, user_registrations=[i(d(t[0]) - one_hour) for t in timestamps], revision_timestamps=timestamps, revision_lengths=10, )
def setUp(self): """ Setup editing data using a regular cohort setup Setup a WikiCohort that will span to all users that we have in the project. """ DatabaseTest.setUp(self) self.common_cohort_4() self.create_wiki_cohort() self.cohort_service = CohortService() self.wiki_cohort = self.cohort_service.get( self.session, self.owner_user_id, by_id=self.basic_wiki_cohort.id ) # This should return an empty list, but if it returned anything, the objects # would be WikiUserStore objects, so we should treat it as such self.user_ids = [ w.mediawiki_userid for w in self.cohort_service.get_wikiusers(self.wiki_cohort) ]
def setUp(self): self.editor_count = 100000 self.revision_count = 2 one_hour = timedelta(hours=1) start = datetime.now() - one_hour * self.editor_count * 2 timestamps = [ [i(start + one_hour * (rev + edi)) for rev in range(self.revision_count)] for edi in range(self.editor_count) ] DatabaseTest.setUp(self) self.create_test_cohort( editor_count=self.editor_count, revisions_per_editor=self.revision_count, user_registrations=[i(d(t[0]) - one_hour) for t in timestamps], revision_timestamps=timestamps, revision_lengths=10, )
def setUp(self): """ NOTE: self.cohort is a CohortStore object. When testing CohortService, one should use logical cohort objects (ie. FixedCohort, WikiCohort, etc.) """ DatabaseTest.setUp(self) self.common_cohort_1() self.editor_user_ids = [e.user_id for e in self.editors] self.cohort_service = CohortService() empty_cohort = CohortStore(enabled=True, class_name='Cohort') self.session.add(empty_cohort) self.session.commit() empty_cohort_user = CohortUserStore( user_id=self.owner_user_id, cohort_id=empty_cohort.id, role=CohortUserRole.OWNER, ) self.session.add(empty_cohort_user) self.session.commit() self.invalid_empty_cohort = CohortStore( enabled=True, class_name='FixedCohort', validated=True ) self.session.add(self.invalid_empty_cohort) self.session.commit() invalid_empty_cohort_user = CohortUserStore( user_id=self.owner_user_id, cohort_id=self.invalid_empty_cohort.id, role=CohortUserRole.OWNER, ) self.session.add(invalid_empty_cohort_user) self.session.commit() self.empty_cohort = self.cohort_service.get( self.session, self.owner_user_id, by_id=empty_cohort.id ) self.fixed_cohort = self.cohort_service.get( self.session, self.owner_user_id, by_id=self.cohort.id )
def setUp(self): """ NOTE: self.cohort is a CohortStore object. When testing CohortService, one should use logical cohort objects (ie. FixedCohort, WikiCohort, etc.) """ DatabaseTest.setUp(self) self.common_cohort_1() self.editor_user_ids = [e.user_id for e in self.editors] self.cohort_service = CohortService() empty_cohort = CohortStore(enabled=True, class_name='Cohort') self.session.add(empty_cohort) self.session.commit() empty_cohort_user = CohortUserStore( user_id=self.owner_user_id, cohort_id=empty_cohort.id, role=CohortUserRole.OWNER, ) self.session.add(empty_cohort_user) self.session.commit() self.invalid_empty_cohort = CohortStore(enabled=True, class_name='FixedCohort', validated=True) self.session.add(self.invalid_empty_cohort) self.session.commit() invalid_empty_cohort_user = CohortUserStore( user_id=self.owner_user_id, cohort_id=self.invalid_empty_cohort.id, role=CohortUserRole.OWNER, ) self.session.add(invalid_empty_cohort_user) self.session.commit() self.empty_cohort = self.cohort_service.get(self.session, self.owner_user_id, by_id=empty_cohort.id) self.fixed_cohort = self.cohort_service.get(self.session, self.owner_user_id, by_id=self.cohort.id)
def setUp(self): DatabaseTest.setUp(self) # turn off the scheduler for this test self.save_schedule = queue.conf['CELERYBEAT_SCHEDULE'] queue.conf['CELERYBEAT_SCHEDULE'] = {} self.common_cohort_1() uid = self.owner_user_id self.today = strip_time(datetime.today()) ago_25 = self.today - timedelta(days=25) ago_35 = self.today - timedelta(days=35) ago_05 = self.today - timedelta(days=5) p = { 'metric': { 'start_date': ago_05, 'end_date': self.today, 'name': 'NamespaceEdits', }, 'recurrent': True, 'cohort': {'id': self.cohort.id, 'name': self.cohort.name}, 'name': 'test-recurrent-reports', } ps = stringify(p) self.reports = [ PersistentReport(recurrent=True, created=ago_25, parameters=ps, user_id=uid), PersistentReport(recurrent=True, created=ago_35, parameters=ps, user_id=uid), PersistentReport(recurrent=True, created=ago_05, parameters=ps, user_id=uid), ] self.session.add_all(self.reports) self.session.commit() self.report_runs = [] for d in range(0, 35): day = self.today - timedelta(days=d) p['metric']['start_date'] = day - timedelta(days=1) p['metric']['end_date'] = day p['recurrent'] = False ps = stringify(p) if d not in [1, 2, 11] and d < 26: self.report_runs.append(PersistentReport( recurrent_parent_id=self.reports[0].id, created=day, status='SUCCESS', parameters=ps, user_id=uid, )) if d not in [1, 2, 11, 31, 33]: self.report_runs.append(PersistentReport( recurrent_parent_id=self.reports[1].id, created=day, status='SUCCESS', parameters=ps, user_id=uid, )) if d not in [1, 2] and d < 6: self.report_runs.append(PersistentReport( recurrent_parent_id=self.reports[2].id, created=day, status='SUCCESS', parameters=ps, user_id=uid, )) self.session.add_all(self.report_runs) self.session.commit()
def setUp(self): DatabaseTest.setUp(self) self.common_cohort_1()
def setUp(self): DatabaseTest.setUp(self) # turn off the scheduler for this test self.save_schedule = queue.conf['CELERYBEAT_SCHEDULE'] queue.conf['CELERYBEAT_SCHEDULE'] = {} self.common_cohort_1() self.uid = self.owner_user_id self.today = strip_time(datetime.today()) # NOTE: running with 20 just makes the tests run faster, but any value > 11 works self.no_more_than = 20 self.missed_by_index = { 0: [1, 2, 11], 1: [1, 2, 11, self.no_more_than + 1, self.no_more_than + 3], 2: [1, 2], 3: range(0, self.no_more_than + 10), } self.ago_by_index = { 0: 26, 1: self.no_more_than + 10, 2: 6, 3: self.no_more_than + 10 } age = { i: self.today - timedelta(days=v - 1) for i, v in self.ago_by_index.items() } self.p = { 'metric': { 'start_date': age[0], 'end_date': self.today, 'name': 'NamespaceEdits', }, 'recurrent': True, 'cohort': { 'id': self.cohort.id, 'name': self.cohort.name }, 'name': 'test-recurrent-reports', } ps = stringify(self.p) self.reports = [ ReportStore(recurrent=True, created=age[0], parameters=ps, user_id=self.uid), ReportStore(recurrent=True, created=age[1], parameters=ps, user_id=self.uid), ReportStore(recurrent=True, created=age[2], parameters=ps, user_id=self.uid), ReportStore(recurrent=True, created=age[3], parameters=ps, user_id=self.uid), ] self.session.add_all(self.reports) self.session.commit()
def setUp(self): DatabaseTest.setUp(self) self.expand = CentralAuthService().expand_via_centralauth
def tearDown(self): queue.conf['MAX_INSTANCES_PER_RECURRENT_REPORT'] = self.save_instances DatabaseTest.tearDown(self)
def setUp(self): DatabaseTest.setUp(self) self.test_report_path = os.path.join(get_absolute_path(), os.pardir, 'tests')
def setUp(self): DatabaseTest.setUp(self) self.tag_service = TagService()