def testTimeFilter(self): self._createDummies() avatar_links.init_links(self._avatar1, client=self._redis, assumeEvents=True) # Stupid values that always result in no results self.assertEqual(avatar_links.get_links(self._avatar1, minDT=99999999, client=self._redis), OrderedDict()) self.assertEqual(avatar_links.get_links(self._avatar1, maxDT=0, client=self._redis), OrderedDict()) self.assertEqual(avatar_links.get_links(self._avatar1, minDT=200000, maxDT=0, client=self._redis), OrderedDict()) # Only a minimum self.assertEqual(avatar_links.get_links(self._avatar1, minDT=150000, client=self._redis), OrderedDict([('4', set(['conference_manager', 'conference_participant'])), ('3', set(['conference_manager']))])) # Only a maximum self.assertEqual(avatar_links.get_links(self._avatar1, maxDT=150000, client=self._redis), OrderedDict([('2', set(['conference_participant'])), ('1', set(['conference_participant'])), ('4', set(['conference_manager', 'conference_participant']))])) # Both minimum and maximum self.assertEqual(avatar_links.get_links(self._avatar1, minDT=100000, maxDT=150000, client=self._redis), OrderedDict([('1', set(['conference_participant'])), ('4', set(['conference_manager', 'conference_participant']))])) # Same value for both self.assertEqual(avatar_links.get_links(self._avatar1, minDT=150000, maxDT=150000, client=self._redis), OrderedDict([('4', set(['conference_manager', 'conference_participant']))]))
def testModifyLinks(self): self._createDummies() avatar_links.init_links(self._avatar1, client=self._redis, assumeEvents=True) self.assertEqual(avatar_links.get_links(self._avatar1, client=self._redis), OrderedDict([('2', set(['conference_participant'])), ('1', set(['conference_participant'])), ('4', set(['conference_manager', 'conference_participant'])), ('3', set(['conference_manager']))])) # Add a new role for an existing event avatar_links.add_link(self._avatar1, self._event1, 'conference_manager', client=self._redis) self.assertEqual(avatar_links.get_links(self._avatar1, client=self._redis), OrderedDict([('2', set(['conference_participant'])), ('1', set(['conference_manager', 'conference_participant'])), ('4', set(['conference_manager', 'conference_participant'])), ('3', set(['conference_manager']))])) # Delete a role from an event with multiple roles avatar_links.del_link(self._avatar1, self._event1, 'conference_participant', client=self._redis) self.assertEqual(avatar_links.get_links(self._avatar1, client=self._redis), OrderedDict([('2', set(['conference_participant'])), ('1', set(['conference_manager'])), ('4', set(['conference_manager', 'conference_participant'])), ('3', set(['conference_manager']))])) # Delete a role from an event with just one role avatar_links.del_link(self._avatar1, self._event1, 'conference_manager', client=self._redis) self.assertEqual(avatar_links.get_links(self._avatar1, client=self._redis), OrderedDict([('2', set(['conference_participant'])), ('4', set(['conference_manager', 'conference_participant'])), ('3', set(['conference_manager']))])) # Add a role for a new event avatar_links.add_link(self._avatar1, self._event1, 'conference_manager', client=self._redis) self.assertEqual(avatar_links.get_links(self._avatar1, client=self._redis), OrderedDict([('2', set(['conference_participant'])), ('1', set(['conference_manager'])), ('4', set(['conference_manager', 'conference_participant'])), ('3', set(['conference_manager']))]))
def testLinkModificationsOnlyAffectCorrectAvatar(self): self._createDummies() avatar_links.init_links(self._avatar1, client=self._redis, assumeEvents=True) avatar_links.init_links(self._avatar2, client=self._redis, assumeEvents=True) # No change may touch other avatars links = avatar_links.get_links(self._avatar2, client=self._redis) avatar_links.add_link(self._avatar1, self._event1, 'foo', client=self._redis) self.assertEqual( links, avatar_links.get_links(self._avatar2, client=self._redis)) avatar_links.del_link(self._avatar1, self._event4, 'conference_manager', client=self._redis) self.assertEqual( links, avatar_links.get_links(self._avatar2, client=self._redis)) avatar_links.del_link(self._avatar1, self._event4, 'conference_participant', client=self._redis) self.assertEqual( links, avatar_links.get_links(self._avatar2, client=self._redis)) avatar_links.delete_avatar(self._avatar1, client=self._redis) self.assertEqual( links, avatar_links.get_links(self._avatar2, client=self._redis))
def testDeleteAvatar(self): self._createDummies() avatar_links.init_links(self._avatar1, client=self._redis, assumeEvents=True) keys = frozenset(self._redis.keys()) links = avatar_links.get_links(self._avatar1, client=self._redis) avatar_links.init_links(self._avatar2, client=self._redis, assumeEvents=True) avatar_links.delete_avatar(self._avatar2, client=self._redis) self.assertEqual(keys, frozenset(self._redis.keys())) self.assertEqual(links, avatar_links.get_links(self._avatar1, client=self._redis))
def testImport(self): self.assertFalse(self._redis.keys()) self._createDummies() avatar_links.init_links(self._avatar1, client=self._redis, assumeEvents=True) self.assertEqual( frozenset(self._redis.keys()), frozenset([ 'avatar-event-links/avatar_events:1', 'avatar-event-links/event_avatars:2', 'avatar-event-links/event_avatars:3', 'avatar-event-links/event_avatars:1', 'avatar-event-links/event_avatars:4', 'avatar-event-links/avatar_event_roles:1:4', 'avatar-event-links/avatar_event_roles:1:3', 'avatar-event-links/avatar_event_roles:1:2', 'avatar-event-links/avatar_event_roles:1:1' ])) avatar_links.init_links(self._avatar2, client=self._redis, assumeEvents=True) self.assertEqual( frozenset(self._redis.keys()), frozenset([ 'avatar-event-links/avatar_events:1', 'avatar-event-links/avatar_events:2', 'avatar-event-links/event_avatars:2', 'avatar-event-links/event_avatars:3', 'avatar-event-links/event_avatars:1', 'avatar-event-links/event_avatars:4', 'avatar-event-links/avatar_event_roles:1:4', 'avatar-event-links/avatar_event_roles:1:3', 'avatar-event-links/avatar_event_roles:1:2', 'avatar-event-links/avatar_event_roles:1:1', 'avatar-event-links/avatar_event_roles:2:4', 'avatar-event-links/avatar_event_roles:2:3', 'avatar-event-links/avatar_event_roles:2:1' ])) self.assertEqual( avatar_links.get_links(self._avatar1, client=self._redis), OrderedDict([('2', set(['conference_participant'])), ('1', set(['conference_participant'])), ('4', set(['conference_manager', 'conference_participant'])), ('3', set(['conference_manager']))])) self.assertEqual( avatar_links.get_links(self._avatar2, client=self._redis), OrderedDict([('1', set(['conference_manager'])), ('4', set(['conference_manager'])), ('3', set(['conference_manager']))]))
def testModifyEvent(self): self._createDummies() avatar_links.init_links(self._avatar1, client=self._redis, assumeEvents=True) avatar_links.init_links(self._avatar2, client=self._redis, assumeEvents=True) self.assertEqual( avatar_links.get_links(self._avatar1, client=self._redis), OrderedDict([('2', set(['conference_participant'])), ('1', set(['conference_participant'])), ('4', set(['conference_manager', 'conference_participant'])), ('3', set(['conference_manager']))])) self.assertEqual( avatar_links.get_links(self._avatar2, client=self._redis), OrderedDict([('1', set(['conference_manager'])), ('4', set(['conference_manager'])), ('3', set(['conference_manager']))])) # Delete whole event avatar_links.delete_event(self._event1, client=self._redis) self.assertEqual( avatar_links.get_links(self._avatar1, client=self._redis), OrderedDict([('2', set(['conference_participant'])), ('4', set(['conference_manager', 'conference_participant'])), ('3', set(['conference_manager']))])) self.assertEqual( avatar_links.get_links(self._avatar2, client=self._redis), OrderedDict([('4', set(['conference_manager'])), ('3', set(['conference_manager']))])) # Modify event start time so the order changes self._event4.start_date = self._event3.start_date + 10 avatar_links.update_event_time(self._event4, client=self._redis) self.assertEqual( avatar_links.get_links(self._avatar1, client=self._redis), OrderedDict([('2', set(['conference_participant'])), ('3', set(['conference_manager'])), ('4', set(['conference_manager', 'conference_participant']))])) self.assertEqual( avatar_links.get_links(self._avatar2, client=self._redis), OrderedDict([('3', set(['conference_manager'])), ('4', set(['conference_manager']))]))
def testMergeAvatars(self): self._createDummies() avatar_links.init_links(self._avatar1, client=self._redis, assumeEvents=True) keys = frozenset(self._redis.keys()) avatar_links.init_links(self._avatar2, client=self._redis, assumeEvents=True) avatar_links.merge_avatars(self._avatar1, self._avatar2, client=self._redis) self.assertEqual(keys, frozenset(self._redis.keys())) self.assertEqual(avatar_links.get_links(self._avatar2, client=self._redis), OrderedDict()) self.assertEqual(avatar_links.get_links(self._avatar1, client=self._redis), OrderedDict([('2', set(['conference_participant'])), ('1', set(['conference_manager', 'conference_participant'])), ('4', set(['conference_manager', 'conference_participant'])), ('3', set(['conference_manager']))]))
def testLinkModificationsOnlyAffectCorrectAvatar(self): self._createDummies() avatar_links.init_links(self._avatar1, client=self._redis, assumeEvents=True) avatar_links.init_links(self._avatar2, client=self._redis, assumeEvents=True) # No change may touch other avatars links = avatar_links.get_links(self._avatar2, client=self._redis) avatar_links.add_link(self._avatar1, self._event1, 'foo', client=self._redis) self.assertEqual(links, avatar_links.get_links(self._avatar2, client=self._redis)) avatar_links.del_link(self._avatar1, self._event4, 'conference_manager', client=self._redis) self.assertEqual(links, avatar_links.get_links(self._avatar2, client=self._redis)) avatar_links.del_link(self._avatar1, self._event4, 'conference_participant', client=self._redis) self.assertEqual(links, avatar_links.get_links(self._avatar2, client=self._redis)) avatar_links.delete_avatar(self._avatar1, client=self._redis) self.assertEqual(links, avatar_links.get_links(self._avatar2, client=self._redis))
def redisLinkedTo(dbi, withRBDB, prevVersion): """Import linkedTo information into Redis""" if not Config.getInstance().getRedisConnectionURL(): print console.colored(" Redis not configured, skipping", 'yellow') return with redis_client.pipeline(transaction=False) as pipe: for i, avatar in enumerate(AvatarHolder()._getIdx().itervalues()): avatar_links.init_links(avatar, client=pipe) if i % 1000 == 0: pipe.execute() dbi.sync() print '\r %d' % i, sys.stdout.flush() pipe.execute() print '\r Done '
def convertLinkedTo(dbi, withRBDB, prevVersion): """Convert Avatar.linkedTo structure to use OOTreeSets and import linkedTo information into Redis (if enabled)""" print 'Note: Some links might point to broken objects which will be skipped automatically.' use_redis = Config.getInstance().getRedisConnectionURL() if use_redis: pipe = redis_client.pipeline(transaction=False) for i, avatar in enumerate(AvatarHolder()._getIdx().itervalues()): avatar.updateLinkedTo( ) # just in case some avatars do not have all fields linkedTo = avatar.linkedTo avatar.resetLinkedTo() # nuke old links for field, data in avatar.linkedToMap.iteritems(): for role in data['roles']: if not linkedTo[field][role]: continue todo = set(linkedTo[field][role]) # We have broken objects in the database which will fail in the getConference() call. If the current # object type has such a method call it on each object and skip it in case it raises an AttributeError if hasattr(linkedTo[field][role][0], 'getConference'): for obj in linkedTo[field][role]: try: obj.getConference() except AttributeError, e: print ' \tSkipping broken object in %s/%s/%s: %r' % ( avatar.getId(), field, role, obj) todo.remove(obj) avatar.linkedTo[field][role].update(todo) if use_redis: avatar_links.init_links(avatar, client=pipe) if i % 1000 == 0: if use_redis: pipe.execute() dbi.commit() print '\r %d' % i, sys.stdout.flush()
def convertLinkedTo(dbi, withRBDB, prevVersion): """Convert Avatar.linkedTo structure to use OOTreeSets and import linkedTo information into Redis (if enabled)""" print 'Note: Some links might point to broken objects which will be skipped automatically.' use_redis = Config.getInstance().getRedisConnectionURL() if use_redis: pipe = redis_client.pipeline(transaction=False) for i, avatar in enumerate(AvatarHolder()._getIdx().itervalues()): avatar.updateLinkedTo() # just in case some avatars do not have all fields linkedTo = avatar.linkedTo avatar.resetLinkedTo() # nuke old links for field, data in avatar.linkedToMap.iteritems(): for role in data['roles']: if not linkedTo[field][role]: continue todo = set(linkedTo[field][role]) # We have broken objects in the database which will fail in the getConference() call. If the current # object type has such a method call it on each object and skip it in case it raises an AttributeError if hasattr(linkedTo[field][role][0], 'getConference'): for obj in linkedTo[field][role]: try: obj.getConference() except AttributeError, e: print ' \tSkipping broken object in %s/%s/%s: %r' % (avatar.getId(), field, role, obj) todo.remove(obj) avatar.linkedTo[field][role].update(todo) if use_redis: avatar_links.init_links(avatar, client=pipe) if i % 1000 == 0: if use_redis: pipe.execute() dbi.commit() print '\r %d' % i, sys.stdout.flush()
def testImport(self): self.assertFalse(self._redis.keys()) self._createDummies() avatar_links.init_links(self._avatar1, client=self._redis, assumeEvents=True) self.assertEqual(frozenset(self._redis.keys()), frozenset(['avatar-event-links/avatar_events:1', 'avatar-event-links/event_avatars:2', 'avatar-event-links/event_avatars:3', 'avatar-event-links/event_avatars:1', 'avatar-event-links/event_avatars:4', 'avatar-event-links/avatar_event_roles:1:4', 'avatar-event-links/avatar_event_roles:1:3', 'avatar-event-links/avatar_event_roles:1:2', 'avatar-event-links/avatar_event_roles:1:1'])) avatar_links.init_links(self._avatar2, client=self._redis, assumeEvents=True) self.assertEqual(frozenset(self._redis.keys()), frozenset(['avatar-event-links/avatar_events:1', 'avatar-event-links/avatar_events:2', 'avatar-event-links/event_avatars:2', 'avatar-event-links/event_avatars:3', 'avatar-event-links/event_avatars:1', 'avatar-event-links/event_avatars:4', 'avatar-event-links/avatar_event_roles:1:4', 'avatar-event-links/avatar_event_roles:1:3', 'avatar-event-links/avatar_event_roles:1:2', 'avatar-event-links/avatar_event_roles:1:1', 'avatar-event-links/avatar_event_roles:2:4', 'avatar-event-links/avatar_event_roles:2:3', 'avatar-event-links/avatar_event_roles:2:1'])) self.assertEqual(avatar_links.get_links(self._avatar1, client=self._redis), OrderedDict([('2', set(['conference_participant'])), ('1', set(['conference_participant'])), ('4', set(['conference_manager', 'conference_participant'])), ('3', set(['conference_manager']))])) self.assertEqual(avatar_links.get_links(self._avatar2, client=self._redis), OrderedDict([('1', set(['conference_manager'])), ('4', set(['conference_manager'])), ('3', set(['conference_manager']))]))
def testModifyEvent(self): self._createDummies() avatar_links.init_links(self._avatar1, client=self._redis, assumeEvents=True) avatar_links.init_links(self._avatar2, client=self._redis, assumeEvents=True) self.assertEqual(avatar_links.get_links(self._avatar1, client=self._redis), OrderedDict([('2', set(['conference_participant'])), ('1', set(['conference_participant'])), ('4', set(['conference_manager', 'conference_participant'])), ('3', set(['conference_manager']))])) self.assertEqual(avatar_links.get_links(self._avatar2, client=self._redis), OrderedDict([('1', set(['conference_manager'])), ('4', set(['conference_manager'])), ('3', set(['conference_manager']))])) # Delete whole event avatar_links.delete_event(self._event1, client=self._redis) self.assertEqual(avatar_links.get_links(self._avatar1, client=self._redis), OrderedDict([('2', set(['conference_participant'])), ('4', set(['conference_manager', 'conference_participant'])), ('3', set(['conference_manager']))])) self.assertEqual(avatar_links.get_links(self._avatar2, client=self._redis), OrderedDict([('4', set(['conference_manager'])), ('3', set(['conference_manager']))])) # Modify event start time so the order changes self._event4.start_date = self._event3.start_date + 10 avatar_links.update_event_time(self._event4, client=self._redis) self.assertEqual(avatar_links.get_links(self._avatar1, client=self._redis), OrderedDict([('2', set(['conference_participant'])), ('3', set(['conference_manager'])), ('4', set(['conference_manager', 'conference_participant']))])) self.assertEqual(avatar_links.get_links(self._avatar2, client=self._redis), OrderedDict([('3', set(['conference_manager'])), ('4', set(['conference_manager']))]))
def _getAnswer(self): avatar_links.delete_avatar(self._user) # clean start avatar_links.init_links(self._user)
def split_avatars(file): """Step 2: Split the merged avatars - run this on the POST-MERGE database""" data = json.load(file) if len(data) != 2: echo('Invalid data: Expected two items, found {}'.format(len(data))) raise click.Abort() avatars = [AvatarHolder().getById(x['id']) for x in data] if avatars.count(None) != 1: echo('Expected exactly one unreferenced avatar, got {}'.format(avatars)) raise click.Abort() if avatars[0] is None: orig = avatars[1] orig_data = data[1] merged_data = data[0] else: orig = avatars[0] orig_data = data[0] merged_data = data[1] merged = [av for av in orig._mergeFrom if av.getId() == merged_data['id']] if len(merged) != 1: echo("Avatars don't seem to be merged into each other") raise click.Abort() merged = merged[0] if merged not in orig._mergeFrom or merged._mergeTo != orig: echo("Avatars don't seem to be merged into each other") raise click.Abort() # Remove merge references orig._mergeFrom.remove(merged) merged._mergeTo = None # Clean secondary emails orig.setSecondaryEmails(orig_data['secondary_emails']) merged.setSecondaryEmails(merged_data['secondary_emails']) # Clean identities for identity in merged.getIdentityList(create_identities=True): orig.removeIdentity(identity) identity.setUser(merged) # Re-index the avatars # noinspection PyCallByClass ObjectHolder.add(AvatarHolder(), merged) for index in {'organisation', 'email', 'name', 'surName', 'status'}: IndexesHolder().getById(index).unindexUser(orig) IndexesHolder().getById(index).indexUser(orig) IndexesHolder().getById(index).indexUser(merged) # Restore links for the merged avatar for key, links in merged_data['links'].iteritems(): objtype, role = key.split('.') for link in links: # TODO: Add extra items here as necessary (use show_links to check if anything is missing)! if objtype == 'conference': event = ConferenceHolder().getById(link, quiet=True) if event is None: echo('Event not found: {}'.format(link)) continue if event not in set(orig.linkedTo[objtype][role]): echo('{!r} not linked to avatar via {}'.format(event, key)) continue if role == 'access': event.revokeAccess(orig) event.grantAccess(merged) else: echo('Unexpected link type: {}'.format(key)) raise click.Abort() # print 'Updated {} - {!r}'.format(key, event) elif objtype == 'contribution': event_id, contrib_id = link event = ConferenceHolder().getById(event_id) if event is None: echo('Event not found: {}'.format(event_id)) continue contrib = event.getContributionById(contrib_id) if contrib is None: echo('Contrib not found: {}.{}'.format(event_id, contrib_id)) continue if contrib not in set(orig.linkedTo[objtype][role]): echo('{} not linked to avatar via {}'.format(contrib, key)) continue if role == 'submission': contrib.revokeSubmission(orig) contrib.grantSubmission(merged) elif role == 'manager': contrib.revokeModification(orig) contrib.grantModification(merged) else: echo('Unexpected link type: {}'.format(key)) raise click.Abort() # print 'Updated {} - {}'.format(key, contrib) else: # This results in inconsistent data in redis, but since we flush the redis # links after a successful merge, that's not an issue. Also, we only reach # this branch if someone is using different versions of this script... echo('Unexpected link type: {}'.format(key)) raise click.Abort() if not click.confirm('Do you want to commit the changes now?'): transaction.abort() # not really necessary, but let's stay on the safe side raise click.Abort() transaction.commit() # Refresh redis links avatar_links.delete_avatar(orig) avatar_links.delete_avatar(merged) avatar_links.init_links(orig) avatar_links.init_links(merged) # Delete suggestions suggestions.delete_avatar(orig) suggestions.delete_avatar(merged)
def split_avatars(file): """Step 2: Split the merged avatars - run this on the POST-MERGE database""" data = json.load(file) if len(data) != 2: echo('Invalid data: Expected two items, found {}'.format(len(data))) raise click.Abort() avatars = [AvatarHolder().getById(x['id']) for x in data] if avatars.count(None) != 1: echo( 'Expected exactly one unreferenced avatar, got {}'.format(avatars)) raise click.Abort() if avatars[0] is None: orig = avatars[1] orig_data = data[1] merged_data = data[0] else: orig = avatars[0] orig_data = data[0] merged_data = data[1] merged = [av for av in orig._mergeFrom if av.getId() == merged_data['id']] if len(merged) != 1: echo("Avatars don't seem to be merged into each other") raise click.Abort() merged = merged[0] if merged not in orig._mergeFrom or merged._mergeTo != orig: echo("Avatars don't seem to be merged into each other") raise click.Abort() # Remove merge references orig._mergeFrom.remove(merged) merged._mergeTo = None # Clean secondary emails orig.setSecondaryEmails(orig_data['secondary_emails']) merged.setSecondaryEmails(merged_data['secondary_emails']) # Clean identities for identity in merged.getIdentityList(create_identities=True): orig.removeIdentity(identity) identity.setUser(merged) # Re-index the avatars # noinspection PyCallByClass ObjectHolder.add(AvatarHolder(), merged) for index in {'organisation', 'email', 'name', 'surName', 'status'}: IndexesHolder().getById(index).unindexUser(orig) IndexesHolder().getById(index).indexUser(orig) IndexesHolder().getById(index).indexUser(merged) # Restore links for the merged avatar for key, links in merged_data['links'].iteritems(): objtype, role = key.split('.') for link in links: # TODO: Add extra items here as necessary (use show_links to check if anything is missing)! if objtype == 'conference': event = ConferenceHolder().getById(link, quiet=True) if event is None: echo('Event not found: {}'.format(link)) continue if event not in set(orig.linkedTo[objtype][role]): echo('{!r} not linked to avatar via {}'.format(event, key)) continue if role == 'access': event.revokeAccess(orig) event.grantAccess(merged) else: echo('Unexpected link type: {}'.format(key)) raise click.Abort() # print 'Updated {} - {!r}'.format(key, event) elif objtype == 'contribution': event_id, contrib_id = link event = ConferenceHolder().getById(event_id) if event is None: echo('Event not found: {}'.format(event_id)) continue contrib = event.getContributionById(contrib_id) if contrib is None: echo('Contrib not found: {}.{}'.format( event_id, contrib_id)) continue if contrib not in set(orig.linkedTo[objtype][role]): echo('{} not linked to avatar via {}'.format(contrib, key)) continue if role == 'submission': contrib.revokeSubmission(orig) contrib.grantSubmission(merged) elif role == 'manager': contrib.revokeModification(orig) contrib.grantModification(merged) else: echo('Unexpected link type: {}'.format(key)) raise click.Abort() # print 'Updated {} - {}'.format(key, contrib) else: # This results in inconsistent data in redis, but since we flush the redis # links after a successful merge, that's not an issue. Also, we only reach # this branch if someone is using different versions of this script... echo('Unexpected link type: {}'.format(key)) raise click.Abort() if not click.confirm('Do you want to commit the changes now?'): transaction.abort( ) # not really necessary, but let's stay on the safe side raise click.Abort() transaction.commit() # Refresh redis links avatar_links.delete_avatar(orig) avatar_links.delete_avatar(merged) avatar_links.init_links(orig) avatar_links.init_links(merged) # Delete suggestions suggestions.delete_avatar(orig) suggestions.delete_avatar(merged)