Exemple #1
0
    def __init__(self,
                 main_workspace="~/.repo",
                 repo_kind='hg',
                 main_source=None):
        self.dvcs = DepotOperations.get_depot_operations(repo_kind)
        try:
            self.main_work_path = os.path.expanduser(main_workspace)
            logger.debug('Main workspace: %s' % self.main_work_path)
            self.main_cache_path = os.path.join(self.main_work_path,
                                                DepotManager.cache_name)
            self.squadron_roster_path = os.path.join(
                self.main_work_path, DepotManager.squadron_roster_name)

            # Create the environment.
            if not os.path.isdir(self.main_work_path):
                os.makedirs(self.main_work_path)

            # Create main cache.
            if not self.dvcs.is_a_depot(self.main_cache_path):
                self.main_cache = self.dvcs.init_depot(self.main_cache_path,
                                                       source=main_source)
            else:
                self.main_cache = Depot(self.main_cache_path, None, self.dvcs)

            self.roster = Roster(self.squadron_roster_path)

        except Exception as e:
            raise CloneProvisionError(e)
 def test_add_limit(self):
     # tests the limit imposed to the creation of clones
     roster = Roster(':memory:', max_clones=1)
     roster.add('/test1', 1, 'test')
     self.assertIn('/test1', roster)
     self.assertRaises(MaxClonesLimitReached, roster.add, '/test2', 1,
                       'test')
 def test_add_limit(self):
     # tests the limit imposed to the creation of clones
     roster = Roster(':memory:', max_clones=1)
     roster.add('/test1', 1, 'test')
     self.assertIn('/test1', roster)
     self.assertRaises(
         MaxClonesLimitReached, roster.add, '/test2', 1, 'test')
    def test_multiple_rosters_persistence(self):
        fd, database_path = tempfile.mkstemp()
        try:
            roster1 = Roster(database_path)
            roster2 = Roster(database_path)
            r1 = clone_mother(task='1', status=Clone.INUSE)
            r2 = clone_mother(task='2')

            roster1['/test1'] = r1

            def assign(roster, x, y):
                roster[x] = y
            self.assertRaises(RosterError, assign, roster2, '/test1', r2)

            roster1['/test2'] = r2
            roster2.reserve_clone('2', 'test2')
            self.assertRaises(RosterError, roster1.reserve_clone, '1', 'test1')
        finally:
            os.remove(database_path)
    def test_multiple_rosters_persistence(self):
        fd, database_path = tempfile.mkstemp()
        try:
            roster1 = Roster(database_path)
            roster2 = Roster(database_path)
            r1 = clone_mother(task='1', status=Clone.INUSE)
            r2 = clone_mother(task='2')

            roster1['/test1'] = r1

            def assign(roster, x, y):
                roster[x] = y

            self.assertRaises(RosterError, assign, roster2, '/test1', r2)

            roster1['/test2'] = r2
            roster2.reserve_clone('2', 'test2')
            self.assertRaises(RosterError, roster1.reserve_clone, '1', 'test1')
        finally:
            os.remove(database_path)
    def test_free_clone_by_timeout(self):
        timeout = timedelta(seconds=1)
        initial_time = 0.0

        roster = Roster(':memory:', clone_timeout=timeout)
        roster._get_time_ = lambda: initial_time

        r1 = clone_mother(status=Clone.INUSE, task='1')
        r2 = clone_mother(status=Clone.FREE, task='2')
        r3 = clone_mother(status=Clone.INUSE, task='2')
        roster['/test1'] = r1
        roster['/test2'] = r2
        roster['/test3'] = r3

        self.assertListEqual([], roster._get_old_clones_())

        roster._get_time_ = lambda: initial_time + timeout.seconds + 1
        self.assertGreater(len(roster._get_old_clones_()), 0)

        roster._clean_old_clones()
        self.assertListEqual([], roster._get_old_clones_())
    def test_free_clone_by_timeout(self):
        timeout = timedelta(seconds=1)
        initial_time = 0.0

        roster = Roster(':memory:', clone_timeout=timeout)
        roster._get_time_ = lambda: initial_time

        r1 = clone_mother(status=Clone.INUSE, task='1')
        r2 = clone_mother(status=Clone.FREE, task='2')
        r3 = clone_mother(status=Clone.INUSE, task='2')
        roster['/test1'] = r1
        roster['/test2'] = r2
        roster['/test3'] = r3

        self.assertListEqual([], roster._get_old_clones_())

        roster._get_time_ = lambda: initial_time + timeout.seconds + 1
        self.assertGreater(len(roster._get_old_clones_()), 0)

        roster._clean_old_clones()
        self.assertListEqual([], roster._get_old_clones_())
Exemple #8
0
class DepotManager(object):
    """
    Acts as an public facing API for working with managed clones.

    :param main_workspace: directory where all the workspaces will be
        created.
    :type main_workspace: string
    :param repo_kind: Repository type
    :type repo_kind: string
    :param main_source: FIXME
    :type main_source: string
    """

    # Name of the main repo cache.
    cache_name = 'main_cache'

    # Name of the file storing the roster.
    squadron_roster_name = 'squadron_roster.db'

    # Prefix for the clones used by the workers.
    workspaces_prefix = 'workspace'

    def __init__(self,
                 main_workspace="~/.repo",
                 repo_kind='hg',
                 main_source=None):
        self.dvcs = DepotOperations.get_depot_operations(repo_kind)
        try:
            self.main_work_path = os.path.expanduser(main_workspace)
            logger.debug('Main workspace: %s' % self.main_work_path)
            self.main_cache_path = os.path.join(self.main_work_path,
                                                DepotManager.cache_name)
            self.squadron_roster_path = os.path.join(
                self.main_work_path, DepotManager.squadron_roster_name)

            # Create the environment.
            if not os.path.isdir(self.main_work_path):
                os.makedirs(self.main_work_path)

            # Create main cache.
            if not self.dvcs.is_a_depot(self.main_cache_path):
                self.main_cache = self.dvcs.init_depot(self.main_cache_path,
                                                       source=main_source)
            else:
                self.main_cache = Depot(self.main_cache_path, None, self.dvcs)

            self.roster = Roster(self.squadron_roster_path)

        except Exception as e:
            raise CloneProvisionError(e)

    def _provision_new_clone(self):
        try:
            # Create a new safe directory for the clone.
            clone_directory = tempfile.mkdtemp(
                prefix=DepotManager.workspaces_prefix, dir=self.main_work_path)

            # Create repo (Using the cache)
            result = self.dvcs.init_depot(clone_directory,
                                          parent=self.main_cache)

        except Exception:
            logger.exception("Error provisioning new clone")
            raise CloneProvisionError("Error provisioning new clone")
        return result

    def give_me_depot(self,
                      task_guid,
                      task_name,
                      requirements=None,
                      default_source=None):
        """
        Reserves or prepares a new repository workspace.

        :param task_guid: Identifier of the task reserving the clone.
        :param task_name: Name of the task for information purposes
        :param requirements: requirements to pull
        :param default_source: default clone source
        :returns: a free repo.
        :rtype: :py:class:`~repoman.depot.Depot`
        :raises RepoProvisionError: When a new repo cannot be provisioned.
        """
        assert task_guid, "Error getting clone, task_guid is mandatory"
        assert task_name, "Error getting clone, task_name is mandatory"
        try:
            roster_entry = self.roster.reserve_clone(task_guid, task_name)
            logger.debug('roster: %s' % roster_entry)
            clone = self.dvcs.get_depot_from_path(roster_entry.path,
                                                  parent=self.main_cache)
        except RosterError:
            logger.debug('no roster entry found, cloning')
            # Create a new clone in the squadron if none are free
            clone = self._provision_new_clone()
            self.roster.add(clone.path, task_guid, task_name)

        if default_source is not None:
            clone.set_source(default_source)

        if requirements is not None:
            # Request the refresh to comply with the requirements.
            clone.request_refresh(requirements)

        return clone

    def give_me_depot_from_path(self, path):
        """
        Gets a repository from the current path without checking its state, no
        matter if it's FREE or INUSE

        :param path: depot path to get
        :type path: string
        """
        if self.dvcs.is_a_depot(path):
            return self.dvcs.get_depot_from_path(path, parent=self.main_cache)
        raise CloneProvisionError(
            "Error getting clone from path %s, it doesn't exist" % path)

    def free_depot(self, depot, task_guid):
        """
        Frees a repository for new uses.

        :param clone: a RepoWorkspace to be freed from use.
        :param task_guid: Identifier of the task reserving the clone.
        :raises RepoFreeError: When a repo cannot be freed.
        """
        self.dvcs.clear_depot(depot.path)
        self.roster.free_clone(self.get_not_available_clone(depot.path),
                               task_guid)

    @staticmethod
    def _get_first_matching_clone(clone_list, path):
        for clone in clone_list:
            if clone.path == path:
                return clone
        return None

    def get_available_clone(self, path):
        """
        :returns: a clone with the available clone specified by path
        :rtype: RepoWorkspace
        """
        clone_list = self.roster.get_available()
        return self._get_first_matching_clone(clone_list, path)

    def get_not_available_clone(self, path):
        """
        :returns: a clone with the not available clone specified by path
        :rtype: RepoWorkspace
        """
        clone_list = self.roster.get_not_available()
        return self._get_first_matching_clone(clone_list, path)
 def setUp(self):
     self.roster = Roster(':memory:')
class TestRoster(unittest.TestCase):
    def setUp(self):
        self.roster = Roster(':memory:')

    def test_update(self):
        self.roster['/test'] = clone_mother(path='/test', status=Clone.INUSE)
        res = clone_mother(path='/test', status=Clone.INUSE)
        self.assertTrue(res.__eq__(self.roster['/test']))

    def test_delete(self):
        self.roster['/test1'] = clone_mother(status=Clone.INUSE)
        self.roster['/test2'] = clone_mother(status=Clone.INUSE)
        self.roster.clear()
        self.assertEquals(0, len(self.roster))

    def test_missing(self):
        self.assertRaises(KeyError, lambda x: self.roster[x], '/test')

    def test_fail_to_reserve_without_clones(self):
        self.assertRaises(RosterError, self.roster.reserve_clone, '1', 'test')

    def test_fail_to_reserve_without_free_clones(self):
        self.roster['/test'] = clone_mother(status=Clone.INUSE)
        self.assertRaises(RosterError, self.roster.reserve_clone, '1', 'test')

    def test_get_free(self):
        r1 = clone_mother(status=Clone.INUSE)
        r2 = clone_mother(status=Clone.FREE)
        self.roster['/test1'] = r1
        self.roster['/test2'] = r2
        r2.status = Clone.INUSE
        self.assertEquals(r2, self.roster.reserve_clone('1', 'test'))

    def test_clone_str(self):
        r1 = clone_mother(status=Clone.INUSE)
        self.assertEquals(r1.__str__(), str(r1.__dict__))
        self.assertEquals(r1.__repr__(), str(r1.__dict__))

    def test_free_clone(self):
        r1 = clone_mother(status=Clone.INUSE, task='1')
        r2 = clone_mother(status=Clone.FREE, task='2')
        r3 = clone_mother(status=Clone.INUSE, task='2')

        self.roster['/test1'] = r1
        self.roster['/test2'] = r2
        self.roster['/test3'] = r3
        self.roster.free_clone(r1, '1')
        r1 = self.roster['/test1']
        self.assertEquals(Clone.FREE, r1.status)
        # Check cannot remove elements from the roster not owned.
        self.assertRaises(RosterError, self.roster.free_clone, r3, 1)

    def test_fail_to_modify_others_clone(self):
        r1 = clone_mother(path='/test', status=Clone.INUSE, task='2')
        r2 = clone_mother(path='/test', status=Clone.INUSE, task='1')
        self.roster['/test'] = r1

        def assign(x, y):
            self.roster[x] = y

        self.assertRaises(RosterError, assign, '/test', r2)

    def test_add(self):
        self.roster.add('/test', 1, 'test')
        self.assertIn('/test', self.roster)
        self.assertRaises(RosterError, self.roster.add, '/test', 1, 'test')
        r1 = self.roster.add('/test1', 1, 'test')
        self.assertIn(r1, self.roster.values())

    def test_iter(self):
        self.assertListEqual([], list(self.roster))
        r1 = self.roster.add('/test1', 1, 'test')
        r2 = self.roster.add('/test2', 1, 'test')
        r3 = self.roster.add('/test3', 1, 'test')
        repo_list = [u'/test1', u'/test2', u'/test3']
        self.assertListEqual(repo_list, list(self.roster))
        self.assertListEqual([r1, r2, r3], list(self.roster.values()))

    def test_get_available(self):
        self.assertListEqual([], self.roster.get_available())
        r1 = self.roster.add('/test1', u'1', 'test')
        r2 = self.roster.add('/test2', u'2', 'test')
        r3 = self.roster.add('/test3', u'1', 'test')
        self.assertListEqual([], self.roster.get_available())
        self.roster.free_clone(r1, u'1')
        self.roster.free_clone(r2, u'2')
        self.roster.free_clone(r3, u'1')
        self.assertListEqual([r1, r2, r3], self.roster.get_available())

    def test_get_not_available(self):
        self.assertListEqual([], self.roster.get_not_available())
        self.roster.add('/test', u'1', 'test')
        r = self.roster['/test']
        self.assertListEqual([r], self.roster.get_not_available())
        self.roster.free_clone(r, u'1')
        self.assertListEqual([], self.roster.get_not_available())

    def test_get_single(self):
        self.assertListEqual([], self.roster.get_available())
        r1 = self.roster.add('/test1', u'1', 'test')
        self.assertEquals(r1, self.roster['/test1'])
        r2 = self.roster.add('/test2', u'1', 'test')
        self.assertEquals(r2, self.roster['/test2'])

    def test_add_limit(self):
        # tests the limit imposed to the creation of clones
        roster = Roster(':memory:', max_clones=1)
        roster.add('/test1', 1, 'test')
        self.assertIn('/test1', roster)
        self.assertRaises(MaxClonesLimitReached, roster.add, '/test2', 1,
                          'test')

    def test_free_clone_by_timeout(self):
        timeout = timedelta(seconds=1)
        initial_time = 0.0

        roster = Roster(':memory:', clone_timeout=timeout)
        roster._get_time_ = lambda: initial_time

        r1 = clone_mother(status=Clone.INUSE, task='1')
        r2 = clone_mother(status=Clone.FREE, task='2')
        r3 = clone_mother(status=Clone.INUSE, task='2')
        roster['/test1'] = r1
        roster['/test2'] = r2
        roster['/test3'] = r3

        self.assertListEqual([], roster._get_old_clones_())

        roster._get_time_ = lambda: initial_time + timeout.seconds + 1
        self.assertGreater(len(roster._get_old_clones_()), 0)

        roster._clean_old_clones()
        self.assertListEqual([], roster._get_old_clones_())

    def test_multiple_rosters_persistence(self):
        fd, database_path = tempfile.mkstemp()
        try:
            roster1 = Roster(database_path)
            roster2 = Roster(database_path)
            r1 = clone_mother(task='1', status=Clone.INUSE)
            r2 = clone_mother(task='2')

            roster1['/test1'] = r1

            def assign(roster, x, y):
                roster[x] = y

            self.assertRaises(RosterError, assign, roster2, '/test1', r2)

            roster1['/test2'] = r2
            roster2.reserve_clone('2', 'test2')
            self.assertRaises(RosterError, roster1.reserve_clone, '1', 'test1')
        finally:
            os.remove(database_path)
 def setUp(self):
     self.roster = Roster(':memory:')
class TestRoster(unittest.TestCase):
    def setUp(self):
        self.roster = Roster(':memory:')

    def test_update(self):
        self.roster['/test'] = clone_mother(path='/test', status=Clone.INUSE)
        res = clone_mother(path='/test', status=Clone.INUSE)
        self.assertTrue(res.__eq__(self.roster['/test']))

    def test_delete(self):
        self.roster['/test1'] = clone_mother(status=Clone.INUSE)
        self.roster['/test2'] = clone_mother(status=Clone.INUSE)
        self.roster.clear()
        self.assertEquals(0, len(self.roster))

    def test_missing(self):
        self.assertRaises(KeyError, lambda x: self.roster[x], '/test')

    def test_fail_to_reserve_without_clones(self):
        self.assertRaises(RosterError, self.roster.reserve_clone, '1', 'test')

    def test_fail_to_reserve_without_free_clones(self):
        self.roster['/test'] = clone_mother(status=Clone.INUSE)
        self.assertRaises(RosterError, self.roster.reserve_clone, '1', 'test')

    def test_get_free(self):
        r1 = clone_mother(status=Clone.INUSE)
        r2 = clone_mother(status=Clone.FREE)
        self.roster['/test1'] = r1
        self.roster['/test2'] = r2
        r2.status = Clone.INUSE
        self.assertEquals(r2, self.roster.reserve_clone('1', 'test'))

    def test_clone_str(self):
        r1 = clone_mother(status=Clone.INUSE)
        self.assertEquals(r1.__str__(), str(r1.__dict__))
        self.assertEquals(r1.__repr__(), str(r1.__dict__))

    def test_free_clone(self):
        r1 = clone_mother(status=Clone.INUSE, task='1')
        r2 = clone_mother(status=Clone.FREE, task='2')
        r3 = clone_mother(status=Clone.INUSE, task='2')

        self.roster['/test1'] = r1
        self.roster['/test2'] = r2
        self.roster['/test3'] = r3
        self.roster.free_clone(r1, '1')
        r1 = self.roster['/test1']
        self.assertEquals(Clone.FREE, r1.status)
        # Check cannot remove elements from the roster not owned.
        self.assertRaises(RosterError, self.roster.free_clone, r3, 1)

    def test_fail_to_modify_others_clone(self):
        r1 = clone_mother(path='/test', status=Clone.INUSE, task='2')
        r2 = clone_mother(path='/test', status=Clone.INUSE, task='1')
        self.roster['/test'] = r1

        def assign(x, y):
            self.roster[x] = y
        self.assertRaises(RosterError, assign, '/test', r2)

    def test_add(self):
        self.roster.add('/test', 1, 'test')
        self.assertIn('/test', self.roster)
        self.assertRaises(RosterError, self.roster.add, '/test', 1, 'test')
        r1 = self.roster.add('/test1', 1, 'test')
        self.assertIn(r1, self.roster.values())

    def test_iter(self):
        self.assertListEqual([], list(self.roster))
        r1 = self.roster.add('/test1', 1, 'test')
        r2 = self.roster.add('/test2', 1, 'test')
        r3 = self.roster.add('/test3', 1, 'test')
        repo_list = [u'/test1', u'/test2', u'/test3']
        self.assertListEqual(repo_list, list(self.roster))
        self.assertListEqual([r1, r2, r3], self.roster.values())

    def test_get_available(self):
        self.assertListEqual([], self.roster.get_available())
        r1 = self.roster.add('/test1', u'1', 'test')
        r2 = self.roster.add('/test2', u'2', 'test')
        r3 = self.roster.add('/test3', u'1', 'test')
        self.assertListEqual([], self.roster.get_available())
        self.roster.free_clone(r1, u'1')
        self.roster.free_clone(r2, u'2')
        self.roster.free_clone(r3, u'1')
        self.assertListEqual([r1, r2, r3], self.roster.get_available())

    def test_get_not_available(self):
        self.assertListEqual([], self.roster.get_not_available())
        self.roster.add('/test', u'1', 'test')
        r = self.roster['/test']
        self.assertListEqual([r], self.roster.get_not_available())
        self.roster.free_clone(r, u'1')
        self.assertListEqual([], self.roster.get_not_available())

    def test_get_single(self):
        self.assertListEqual([], self.roster.get_available())
        r1 = self.roster.add('/test1', u'1', 'test')
        self.assertEquals(r1, self.roster['/test1'])
        r2 = self.roster.add('/test2', u'1', 'test')
        self.assertEquals(r2, self.roster['/test2'])

    def test_add_limit(self):
        # tests the limit imposed to the creation of clones
        roster = Roster(':memory:', max_clones=1)
        roster.add('/test1', 1, 'test')
        self.assertIn('/test1', roster)
        self.assertRaises(
            MaxClonesLimitReached, roster.add, '/test2', 1, 'test')

    def test_free_clone_by_timeout(self):
        timeout = timedelta(seconds=1)
        initial_time = 0.0

        roster = Roster(':memory:', clone_timeout=timeout)
        roster._get_time_ = lambda: initial_time

        r1 = clone_mother(status=Clone.INUSE, task='1')
        r2 = clone_mother(status=Clone.FREE, task='2')
        r3 = clone_mother(status=Clone.INUSE, task='2')
        roster['/test1'] = r1
        roster['/test2'] = r2
        roster['/test3'] = r3

        self.assertListEqual([], roster._get_old_clones_())

        roster._get_time_ = lambda: initial_time + timeout.seconds + 1
        self.assertGreater(len(roster._get_old_clones_()), 0)

        roster._clean_old_clones()
        self.assertListEqual([], roster._get_old_clones_())

    def test_multiple_rosters_persistence(self):
        fd, database_path = tempfile.mkstemp()
        try:
            roster1 = Roster(database_path)
            roster2 = Roster(database_path)
            r1 = clone_mother(task='1', status=Clone.INUSE)
            r2 = clone_mother(task='2')

            roster1['/test1'] = r1

            def assign(roster, x, y):
                roster[x] = y
            self.assertRaises(RosterError, assign, roster2, '/test1', r2)

            roster1['/test2'] = r2
            roster2.reserve_clone('2', 'test2')
            self.assertRaises(RosterError, roster1.reserve_clone, '1', 'test1')
        finally:
            os.remove(database_path)