def __init__(self,
main_workspace="~/.repo",
repo_kind='hg',
main_source=None):
self.dvcs = DepotOperations.get_depot_operations(repo_kind)
try:
self.main_work_path = os.path.expanduser(main_workspace)
logger.debug('Main workspace: %s' % self.main_work_path)
self.main_cache_path = os.path.join(self.main_work_path,
DepotManager.cache_name)
self.squadron_roster_path = os.path.join(
self.main_work_path, DepotManager.squadron_roster_name)
# Create the environment.
if not os.path.isdir(self.main_work_path):
os.makedirs(self.main_work_path)
# Create main cache.
if not self.dvcs.is_a_depot(self.main_cache_path):
self.main_cache = self.dvcs.init_depot(self.main_cache_path,
source=main_source)
else:
self.main_cache = Depot(self.main_cache_path, None, self.dvcs)
self.roster = Roster(self.squadron_roster_path)
except Exception as e:
raise CloneProvisionError(e)
def test_add_limit(self):
# tests the limit imposed to the creation of clones
roster = Roster(':memory:', max_clones=1)
roster.add('/test1', 1, 'test')
self.assertIn('/test1', roster)
self.assertRaises(MaxClonesLimitReached, roster.add, '/test2', 1,
'test')
def test_add_limit(self):
# tests the limit imposed to the creation of clones
roster = Roster(':memory:', max_clones=1)
roster.add('/test1', 1, 'test')
self.assertIn('/test1', roster)
self.assertRaises(
MaxClonesLimitReached, roster.add, '/test2', 1, 'test')
def test_multiple_rosters_persistence(self):
fd, database_path = tempfile.mkstemp()
try:
roster1 = Roster(database_path)
roster2 = Roster(database_path)
r1 = clone_mother(task='1', status=Clone.INUSE)
r2 = clone_mother(task='2')
roster1['/test1'] = r1
def assign(roster, x, y):
roster[x] = y
self.assertRaises(RosterError, assign, roster2, '/test1', r2)
roster1['/test2'] = r2
roster2.reserve_clone('2', 'test2')
self.assertRaises(RosterError, roster1.reserve_clone, '1', 'test1')
finally:
os.remove(database_path)
def test_multiple_rosters_persistence(self):
fd, database_path = tempfile.mkstemp()
try:
roster1 = Roster(database_path)
roster2 = Roster(database_path)
r1 = clone_mother(task='1', status=Clone.INUSE)
r2 = clone_mother(task='2')
roster1['/test1'] = r1
def assign(roster, x, y):
roster[x] = y
self.assertRaises(RosterError, assign, roster2, '/test1', r2)
roster1['/test2'] = r2
roster2.reserve_clone('2', 'test2')
self.assertRaises(RosterError, roster1.reserve_clone, '1', 'test1')
finally:
os.remove(database_path)
def test_free_clone_by_timeout(self):
timeout = timedelta(seconds=1)
initial_time = 0.0
roster = Roster(':memory:', clone_timeout=timeout)
roster._get_time_ = lambda: initial_time
r1 = clone_mother(status=Clone.INUSE, task='1')
r2 = clone_mother(status=Clone.FREE, task='2')
r3 = clone_mother(status=Clone.INUSE, task='2')
roster['/test1'] = r1
roster['/test2'] = r2
roster['/test3'] = r3
self.assertListEqual([], roster._get_old_clones_())
roster._get_time_ = lambda: initial_time + timeout.seconds + 1
self.assertGreater(len(roster._get_old_clones_()), 0)
roster._clean_old_clones()
self.assertListEqual([], roster._get_old_clones_())
def test_free_clone_by_timeout(self):
timeout = timedelta(seconds=1)
initial_time = 0.0
roster = Roster(':memory:', clone_timeout=timeout)
roster._get_time_ = lambda: initial_time
r1 = clone_mother(status=Clone.INUSE, task='1')
r2 = clone_mother(status=Clone.FREE, task='2')
r3 = clone_mother(status=Clone.INUSE, task='2')
roster['/test1'] = r1
roster['/test2'] = r2
roster['/test3'] = r3
self.assertListEqual([], roster._get_old_clones_())
roster._get_time_ = lambda: initial_time + timeout.seconds + 1
self.assertGreater(len(roster._get_old_clones_()), 0)
roster._clean_old_clones()
self.assertListEqual([], roster._get_old_clones_())
class DepotManager(object):
"""
Acts as an public facing API for working with managed clones.
:param main_workspace: directory where all the workspaces will be
created.
:type main_workspace: string
:param repo_kind: Repository type
:type repo_kind: string
:param main_source: FIXME
:type main_source: string
"""
# Name of the main repo cache.
cache_name = 'main_cache'
# Name of the file storing the roster.
squadron_roster_name = 'squadron_roster.db'
# Prefix for the clones used by the workers.
workspaces_prefix = 'workspace'
def __init__(self,
main_workspace="~/.repo",
repo_kind='hg',
main_source=None):
self.dvcs = DepotOperations.get_depot_operations(repo_kind)
try:
self.main_work_path = os.path.expanduser(main_workspace)
logger.debug('Main workspace: %s' % self.main_work_path)
self.main_cache_path = os.path.join(self.main_work_path,
DepotManager.cache_name)
self.squadron_roster_path = os.path.join(
self.main_work_path, DepotManager.squadron_roster_name)
# Create the environment.
if not os.path.isdir(self.main_work_path):
os.makedirs(self.main_work_path)
# Create main cache.
if not self.dvcs.is_a_depot(self.main_cache_path):
self.main_cache = self.dvcs.init_depot(self.main_cache_path,
source=main_source)
else:
self.main_cache = Depot(self.main_cache_path, None, self.dvcs)
self.roster = Roster(self.squadron_roster_path)
except Exception as e:
raise CloneProvisionError(e)
def _provision_new_clone(self):
try:
# Create a new safe directory for the clone.
clone_directory = tempfile.mkdtemp(
prefix=DepotManager.workspaces_prefix, dir=self.main_work_path)
# Create repo (Using the cache)
result = self.dvcs.init_depot(clone_directory,
parent=self.main_cache)
except Exception:
logger.exception("Error provisioning new clone")
raise CloneProvisionError("Error provisioning new clone")
return result
def give_me_depot(self,
task_guid,
task_name,
requirements=None,
default_source=None):
"""
Reserves or prepares a new repository workspace.
:param task_guid: Identifier of the task reserving the clone.
:param task_name: Name of the task for information purposes
:param requirements: requirements to pull
:param default_source: default clone source
:returns: a free repo.
:rtype: :py:class:`~repoman.depot.Depot`
:raises RepoProvisionError: When a new repo cannot be provisioned.
"""
assert task_guid, "Error getting clone, task_guid is mandatory"
assert task_name, "Error getting clone, task_name is mandatory"
try:
roster_entry = self.roster.reserve_clone(task_guid, task_name)
logger.debug('roster: %s' % roster_entry)
clone = self.dvcs.get_depot_from_path(roster_entry.path,
parent=self.main_cache)
except RosterError:
logger.debug('no roster entry found, cloning')
# Create a new clone in the squadron if none are free
clone = self._provision_new_clone()
self.roster.add(clone.path, task_guid, task_name)
if default_source is not None:
clone.set_source(default_source)
if requirements is not None:
# Request the refresh to comply with the requirements.
clone.request_refresh(requirements)
return clone
def give_me_depot_from_path(self, path):
"""
Gets a repository from the current path without checking its state, no
matter if it's FREE or INUSE
:param path: depot path to get
:type path: string
"""
if self.dvcs.is_a_depot(path):
return self.dvcs.get_depot_from_path(path, parent=self.main_cache)
raise CloneProvisionError(
"Error getting clone from path %s, it doesn't exist" % path)
def free_depot(self, depot, task_guid):
"""
Frees a repository for new uses.
:param clone: a RepoWorkspace to be freed from use.
:param task_guid: Identifier of the task reserving the clone.
:raises RepoFreeError: When a repo cannot be freed.
"""
self.dvcs.clear_depot(depot.path)
self.roster.free_clone(self.get_not_available_clone(depot.path),
task_guid)
@staticmethod
def _get_first_matching_clone(clone_list, path):
for clone in clone_list:
if clone.path == path:
return clone
return None
def get_available_clone(self, path):
"""
:returns: a clone with the available clone specified by path
:rtype: RepoWorkspace
"""
clone_list = self.roster.get_available()
return self._get_first_matching_clone(clone_list, path)
def get_not_available_clone(self, path):
"""
:returns: a clone with the not available clone specified by path
:rtype: RepoWorkspace
"""
clone_list = self.roster.get_not_available()
return self._get_first_matching_clone(clone_list, path)
def setUp(self):
self.roster = Roster(':memory:')
class TestRoster(unittest.TestCase):
def setUp(self):
self.roster = Roster(':memory:')
def test_update(self):
self.roster['/test'] = clone_mother(path='/test', status=Clone.INUSE)
res = clone_mother(path='/test', status=Clone.INUSE)
self.assertTrue(res.__eq__(self.roster['/test']))
def test_delete(self):
self.roster['/test1'] = clone_mother(status=Clone.INUSE)
self.roster['/test2'] = clone_mother(status=Clone.INUSE)
self.roster.clear()
self.assertEquals(0, len(self.roster))
def test_missing(self):
self.assertRaises(KeyError, lambda x: self.roster[x], '/test')
def test_fail_to_reserve_without_clones(self):
self.assertRaises(RosterError, self.roster.reserve_clone, '1', 'test')
def test_fail_to_reserve_without_free_clones(self):
self.roster['/test'] = clone_mother(status=Clone.INUSE)
self.assertRaises(RosterError, self.roster.reserve_clone, '1', 'test')
def test_get_free(self):
r1 = clone_mother(status=Clone.INUSE)
r2 = clone_mother(status=Clone.FREE)
self.roster['/test1'] = r1
self.roster['/test2'] = r2
r2.status = Clone.INUSE
self.assertEquals(r2, self.roster.reserve_clone('1', 'test'))
def test_clone_str(self):
r1 = clone_mother(status=Clone.INUSE)
self.assertEquals(r1.__str__(), str(r1.__dict__))
self.assertEquals(r1.__repr__(), str(r1.__dict__))
def test_free_clone(self):
r1 = clone_mother(status=Clone.INUSE, task='1')
r2 = clone_mother(status=Clone.FREE, task='2')
r3 = clone_mother(status=Clone.INUSE, task='2')
self.roster['/test1'] = r1
self.roster['/test2'] = r2
self.roster['/test3'] = r3
self.roster.free_clone(r1, '1')
r1 = self.roster['/test1']
self.assertEquals(Clone.FREE, r1.status)
# Check cannot remove elements from the roster not owned.
self.assertRaises(RosterError, self.roster.free_clone, r3, 1)
def test_fail_to_modify_others_clone(self):
r1 = clone_mother(path='/test', status=Clone.INUSE, task='2')
r2 = clone_mother(path='/test', status=Clone.INUSE, task='1')
self.roster['/test'] = r1
def assign(x, y):
self.roster[x] = y
self.assertRaises(RosterError, assign, '/test', r2)
def test_add(self):
self.roster.add('/test', 1, 'test')
self.assertIn('/test', self.roster)
self.assertRaises(RosterError, self.roster.add, '/test', 1, 'test')
r1 = self.roster.add('/test1', 1, 'test')
self.assertIn(r1, self.roster.values())
def test_iter(self):
self.assertListEqual([], list(self.roster))
r1 = self.roster.add('/test1', 1, 'test')
r2 = self.roster.add('/test2', 1, 'test')
r3 = self.roster.add('/test3', 1, 'test')
repo_list = [u'/test1', u'/test2', u'/test3']
self.assertListEqual(repo_list, list(self.roster))
self.assertListEqual([r1, r2, r3], list(self.roster.values()))
def test_get_available(self):
self.assertListEqual([], self.roster.get_available())
r1 = self.roster.add('/test1', u'1', 'test')
r2 = self.roster.add('/test2', u'2', 'test')
r3 = self.roster.add('/test3', u'1', 'test')
self.assertListEqual([], self.roster.get_available())
self.roster.free_clone(r1, u'1')
self.roster.free_clone(r2, u'2')
self.roster.free_clone(r3, u'1')
self.assertListEqual([r1, r2, r3], self.roster.get_available())
def test_get_not_available(self):
self.assertListEqual([], self.roster.get_not_available())
self.roster.add('/test', u'1', 'test')
r = self.roster['/test']
self.assertListEqual([r], self.roster.get_not_available())
self.roster.free_clone(r, u'1')
self.assertListEqual([], self.roster.get_not_available())
def test_get_single(self):
self.assertListEqual([], self.roster.get_available())
r1 = self.roster.add('/test1', u'1', 'test')
self.assertEquals(r1, self.roster['/test1'])
r2 = self.roster.add('/test2', u'1', 'test')
self.assertEquals(r2, self.roster['/test2'])
def test_add_limit(self):
# tests the limit imposed to the creation of clones
roster = Roster(':memory:', max_clones=1)
roster.add('/test1', 1, 'test')
self.assertIn('/test1', roster)
self.assertRaises(MaxClonesLimitReached, roster.add, '/test2', 1,
'test')
def test_free_clone_by_timeout(self):
timeout = timedelta(seconds=1)
initial_time = 0.0
roster = Roster(':memory:', clone_timeout=timeout)
roster._get_time_ = lambda: initial_time
r1 = clone_mother(status=Clone.INUSE, task='1')
r2 = clone_mother(status=Clone.FREE, task='2')
r3 = clone_mother(status=Clone.INUSE, task='2')
roster['/test1'] = r1
roster['/test2'] = r2
roster['/test3'] = r3
self.assertListEqual([], roster._get_old_clones_())
roster._get_time_ = lambda: initial_time + timeout.seconds + 1
self.assertGreater(len(roster._get_old_clones_()), 0)
roster._clean_old_clones()
self.assertListEqual([], roster._get_old_clones_())
def test_multiple_rosters_persistence(self):
fd, database_path = tempfile.mkstemp()
try:
roster1 = Roster(database_path)
roster2 = Roster(database_path)
r1 = clone_mother(task='1', status=Clone.INUSE)
r2 = clone_mother(task='2')
roster1['/test1'] = r1
def assign(roster, x, y):
roster[x] = y
self.assertRaises(RosterError, assign, roster2, '/test1', r2)
roster1['/test2'] = r2
roster2.reserve_clone('2', 'test2')
self.assertRaises(RosterError, roster1.reserve_clone, '1', 'test1')
finally:
os.remove(database_path)
def setUp(self):
self.roster = Roster(':memory:')
class TestRoster(unittest.TestCase):
def setUp(self):
self.roster = Roster(':memory:')
def test_update(self):
self.roster['/test'] = clone_mother(path='/test', status=Clone.INUSE)
res = clone_mother(path='/test', status=Clone.INUSE)
self.assertTrue(res.__eq__(self.roster['/test']))
def test_delete(self):
self.roster['/test1'] = clone_mother(status=Clone.INUSE)
self.roster['/test2'] = clone_mother(status=Clone.INUSE)
self.roster.clear()
self.assertEquals(0, len(self.roster))
def test_missing(self):
self.assertRaises(KeyError, lambda x: self.roster[x], '/test')
def test_fail_to_reserve_without_clones(self):
self.assertRaises(RosterError, self.roster.reserve_clone, '1', 'test')
def test_fail_to_reserve_without_free_clones(self):
self.roster['/test'] = clone_mother(status=Clone.INUSE)
self.assertRaises(RosterError, self.roster.reserve_clone, '1', 'test')
def test_get_free(self):
r1 = clone_mother(status=Clone.INUSE)
r2 = clone_mother(status=Clone.FREE)
self.roster['/test1'] = r1
self.roster['/test2'] = r2
r2.status = Clone.INUSE
self.assertEquals(r2, self.roster.reserve_clone('1', 'test'))
def test_clone_str(self):
r1 = clone_mother(status=Clone.INUSE)
self.assertEquals(r1.__str__(), str(r1.__dict__))
self.assertEquals(r1.__repr__(), str(r1.__dict__))
def test_free_clone(self):
r1 = clone_mother(status=Clone.INUSE, task='1')
r2 = clone_mother(status=Clone.FREE, task='2')
r3 = clone_mother(status=Clone.INUSE, task='2')
self.roster['/test1'] = r1
self.roster['/test2'] = r2
self.roster['/test3'] = r3
self.roster.free_clone(r1, '1')
r1 = self.roster['/test1']
self.assertEquals(Clone.FREE, r1.status)
# Check cannot remove elements from the roster not owned.
self.assertRaises(RosterError, self.roster.free_clone, r3, 1)
def test_fail_to_modify_others_clone(self):
r1 = clone_mother(path='/test', status=Clone.INUSE, task='2')
r2 = clone_mother(path='/test', status=Clone.INUSE, task='1')
self.roster['/test'] = r1
def assign(x, y):
self.roster[x] = y
self.assertRaises(RosterError, assign, '/test', r2)
def test_add(self):
self.roster.add('/test', 1, 'test')
self.assertIn('/test', self.roster)
self.assertRaises(RosterError, self.roster.add, '/test', 1, 'test')
r1 = self.roster.add('/test1', 1, 'test')
self.assertIn(r1, self.roster.values())
def test_iter(self):
self.assertListEqual([], list(self.roster))
r1 = self.roster.add('/test1', 1, 'test')
r2 = self.roster.add('/test2', 1, 'test')
r3 = self.roster.add('/test3', 1, 'test')
repo_list = [u'/test1', u'/test2', u'/test3']
self.assertListEqual(repo_list, list(self.roster))
self.assertListEqual([r1, r2, r3], self.roster.values())
def test_get_available(self):
self.assertListEqual([], self.roster.get_available())
r1 = self.roster.add('/test1', u'1', 'test')
r2 = self.roster.add('/test2', u'2', 'test')
r3 = self.roster.add('/test3', u'1', 'test')
self.assertListEqual([], self.roster.get_available())
self.roster.free_clone(r1, u'1')
self.roster.free_clone(r2, u'2')
self.roster.free_clone(r3, u'1')
self.assertListEqual([r1, r2, r3], self.roster.get_available())
def test_get_not_available(self):
self.assertListEqual([], self.roster.get_not_available())
self.roster.add('/test', u'1', 'test')
r = self.roster['/test']
self.assertListEqual([r], self.roster.get_not_available())
self.roster.free_clone(r, u'1')
self.assertListEqual([], self.roster.get_not_available())
def test_get_single(self):
self.assertListEqual([], self.roster.get_available())
r1 = self.roster.add('/test1', u'1', 'test')
self.assertEquals(r1, self.roster['/test1'])
r2 = self.roster.add('/test2', u'1', 'test')
self.assertEquals(r2, self.roster['/test2'])
def test_add_limit(self):
# tests the limit imposed to the creation of clones
roster = Roster(':memory:', max_clones=1)
roster.add('/test1', 1, 'test')
self.assertIn('/test1', roster)
self.assertRaises(
MaxClonesLimitReached, roster.add, '/test2', 1, 'test')
def test_free_clone_by_timeout(self):
timeout = timedelta(seconds=1)
initial_time = 0.0
roster = Roster(':memory:', clone_timeout=timeout)
roster._get_time_ = lambda: initial_time
r1 = clone_mother(status=Clone.INUSE, task='1')
r2 = clone_mother(status=Clone.FREE, task='2')
r3 = clone_mother(status=Clone.INUSE, task='2')
roster['/test1'] = r1
roster['/test2'] = r2
roster['/test3'] = r3
self.assertListEqual([], roster._get_old_clones_())
roster._get_time_ = lambda: initial_time + timeout.seconds + 1
self.assertGreater(len(roster._get_old_clones_()), 0)
roster._clean_old_clones()
self.assertListEqual([], roster._get_old_clones_())
def test_multiple_rosters_persistence(self):
fd, database_path = tempfile.mkstemp()
try:
roster1 = Roster(database_path)
roster2 = Roster(database_path)
r1 = clone_mother(task='1', status=Clone.INUSE)
r2 = clone_mother(task='2')
roster1['/test1'] = r1
def assign(roster, x, y):
roster[x] = y
self.assertRaises(RosterError, assign, roster2, '/test1', r2)
roster1['/test2'] = r2
roster2.reserve_clone('2', 'test2')
self.assertRaises(RosterError, roster1.reserve_clone, '1', 'test1')
finally:
os.remove(database_path)
