def do_query(next_in_queue, free_ccm, semaphore, delim, p_queue, use_cache):
(project, parent_proj) = next_in_queue
ccm_addr, database = get_and_lock_free_ccm_addr(free_ccm)
ccm = SynergySession(database, ccm_addr=ccm_addr)
ccm.keep_session_alive = True
# logger.debug('Querying: %s' % project.get_object_name())
result = get_members(project, ccm, parent_proj)
if use_cache:
objects = []
na_obj = []
for item in result:
try:
objects.append(ccm_cache.get_object(item['objectname'], ccm))
except ccm_cache.ObjectCacheException:
objects.append(SynergyObject(item['objectname'], delim))
na_obj.append(item['objectname'])
if na_obj:
logger.warning("Objects not avaliable in this db:")
logger.warning(', '.join(na_obj))
else:
objects = [SynergyObject(item['objectname'], delim) for item in result]
# if a project is being queried it might have more than one dir with the
# same name as the project associated, find the directory that has the
# project associated as the directory's parent
if project.get_type() == 'project':
if len(objects) > 1:
objects = find_root_project(project, objects, ccm)
p_queue.put((project, objects))
entry = free_ccm[ccm_addr]
entry['free'] = True
free_ccm[ccm_addr] = entry
semaphore.release()
def get_objects_in_project_parallel(project, ccmpool=None):
mgr = Manager()
free_ccm = mgr.dict()
for ccm in ccmpool.sessionArray.values():
free_ccm[ccm.getCCM_ADDR()] = True
ccm_addr = get_and_lock_free_ccm_addr(free_ccm)
ccm = SynergySession(None, ccm_addr=ccm_addr)
ccm.keep_session_alive = True
delim = ccm.delim()
# unlock
free_ccm[ccm_addr] = True
semaphore = mgr.Semaphore(ccmpool.nr_sessions)
so = SynergyObject(project, delim)
p_queue = mgr.Queue()
c_queue = mgr.Queue()
c_queue.put((so, None))
p_queue.put(so)
# start the produce and consumer thread
prod = Process(target=producer, args=(c_queue, p_queue, free_ccm))
cons = Process(target=consumer, args=(c_queue, p_queue, free_ccm, semaphore, delim))
prod.start()
cons.start()
print "Waiting to join"
cons.join()
hierarchy = p_queue.get()
prod.join()
return hierarchy
def create_ccm_session_from_config():
f = open('config.p', 'rb')
config = cPickle.load(f)
f.close()
ccm = SynergySession(config['database'])
return ccm
def get_objects_in_project_parallel(project, ccmpool=None, use_cache=False):
""" Get all the objects and paths of project with use of multiple ccm
sessions """
mgr = Manager()
free_ccm = mgr.dict()
for ccm in ccmpool.sessionArray.values():
free_ccm[ccm.getCCM_ADDR()] = {'free': True, 'database': ccm.database}
ccm_addr, database = get_and_lock_free_ccm_addr(free_ccm)
ccm = SynergySession(database, ccm_addr=ccm_addr)
ccm.keep_session_alive = True
delim = ccm.delim()
semaphore = mgr.Semaphore(ccmpool.nr_sessions)
# Starting project
if use_cache:
start_object = ccm_cache.get_object(project, ccm)
else:
start_object = SynergyObject(project, delim)
# unlock update dict entry to inform manager
entry = free_ccm[ccm_addr]
entry['free'] = True
free_ccm[ccm_addr] = entry
p_queue = mgr.Queue()
c_queue = mgr.Queue()
c_queue.put((start_object, None))
p_queue.put(start_object)
# start the produce and consumer thread
prod = Process(target=producer, args=(c_queue, p_queue, free_ccm))
cons = Process(target=consumer,
args=(c_queue, p_queue, free_ccm, semaphore, delim,
use_cache))
prod.start()
cons.start()
logger.debug("Waiting to join")
cons.join()
hierarchy = p_queue.get()
prod.join()
return hierarchy
def get_objects_in_project_parallel(project, ccmpool=None, use_cache=False):
""" Get all the objects and paths of project with use of multiple ccm
sessions """
mgr = Manager()
free_ccm = mgr.dict()
for ccm in ccmpool.sessionArray.values():
free_ccm[ccm.getCCM_ADDR()] = {'free': True, 'database': ccm.database}
ccm_addr, database = get_and_lock_free_ccm_addr(free_ccm)
ccm = SynergySession(database, ccm_addr=ccm_addr)
ccm.keep_session_alive = True
delim = ccm.delim()
semaphore = mgr.Semaphore(ccmpool.nr_sessions)
# Starting project
if use_cache:
start_object = ccm_cache.get_object(project, ccm)
else:
start_object = SynergyObject(project, delim)
# unlock update dict entry to inform manager
entry = free_ccm[ccm_addr]
entry['free'] = True
free_ccm[ccm_addr] = entry
p_queue = mgr.Queue()
c_queue = mgr.Queue()
c_queue.put((start_object, None))
p_queue.put(start_object)
# start the produce and consumer thread
prod = Process(target=producer, args=(c_queue, p_queue, free_ccm))
cons = Process(target=consumer, args=(c_queue, p_queue, free_ccm,
semaphore, delim, use_cache))
prod.start()
cons.start()
logger.debug("Waiting to join")
cons.join()
hierarchy = p_queue.get()
prod.join()
return hierarchy
def do_query(next, free_ccm, semaphore, delim, p_queue):
(project, parent_proj) = next
#print 'Querying:', project.get_object_name()
ccm_addr = get_and_lock_free_ccm_addr(free_ccm)
ccm = SynergySession(None, ccm_addr=ccm_addr)
ccm.keep_session_alive = True
result = get_members(project, ccm, parent_proj)
objects = [SynergyObject(o['objectname'], delim) for o in result]
# if a project is being queried it might have more than one dir with the
# same name as the project associated, find the directory that has the
# project associated as the directory's parent
if project.get_type() == 'project':
if len(objects) > 1:
objects = find_root_project(project, objects, ccm)
p_queue.put((project, objects))
free_ccm[ccm_addr] = True
semaphore.release()
def do_query(next_in_queue, free_ccm, semaphore, delim, p_queue, use_cache):
(project, parent_proj) = next_in_queue
ccm_addr, database = get_and_lock_free_ccm_addr(free_ccm)
ccm = SynergySession(database, ccm_addr=ccm_addr)
ccm.keep_session_alive = True
# logger.debug('Querying: %s' % project.get_object_name())
result = get_members(project, ccm, parent_proj)
if use_cache:
objects = []
na_obj = []
for item in result:
try:
objects.append(ccm_cache.get_object(item['objectname'], ccm))
except ccm_cache.ObjectCacheException:
objects.append(SynergyObject(item['objectname'], delim))
na_obj.append(item['objectname'])
if na_obj:
logger.warning("Objects not avaliable in this db:")
logger.warning(', '.join(na_obj))
else:
objects = [SynergyObject(item['objectname'], delim)
for item in result]
# if a project is being queried it might have more than one dir with the
# same name as the project associated, find the directory that has the
# project associated as the directory's parent
if project.get_type() == 'project':
if len(objects) > 1:
objects = find_root_project(project, objects, ccm)
p_queue.put((project, objects))
entry = free_ccm[ccm_addr]
entry['free'] = True
free_ccm[ccm_addr] = entry
semaphore.release()
def get_objects_in_project_serial(project, ccm=None, database=None, use_cache=False):
""" Get all objects and paths of a project """
if not ccm:
if not database:
raise SynergyException("No ccm instance nor database given\n" +
"Cannot start ccm session!\n")
ccm = SynergySession(database)
else:
logger.debug("ccm instance: %s" % ccm.environment['CCM_ADDR'])
delim = ccm.delim()
if use_cache:
start_object = ccm_cache.get_object(project, ccm)
else:
start_object = SynergyObject(project, delim)
queue = deque([start_object])
hierarchy = {}
dir_structure = {}
proj_lookup = {}
cwd = ''
count = 1
hierarchy[start_object.get_object_name()] = [start_object.name]
dir_structure[start_object.get_object_name()] = ''
while queue:
obj = queue.popleft()
#logger.debug('Processing: %s' % obj.get_object_name())
parent_proj = None
if obj.get_type() == 'dir' or obj.get_type() == 'project':
# Processing a dir set 'working dir'
cwd = dir_structure[obj.get_object_name()]
if obj.get_type() == 'dir':
parent_proj = proj_lookup[obj.get_object_name()]
result = get_members(obj, ccm, parent_proj)
if use_cache:
objects = []
na_obj = []
for item in result:
try:
objects.append(ccm_cache.get_object(item['objectname'], ccm))
except ccm_cache.ObjectCacheException:
objects.append(SynergyObject(item['objectname'], ccm.delim()))
na_obj.append(item['objectname'])
if na_obj:
logger.warning("Objects not avaliable in this db:")
logger.warning(', '.join(na_obj))
else:
objects = [SynergyObject(item['objectname'], delim) for item in result]
# if a project is being queried it might have more than one dir with the
# same name as the project associated, find the directory that has the
# project associated as the directory's parent
if obj.get_type() == 'project':
if len(objects) > 1:
objects = find_root_project(obj, objects, ccm)
for synergy_object in objects:
count += 1
if synergy_object.get_type() == 'dir':
# add the directory to the queue and record its parent project
queue.append(synergy_object)
#logger.debug("object: %s child %s cwd %s" % (obj
# .get_object_name(), o.get_object_name(), cwd))
dir_structure[synergy_object.get_object_name()] = \
'%s%s/' % (cwd, synergy_object.get_name())
if obj.get_type() == 'project':
proj_lookup[synergy_object.get_object_name()] = \
obj.get_object_name()
elif obj.get_type() == 'dir':
proj_lookup[synergy_object.get_object_name()] = \
proj_lookup[obj.get_object_name()]
# Also add the directory to the Hierachy to get empty dirs
if synergy_object.get_object_name() in hierarchy.keys():
hierarchy[synergy_object.get_object_name()].append(
'%s%s' % (cwd, synergy_object.get_name()))
else:
hierarchy[synergy_object.get_object_name()] = \
['%s%s' % (cwd, synergy_object.get_name())]
elif synergy_object.get_type() == 'project':
dir_structure[synergy_object.get_object_name()] = cwd
# Add the project to the queue
queue.append(synergy_object)
#logger.debug("object: %s child %s cwd %s" % (obj
# .get_object_name(), o.get_object_name(), cwd))
# Add the project to the hierarchy,
# so the subprojects for the release/project is known
if synergy_object.get_object_name() in hierarchy.keys():
hierarchy[synergy_object.get_object_name()].append(
'%s%s' % (cwd, synergy_object.get_name()))
else:
hierarchy[synergy_object.get_object_name()] = \
['%s%s' % (cwd, synergy_object.get_name())]
else:
# Add the object to the hierarchy
if obj.get_type() == 'dir':
if synergy_object.get_object_name() in hierarchy.keys():
hierarchy[synergy_object.get_object_name()].append(
'%s%s' % (cwd, synergy_object.get_name()))
else:
hierarchy[synergy_object.get_object_name()] = \
['%s%s' % (cwd, synergy_object.get_name())]
#logger.debug("Object: %s has path %s%s" % (o.get_object_name(), cwd, o.get_name()))
logger.debug("Object count: %6d" % count)
return hierarchy
def create_session(database, engine, command_name, ccm_ui_path, ccm_eng_path, offline, i):
ccm = SynergySession(database, engine, command_name, ccm_ui_path, ccm_eng_path, offline)
ccm.keep_session_alive = True
ccm.sessionID = i
return (i,ccm)
def get_objects_in_project_serial(project,
ccm=None,
database=None,
use_cache=False):
""" Get all objects and paths of a project """
if not ccm:
if not database:
raise SynergyException("No ccm instance nor database given\n" +
"Cannot start ccm session!\n")
ccm = SynergySession(database)
else:
logger.debug("ccm instance: %s" % ccm.environment['CCM_ADDR'])
delim = ccm.delim()
if use_cache:
start_object = ccm_cache.get_object(project, ccm)
else:
start_object = SynergyObject(project, delim)
queue = deque([start_object])
hierarchy = {}
dir_structure = {}
proj_lookup = {}
cwd = ''
count = 1
hierarchy[start_object.get_object_name()] = [start_object.name]
dir_structure[start_object.get_object_name()] = ''
while queue:
obj = queue.popleft()
#logger.debug('Processing: %s' % obj.get_object_name())
parent_proj = None
if obj.get_type() == 'dir' or obj.get_type() == 'project':
# Processing a dir set 'working dir'
cwd = dir_structure[obj.get_object_name()]
if obj.get_type() == 'dir':
parent_proj = proj_lookup[obj.get_object_name()]
result = get_members(obj, ccm, parent_proj)
if use_cache:
objects = []
na_obj = []
for item in result:
try:
objects.append(
ccm_cache.get_object(item['objectname'], ccm))
except ccm_cache.ObjectCacheException:
objects.append(
SynergyObject(item['objectname'], ccm.delim()))
na_obj.append(item['objectname'])
if na_obj:
logger.warning("Objects not avaliable in this db:")
logger.warning(', '.join(na_obj))
else:
objects = [
SynergyObject(item['objectname'], delim) for item in result
]
# if a project is being queried it might have more than one dir with the
# same name as the project associated, find the directory that has the
# project associated as the directory's parent
if obj.get_type() == 'project':
if len(objects) > 1:
objects = find_root_project(obj, objects, ccm)
for synergy_object in objects:
count += 1
if synergy_object.get_type() == 'dir':
# add the directory to the queue and record its parent project
queue.append(synergy_object)
#logger.debug("object: %s child %s cwd %s" % (obj
# .get_object_name(), o.get_object_name(), cwd))
dir_structure[synergy_object.get_object_name()] = \
'%s%s/' % (cwd, synergy_object.get_name())
if obj.get_type() == 'project':
proj_lookup[synergy_object.get_object_name()] = \
obj.get_object_name()
elif obj.get_type() == 'dir':
proj_lookup[synergy_object.get_object_name()] = \
proj_lookup[obj.get_object_name()]
# Also add the directory to the Hierachy to get empty dirs
if synergy_object.get_object_name() in hierarchy.keys():
hierarchy[synergy_object.get_object_name()].append(
'%s%s' % (cwd, synergy_object.get_name()))
else:
hierarchy[synergy_object.get_object_name()] = \
['%s%s' % (cwd, synergy_object.get_name())]
elif synergy_object.get_type() == 'project':
dir_structure[synergy_object.get_object_name()] = cwd
# Add the project to the queue
queue.append(synergy_object)
#logger.debug("object: %s child %s cwd %s" % (obj
# .get_object_name(), o.get_object_name(), cwd))
# Add the project to the hierarchy,
# so the subprojects for the release/project is known
if synergy_object.get_object_name() in hierarchy.keys():
hierarchy[synergy_object.get_object_name()].append(
'%s%s' % (cwd, synergy_object.get_name()))
else:
hierarchy[synergy_object.get_object_name()] = \
['%s%s' % (cwd, synergy_object.get_name())]
else:
# Add the object to the hierarchy
if obj.get_type() == 'dir':
if synergy_object.get_object_name() in hierarchy.keys():
hierarchy[synergy_object.get_object_name()].append(
'%s%s' % (cwd, synergy_object.get_name()))
else:
hierarchy[synergy_object.get_object_name()] = \
['%s%s' % (cwd, synergy_object.get_name())]
#logger.debug("Object: %s has path %s%s" % (o.get_object_name(), cwd, o.get_name()))
logger.debug("Object count: %6d" % count)
return hierarchy
def add_project(project, db, commit_msg, parent=None, branch_name=None, path=None):
print "Starting Synergy..."
ccm = SynergySession(db)
if not project_ok(project, ccm):
raise Exception("Error no such Synergy project")
if not git_project_ok():
raise Exception("Git project not ok, cwd needs to be git project root")
cwd = os.getcwd()
if parent:
# if not on the branch already check it out
if not parent == get_current_branch():
command = ['git', 'checkout', '-q', parent]
run_command(command)
else:
#check out first commit and create branch from here
first_commit = run_command(['git', 'log', '--format=%H']).splitlines()[-1]
command = ['git', 'checkout', '-q', first_commit]
run_command(command)
# setup new branch
if branch_name:
if branch_name != get_current_branch():
run_command(['git', 'checkout', '-q', '-b', branch_name])
# setup path to import project into
if path:
if not os.path.exists(path):
os.makedirs(path)
os.chdir(path)
cwd = os.getcwd()
else:
os.chdir(path)
cwd = os.getcwd()
try:
run_command(['git', 'rm', '-q', '-rf', cwd])
except GitException as ex:
if not 'did not match any files' in ex.value:
raise Exception("Error deleting files in {0}".format(cwd))
else:
# delete everything
try:
run_command(['git', 'rm', '-q', '-rf', cwd])
except GitException as ex:
if not 'did not match any files' in ex.value:
raise Exception("Error deleting files in {0}".format(cwd))
print "Creating snapshot of project"
get_snapshot(project, ccm, cwd)
# add everything to git
print "Adding project to git"
run_command(['git', 'add', '.'])
print "Committing..."
out = run_command(['git', 'commit', '-m', '\n'.join(commit_msg)])
print out
if '(no branch)' in get_current_branch():
# detached head state
current_commit = get_current_commit()
#find which branch contains parent commit
if not parent:
parent = get_parent_commit(current_commit)
branches = run_command(['git', 'branch', '--contains', parent]).splitlines()
branch = None
for branch in branches:
if not '(no branch)' in branch:
break
# Checkout branch and fast-forward
out = run_command(['git', 'checkout', '-q', branch.strip()])
print out
out = run_command(['git', 'merge', current_commit])
print out
print "Done"
def start_sessions(config):
ccm = SynergySession(config['database'])
ccm_pool = SynergySessions(database=config['database'],
nr_sessions=config['max_sessions'])
return ccm, ccm_pool
def get_objects_in_project_serial(project, ccm=None, database=None):
if not ccm:
if not database:
raise SynergyException('No ccm instance nor database given\nCannot start ccm session!\n')
ccm = SynergySession(database)
else:
print "ccm instance:", ccm.environment['CCM_ADDR']
delim = ccm.delim()
so = SynergyObject(project, delim)
queue = deque([so])
hierarchy = {}
dir_structure = {}
proj_lookup = {}
cwd = ''
count = 1
hierarchy[so.get_object_name()] = [so.name]
dir_structure[so.get_object_name()] = ''
while queue:
obj = queue.popleft()
#print 'Processing:', obj.get_object_name()
parent_proj = None
if obj.get_type() == 'dir':
# Processing a dir set 'working dir'
cwd = dir_structure[obj.get_object_name()]
parent_proj = proj_lookup[obj.get_object_name()]
result = get_members(obj, ccm, parent_proj)
objects = [SynergyObject(o['objectname'], delim) for o in result]
# if a project is being queried it might have more than one dir with the
# same name as the project associated, find the directory that has the
# project associated as the directory's parent
if obj.get_type() == 'project':
if len(objects) > 1:
objects = find_root_project(obj, objects, ccm)
for o in objects:
count +=1
if o.get_type() == 'dir':
# add the directory to the queue and record its parent project
queue.append(o)
dir_structure[o.get_object_name()] = '%s%s/' % (cwd, o.get_name())
if obj.get_type() == 'project':
proj_lookup[o.get_object_name()] = obj.get_object_name()
elif obj.get_type() == 'dir':
proj_lookup[o.get_object_name()] = proj_lookup[obj.get_object_name()]
# Also add the directory to the Hierachy to get empty dirs
if o.get_object_name() in hierarchy.keys():
hierarchy[o.get_object_name()].append('%s%s' % (cwd, o.get_name()))
else:
hierarchy[o.get_object_name()] = ['%s%s' % (cwd, o.get_name())]
elif o.get_type() == 'project':
# Add the project to the queue
queue.append(o)
# Add the project to the hierarchy, so the subprojects for the release/project is known
if o.get_object_name() in hierarchy.keys():
hierarchy[o.get_object_name()].append('%s%s' % (cwd, o.get_name()))
else:
hierarchy[o.get_object_name()] = ['%s%s' % (cwd, o.get_name())]
else:
# Add the object to the hierarchy
if obj.get_type() == 'dir':
if o.get_object_name() in hierarchy.keys():
hierarchy[o.get_object_name()].append('%s%s' % (cwd, o.get_name()))
else:
hierarchy[o.get_object_name()] = ['%s%s' % (cwd, o.get_name())]
#print "Object:", o.get_object_name(), 'has path:'
#print '%s%s' % (cwd, o.get_name())
print "Object count: %6d" % count
return hierarchy
