def do_query(next_in_queue, free_ccm, semaphore, delim, p_queue, use_cache):
(project, parent_proj) = next_in_queue
ccm_addr, database = get_and_lock_free_ccm_addr(free_ccm)
ccm = SynergySession(database, ccm_addr=ccm_addr)
ccm.keep_session_alive = True
# logger.debug('Querying: %s' % project.get_object_name())
result = get_members(project, ccm, parent_proj)
if use_cache:
objects = []
na_obj = []
for item in result:
try:
objects.append(ccm_cache.get_object(item['objectname'], ccm))
except ccm_cache.ObjectCacheException:
objects.append(SynergyObject(item['objectname'], delim))
na_obj.append(item['objectname'])
if na_obj:
logger.warning("Objects not avaliable in this db:")
logger.warning(', '.join(na_obj))
else:
objects = [SynergyObject(item['objectname'], delim) for item in result]
# if a project is being queried it might have more than one dir with the
# same name as the project associated, find the directory that has the
# project associated as the directory's parent
if project.get_type() == 'project':
if len(objects) > 1:
objects = find_root_project(project, objects, ccm)
p_queue.put((project, objects))
entry = free_ccm[ccm_addr]
entry['free'] = True
free_ccm[ccm_addr] = entry
semaphore.release()
def get_new_projects(old_objects, new_objects, delim):
new_p = []
new_projects = [
SynergyObject(o, delim) for o in new_objects if ':project:' in o
]
old_projects = [
SynergyObject(o, delim) for o in old_objects if ':project:' in o
]
old_projects_names = [
"%s%s:%s:%s" % (o.name, delim, o.type, o.instance)
for o in old_projects
]
for o in new_projects:
threepartname = "%s%s:%s:%s" % (o.name, delim, o.type, o.instance)
if not threepartname in old_projects_names:
new_p.append(o.get_object_name())
return new_p
def get_all_types(ccm):
delim = ccm.delim()
# Query for all types
result = ccm.query("type='attype'").format("%name").format("%version").format("%type").format("%instance").run()
types = [SynergyObject(t["name"] + delim + t["version"] + ":" + t["type"] + ":" + t["instance"], delim) for t in result]
return types
def get_object_from_ccm(four_part_name, ccm, ccm_cache_path):
"""Try to get the object's meta data from Synergy"""
# convert the four-part-name to a synergy object:
delim = ccm.delim()
#dcm_delim = ccm.dcm_delim()
dcm_delim = '#'
synergy_object = SynergyObject(four_part_name, delim, dcm_delim)
try:
res = ccm.query("name='{0}' and version='{1}' and type='{2}' and instance='{3}'".format(synergy_object.get_name(), synergy_object.get_version(), synergy_object.get_type(), synergy_object.get_instance())).format("%objectname").format("%owner").format("%status").format("%create_time").format("%task").run()
except SynergyException:
raise ObjectCacheException("Couldn't query four-part-name of %s from Synergy" % four_part_name)
if res:
synergy_object.status = res[0]['status']
synergy_object.author = res[0]['owner']
synergy_object.created_time = datetime.strptime(res[0]['create_time'], "%d.%m.%y %H:%M")
# synergy_object.created_time = datetime.strptime(res[0]['create_time'], "%a %b %d %H:%M:%S %Y")
tasks = []
# logger.debug(res[0]['task'])
for t in res[0]['task'].split(','):
if t != '<void>':
logger.debug("Adding task %s", t)
if ':task:' not in t:
tasks.append(task_to_four_part(t, delim, dcm_delim))
else:
tasks.append(t)
synergy_object.tasks = tasks
else:
raise ObjectCacheException("Couldn't extract %s's info from Synergy" % four_part_name)
if synergy_object.get_type() == 'project':
object = create_project_object(synergy_object, ccm)
elif synergy_object.get_type() == 'task':
object = create_task_object(synergy_object, ccm)
else:
object = create_file_or_dir_object(synergy_object, ccm)
# Common among all objects
object.predecessors = get_predecessors(object, ccm)
object.successors = get_successors(object, ccm)
attributes = get_non_blacklisted_attributes(object, ccm)
object.set_attributes(attributes)
if object.get_type() == 'dir':
object = fill_changed_entries(object, ccm)
# Add database info to cache object
object.info_databases.append(ccm.get_database_name())
# write the file to the cache
update_cache(object, ccm, ccm_cache_path)
return object
def get_path_of_object_in_release(object, project_paths):
logger.info("Finding path of %s" %object.get_object_name())
for k, v in project_paths.iteritems():
if k.startswith(object.name):
logger.info("Found similar object %s" % k)
# Check if three part name matches:
tmp = SynergyObject(k, object.separator)
if object.name == tmp.name and object.type == tmp.type and object.instance == tmp.instance:
logger.info("Path of %s: %s" % (object.get_object_name(), ','.join(v)))
# Must be this object
return v
logger.info("No path found for %s" % object.get_object_name())
return []
def get_objects_in_project_parallel(project, ccmpool=None, use_cache=False):
""" Get all the objects and paths of project with use of multiple ccm
sessions """
mgr = Manager()
free_ccm = mgr.dict()
for ccm in ccmpool.sessionArray.values():
free_ccm[ccm.getCCM_ADDR()] = {'free': True, 'database': ccm.database}
ccm_addr, database = get_and_lock_free_ccm_addr(free_ccm)
ccm = SynergySession(database, ccm_addr=ccm_addr)
ccm.keep_session_alive = True
delim = ccm.delim()
semaphore = mgr.Semaphore(ccmpool.nr_sessions)
# Starting project
if use_cache:
start_object = ccm_cache.get_object(project, ccm)
else:
start_object = SynergyObject(project, delim)
# unlock update dict entry to inform manager
entry = free_ccm[ccm_addr]
entry['free'] = True
free_ccm[ccm_addr] = entry
p_queue = mgr.Queue()
c_queue = mgr.Queue()
c_queue.put((start_object, None))
p_queue.put(start_object)
# start the produce and consumer thread
prod = Process(target=producer, args=(c_queue, p_queue, free_ccm))
cons = Process(target=consumer,
args=(c_queue, p_queue, free_ccm, semaphore, delim,
use_cache))
prod.start()
cons.start()
logger.debug("Waiting to join")
cons.join()
hierarchy = p_queue.get()
prod.join()
return hierarchy
def get_object_from_ccm(four_part_name, ccm, ccm_cache_path):
"""Try to get the object's meta data from Synergy"""
# convert the four-part-name to a synergy object:
delim = ccm.delim()
synergy_object = SynergyObject(four_part_name, delim)
try:
res = ccm.query("name='{0}' and version='{1}' and type='{2}' and instance='{3}'".format(synergy_object.get_name(), synergy_object.get_version(), synergy_object.get_type(), synergy_object.get_instance())).format("%objectname").format("%owner").format("%status").format("%create_time").format("%task").run()
except SynergyException:
raise ObjectCacheException("Couldn't query four-part-name of %s from Synergy" % four_part_name)
if res:
synergy_object.status = res[0]['status']
synergy_object.author = res[0]['owner']
synergy_object.created_time = datetime.strptime(res[0]['create_time'], "%a %b %d %H:%M:%S %Y")
tasks = []
for t in res[0]['task'].split(','):
if t != '<void>':
if ':task:' not in t:
tasks.append(task_to_four_part(t, delim))
else:
tasks.append(t)
synergy_object.tasks = tasks
else:
raise ObjectCacheException("Couldn't extract %s's info from Synergy" % four_part_name)
if synergy_object.get_type() == 'project':
object = create_project_object(synergy_object, ccm)
elif synergy_object.get_type() == 'task':
object = create_task_object(synergy_object, ccm)
else:
object = create_file_or_dir_object(synergy_object, ccm)
# Common among all objects
object.predecessors = get_predecessors(object, ccm)
object.successors = get_successors(object, ccm)
attributes = get_non_blacklisted_attributes(object, ccm)
object.set_attributes(attributes)
if object.get_type() == 'dir':
object = fill_changed_entries(object, ccm)
# Add database info to cache object
object.info_databases.append(ccm.get_database_name())
# write the file to the cache
update_cache(object, ccm, ccm_cache_path)
return object
def get_objects_in_project_serial(project, ccm=None, database=None, use_cache=False):
""" Get all objects and paths of a project """
if not ccm:
if not database:
raise SynergyException("No ccm instance nor database given\n" +
"Cannot start ccm session!\n")
ccm = SynergySession(database)
else:
logger.debug("ccm instance: %s" % ccm.environment['CCM_ADDR'])
delim = ccm.delim()
if use_cache:
start_object = ccm_cache.get_object(project, ccm)
else:
start_object = SynergyObject(project, delim)
queue = deque([start_object])
hierarchy = {}
dir_structure = {}
proj_lookup = {}
cwd = ''
count = 1
hierarchy[start_object.get_object_name()] = [start_object.name]
dir_structure[start_object.get_object_name()] = ''
while queue:
obj = queue.popleft()
#logger.debug('Processing: %s' % obj.get_object_name())
parent_proj = None
if obj.get_type() == 'dir' or obj.get_type() == 'project':
# Processing a dir set 'working dir'
cwd = dir_structure[obj.get_object_name()]
if obj.get_type() == 'dir':
parent_proj = proj_lookup[obj.get_object_name()]
result = get_members(obj, ccm, parent_proj)
if use_cache:
objects = []
na_obj = []
for item in result:
try:
objects.append(ccm_cache.get_object(item['objectname'], ccm))
except ccm_cache.ObjectCacheException:
objects.append(SynergyObject(item['objectname'], ccm.delim()))
na_obj.append(item['objectname'])
if na_obj:
logger.warning("Objects not avaliable in this db:")
logger.warning(', '.join(na_obj))
else:
objects = [SynergyObject(item['objectname'], delim) for item in result]
# if a project is being queried it might have more than one dir with the
# same name as the project associated, find the directory that has the
# project associated as the directory's parent
if obj.get_type() == 'project':
if len(objects) > 1:
objects = find_root_project(obj, objects, ccm)
for synergy_object in objects:
count += 1
if synergy_object.get_type() == 'dir':
# add the directory to the queue and record its parent project
queue.append(synergy_object)
#logger.debug("object: %s child %s cwd %s" % (obj
# .get_object_name(), o.get_object_name(), cwd))
dir_structure[synergy_object.get_object_name()] = \
'%s%s/' % (cwd, synergy_object.get_name())
if obj.get_type() == 'project':
proj_lookup[synergy_object.get_object_name()] = \
obj.get_object_name()
elif obj.get_type() == 'dir':
proj_lookup[synergy_object.get_object_name()] = \
proj_lookup[obj.get_object_name()]
# Also add the directory to the Hierachy to get empty dirs
if synergy_object.get_object_name() in hierarchy.keys():
hierarchy[synergy_object.get_object_name()].append(
'%s%s' % (cwd, synergy_object.get_name()))
else:
hierarchy[synergy_object.get_object_name()] = \
['%s%s' % (cwd, synergy_object.get_name())]
elif synergy_object.get_type() == 'project':
dir_structure[synergy_object.get_object_name()] = cwd
# Add the project to the queue
queue.append(synergy_object)
#logger.debug("object: %s child %s cwd %s" % (obj
# .get_object_name(), o.get_object_name(), cwd))
# Add the project to the hierarchy,
# so the subprojects for the release/project is known
if synergy_object.get_object_name() in hierarchy.keys():
hierarchy[synergy_object.get_object_name()].append(
'%s%s' % (cwd, synergy_object.get_name()))
else:
hierarchy[synergy_object.get_object_name()] = \
['%s%s' % (cwd, synergy_object.get_name())]
else:
# Add the object to the hierarchy
if obj.get_type() == 'dir':
if synergy_object.get_object_name() in hierarchy.keys():
hierarchy[synergy_object.get_object_name()].append(
'%s%s' % (cwd, synergy_object.get_name()))
else:
hierarchy[synergy_object.get_object_name()] = \
['%s%s' % (cwd, synergy_object.get_name())]
#logger.debug("Object: %s has path %s%s" % (o.get_object_name(), cwd, o.get_name()))
logger.debug("Object count: %6d" % count)
return hierarchy
def get_objects_in_project_serial(project,
ccm=None,
database=None,
use_cache=False):
""" Get all objects and paths of a project """
if not ccm:
if not database:
raise SynergyException("No ccm instance nor database given\n" +
"Cannot start ccm session!\n")
ccm = SynergySession(database)
else:
logger.debug("ccm instance: %s" % ccm.environment['CCM_ADDR'])
delim = ccm.delim()
if use_cache:
start_object = ccm_cache.get_object(project, ccm)
else:
start_object = SynergyObject(project, delim)
queue = deque([start_object])
hierarchy = {}
dir_structure = {}
proj_lookup = {}
cwd = ''
count = 1
hierarchy[start_object.get_object_name()] = [start_object.name]
dir_structure[start_object.get_object_name()] = ''
while queue:
obj = queue.popleft()
#logger.debug('Processing: %s' % obj.get_object_name())
parent_proj = None
if obj.get_type() == 'dir' or obj.get_type() == 'project':
# Processing a dir set 'working dir'
cwd = dir_structure[obj.get_object_name()]
if obj.get_type() == 'dir':
parent_proj = proj_lookup[obj.get_object_name()]
result = get_members(obj, ccm, parent_proj)
if use_cache:
objects = []
na_obj = []
for item in result:
try:
objects.append(
ccm_cache.get_object(item['objectname'], ccm))
except ccm_cache.ObjectCacheException:
objects.append(
SynergyObject(item['objectname'], ccm.delim()))
na_obj.append(item['objectname'])
if na_obj:
logger.warning("Objects not avaliable in this db:")
logger.warning(', '.join(na_obj))
else:
objects = [
SynergyObject(item['objectname'], delim) for item in result
]
# if a project is being queried it might have more than one dir with the
# same name as the project associated, find the directory that has the
# project associated as the directory's parent
if obj.get_type() == 'project':
if len(objects) > 1:
objects = find_root_project(obj, objects, ccm)
for synergy_object in objects:
count += 1
if synergy_object.get_type() == 'dir':
# add the directory to the queue and record its parent project
queue.append(synergy_object)
#logger.debug("object: %s child %s cwd %s" % (obj
# .get_object_name(), o.get_object_name(), cwd))
dir_structure[synergy_object.get_object_name()] = \
'%s%s/' % (cwd, synergy_object.get_name())
if obj.get_type() == 'project':
proj_lookup[synergy_object.get_object_name()] = \
obj.get_object_name()
elif obj.get_type() == 'dir':
proj_lookup[synergy_object.get_object_name()] = \
proj_lookup[obj.get_object_name()]
# Also add the directory to the Hierachy to get empty dirs
if synergy_object.get_object_name() in hierarchy.keys():
hierarchy[synergy_object.get_object_name()].append(
'%s%s' % (cwd, synergy_object.get_name()))
else:
hierarchy[synergy_object.get_object_name()] = \
['%s%s' % (cwd, synergy_object.get_name())]
elif synergy_object.get_type() == 'project':
dir_structure[synergy_object.get_object_name()] = cwd
# Add the project to the queue
queue.append(synergy_object)
#logger.debug("object: %s child %s cwd %s" % (obj
# .get_object_name(), o.get_object_name(), cwd))
# Add the project to the hierarchy,
# so the subprojects for the release/project is known
if synergy_object.get_object_name() in hierarchy.keys():
hierarchy[synergy_object.get_object_name()].append(
'%s%s' % (cwd, synergy_object.get_name()))
else:
hierarchy[synergy_object.get_object_name()] = \
['%s%s' % (cwd, synergy_object.get_name())]
else:
# Add the object to the hierarchy
if obj.get_type() == 'dir':
if synergy_object.get_object_name() in hierarchy.keys():
hierarchy[synergy_object.get_object_name()].append(
'%s%s' % (cwd, synergy_object.get_name()))
else:
hierarchy[synergy_object.get_object_name()] = \
['%s%s' % (cwd, synergy_object.get_name())]
#logger.debug("Object: %s has path %s%s" % (o.get_object_name(), cwd, o.get_name()))
logger.debug("Object count: %6d" % count)
return hierarchy
def get_objects_in_project_serial(project, ccm=None, database=None):
if not ccm:
if not database:
raise SynergyException('No ccm instance nor database given\nCannot start ccm session!\n')
ccm = SynergySession(database)
else:
print "ccm instance:", ccm.environment['CCM_ADDR']
delim = ccm.delim()
so = SynergyObject(project, delim)
queue = deque([so])
hierarchy = {}
dir_structure = {}
proj_lookup = {}
cwd = ''
count = 1
hierarchy[so.get_object_name()] = [so.name]
dir_structure[so.get_object_name()] = ''
while queue:
obj = queue.popleft()
#print 'Processing:', obj.get_object_name()
parent_proj = None
if obj.get_type() == 'dir':
# Processing a dir set 'working dir'
cwd = dir_structure[obj.get_object_name()]
parent_proj = proj_lookup[obj.get_object_name()]
result = get_members(obj, ccm, parent_proj)
objects = [SynergyObject(o['objectname'], delim) for o in result]
# if a project is being queried it might have more than one dir with the
# same name as the project associated, find the directory that has the
# project associated as the directory's parent
if obj.get_type() == 'project':
if len(objects) > 1:
objects = find_root_project(obj, objects, ccm)
for o in objects:
count +=1
if o.get_type() == 'dir':
# add the directory to the queue and record its parent project
queue.append(o)
dir_structure[o.get_object_name()] = '%s%s/' % (cwd, o.get_name())
if obj.get_type() == 'project':
proj_lookup[o.get_object_name()] = obj.get_object_name()
elif obj.get_type() == 'dir':
proj_lookup[o.get_object_name()] = proj_lookup[obj.get_object_name()]
# Also add the directory to the Hierachy to get empty dirs
if o.get_object_name() in hierarchy.keys():
hierarchy[o.get_object_name()].append('%s%s' % (cwd, o.get_name()))
else:
hierarchy[o.get_object_name()] = ['%s%s' % (cwd, o.get_name())]
elif o.get_type() == 'project':
# Add the project to the queue
queue.append(o)
# Add the project to the hierarchy, so the subprojects for the release/project is known
if o.get_object_name() in hierarchy.keys():
hierarchy[o.get_object_name()].append('%s%s' % (cwd, o.get_name()))
else:
hierarchy[o.get_object_name()] = ['%s%s' % (cwd, o.get_name())]
else:
# Add the object to the hierarchy
if obj.get_type() == 'dir':
if o.get_object_name() in hierarchy.keys():
hierarchy[o.get_object_name()].append('%s%s' % (cwd, o.get_name()))
else:
hierarchy[o.get_object_name()] = ['%s%s' % (cwd, o.get_name())]
#print "Object:", o.get_object_name(), 'has path:'
#print '%s%s' % (cwd, o.get_name())
print "Object count: %6d" % count
return hierarchy
