def find_tasks_from_objects(self, objects, project):
tasks = {}
unconfirmed_tasks = {}
if self.tag in self.history.keys():
if 'tasks' in self.history[self.tag]:
for t in self.history[self.tag]['tasks']:
logger.info("loading old task: %s" % t.get_object_name())
tasks[t.get_object_name()] = t
#Build a list of all tasks
task_list = [
task for o in objects
for task in ccm_cache.get_object(o, self.ccm).get_tasks()
]
num_of_tasks = len(set(task_list))
logger.info("Tasks with associated objects: %i" % num_of_tasks)
task_util = TaskUtil(self.ccm)
for t in set(task_list) - set(tasks.keys()):
try:
task = ccm_cache.get_object(t, self.ccm)
except ccm_cache.ObjectCacheException:
continue
if task.status == 'completed':
if task_util.task_in_project(task, project):
tasks[task.get_object_name()] = task
else:
# try to add it anyway to see what happens...
unconfirmed_tasks[task.get_object_name()] = task
# Add objects to tasks
[
t.add_object(o) for o in objects for t in tasks.values()
if t.get_object_name() in ccm_cache.get_object(
o, self.ccm).get_tasks()
]
# Add objects to unconfirmed tasks
[
t.add_object(o) for o in objects
for t in unconfirmed_tasks.values() if t.get_object_name() in
ccm_cache.get_object(o, self.ccm).get_tasks()
]
logger.info("Sanitizing tasks")
tasks = sanitize_tasks(tasks, unconfirmed_tasks)
logger.info("No of tasks in release %s: %i" %
(project.get_object_name(), len(tasks.keys())))
self.history[self.tag]['tasks'] = tasks.values()
fname = self.outputfile + '_' + self.tag + '_inc'
self.persist_data(fname, self.history[self.tag])
def get_objects_in_project(project,
ccm=None,
database=None,
ccmpool=None,
use_cache=False):
""" Get all objects and paths in project
If use_cache is enabled all objects will stored in cache area
"""
start = time.time()
if ccmpool:
if ccmpool.nr_sessions == 1:
result = get_objects_in_project_serial(project,
ccm=ccmpool[0],
database=database,
use_cache=use_cache)
else:
result = get_objects_in_project_parallel(project,
ccmpool=ccmpool,
use_cache=use_cache)
else:
result = get_objects_in_project_serial(project,
ccm=ccm,
database=database,
use_cache=use_cache)
logger.debug("Time used fetching all objects and paths in %s: %d s.",
project,
time.time() - start)
if use_cache:
ccm_object = ccm_cache.get_object(project)
ccm_object.set_members(result)
ccm_cache.force_cache_update_for_object(ccm_object)
return result
def get_master_tag():
f = open('config.p', 'rb')
config = cPickle.load(f)
f.close()
object = ccm_cache.get_object(config['master'])
tag = object.name + object.separator + object.version
return tag
def do_query(next_in_queue, free_ccm, semaphore, delim, p_queue, use_cache):
(project, parent_proj) = next_in_queue
ccm_addr, database = get_and_lock_free_ccm_addr(free_ccm)
ccm = SynergySession(database, ccm_addr=ccm_addr)
ccm.keep_session_alive = True
# logger.debug('Querying: %s' % project.get_object_name())
result = get_members(project, ccm, parent_proj)
if use_cache:
objects = []
na_obj = []
for item in result:
try:
objects.append(ccm_cache.get_object(item['objectname'], ccm))
except ccm_cache.ObjectCacheException:
objects.append(SynergyObject(item['objectname'], delim))
na_obj.append(item['objectname'])
if na_obj:
logger.warning("Objects not avaliable in this db:")
logger.warning(', '.join(na_obj))
else:
objects = [SynergyObject(item['objectname'], delim) for item in result]
# if a project is being queried it might have more than one dir with the
# same name as the project associated, find the directory that has the
# project associated as the directory's parent
if project.get_type() == 'project':
if len(objects) > 1:
objects = find_root_project(project, objects, ccm)
p_queue.put((project, objects))
entry = free_ccm[ccm_addr]
entry['free'] = True
free_ccm[ccm_addr] = entry
semaphore.release()
def successor_is_released(self, predecessor, fileobject, recursion_depth):
if recursion_depth > self.max_recursion_depth:
return False
recursion_depth += 1
logger.info("Checking if successor is released, for %s by predecessor %s" %(fileobject.get_object_name(), predecessor.get_object_name()))
ret_val = False
successors = predecessor.get_successors()
for s in successors:
if s in self.release_lookup.keys():
return self.release_lookup[s]
if s != fileobject.get_object_name():
try:
successor = ccm_cache.get_object(s, self.ccm)
except ccm_cache.ObjectCacheException as e:
logger.warning(e.value)
return False
logger.info("successor: %s" % successor.get_object_name())
#check releases of successor
releases = successor.get_releases()
if [r for r in releases if r in self.old_subproject_list]:
logger.info("successor: %s is released" % successor.get_object_name())
self.release_lookup[successor.get_object_name()] = True
return True
elif [r for r in releases if r in self.current_subproject_list]:
logger.info("successor: %s is released in current project, don't continue" % successor.get_object_name())
self.release_lookup[successor.get_object_name()] = False
return False
else:
ret_val = self.successor_is_released(successor, fileobject, recursion_depth)
self.release_lookup[successor.get_object_name()] = ret_val
else:
# if there is only one successor and it is the fileobject assume it to be released if the predecessor is in released state
if len(successors) == 1 and predecessor.get_status() == 'released':
return True
return ret_val
def get_master_tag():
f = open('config.p', 'rb')
config = cPickle.load(f)
f.close()
object = ccm_cache.get_object(config['master'])
tag = object.name + object.separator + object.version
return tag
def create_release_graph(objects, release, previous):
release_graph = hypergraph()
release_graph.add_nodes(objects)
release_graph.add_edges([release, previous])
file_objects = [ccm_cache.get_object(o) for o in objects]
for o in file_objects:
link = True
# Bind objects to this release
successors = o.get_successors()
if successors:
for s in successors:
if s not in release_graph.nodes():
link &= True
else:
link &= False
if link:
if o.get_object_name() not in release_graph.links(release):
release_graph.link(o.get_object_name(), release)
# Bind objects to previous release
predecessors = o.get_predecessors()
if predecessors is not None:
for p in predecessors:
if p not in objects:
if not release_graph.has_node(p):
release_graph.add_node(p)
#print "linking", p, "to release", previous
release_graph.link(p, previous)
return release_graph
def update_task_in_history_with_objects(self, task, objects):
# Find the task in history if its there
found = False
for t in self.history[self.tag]['tasks']:
if t.get_object_name() == task:
if t.objects:
t.objects.extend(
[o for o in objects if ':project:' not in o])
t.objects = list(set(t.objects))
else:
t.objects = [o for o in objects if ':project:' not in o]
found = True
if not found:
t = ccm_cache.get_object(task, self.ccm)
if t.type == 'task':
t.objects = [o for o in objects if ':project:' not in o]
self.history[self.tag]['tasks'].append(t)
elif t.type == 'dir':
# Make a task from the directory object
task_obj = TaskObject(
"%s%s%s:task:%s" %
(t.name, self.delim, t.version, t.instance), self.delim,
t.author, t.status, t.created_time, t.tasks)
task_obj.set_attributes(t.attributes)
task_obj.complete_time = t.get_integrate_time()
task_obj.objects = [o for o in objects if ':project:' not in o]
self.history[self.tag]['tasks'].append(task_obj)
def create_release_graph(objects, release, previous):
release_graph = hypergraph()
release_graph.add_nodes(objects)
release_graph.add_edges([release, previous])
file_objects = [ccm_cache.get_object(o) for o in objects]
for o in file_objects:
link = True
# Bind objects to this release
successors = o.get_successors()
if successors:
for s in successors:
if s not in release_graph.nodes():
link &= True
else:
link &=False
if link:
if o.get_object_name() not in release_graph.links(release):
release_graph.link(o.get_object_name(), release)
# Bind objects to previous release
predecessors = o.get_predecessors()
if predecessors is not None:
for p in predecessors:
if p not in objects:
if not release_graph.has_node(p):
release_graph.add_node(p)
#print "linking", p, "to release", previous
release_graph.link(p, previous)
return release_graph
def populate_cache_with_project_and_members(project, ccm, ccmpool):
print "Loading object %s" % project
project_obj = ccm_cache.get_object(project, ccm=ccm)
#assuming no project.members
objects_in_project = ccm_objects_in_project.get_objects_in_project(project, ccm=ccm, ccmpool=ccmpool, use_cache=True)
project_obj.set_members(objects_in_project)
ccm_cache.force_cache_update_for_object(project_obj)
def project_is_some_predecessor(self, project, recursion_depth):
if recursion_depth > self.max_recursion_depth:
return False
recursion_depth += 1
logger.info("Checking if %s is some predecessor of %s or %s " %
(project.get_object_name(), self.current_release,
self.old_release))
successors = project.get_baseline_successor()
for successor in successors:
logger.info("successor: %s" % successor)
if successor in self.old_subproject_list:
logger.info("Found %s in previous subprojects" % successor)
return True
elif successor in self.current_subproject_list:
logger.info("Found %s in current subprojects" % successor)
return True
else:
try:
successor = ccm_cache.get_object(successor, self.ccm)
except ccm_cache.ObjectCacheException:
return False
if self.project_is_some_predecessor(successor,
recursion_depth):
return True
return False
def update_project_with_members(project, ccm, ccmpool):
print "Loading object %s" % project
project_obj = ccm_cache.get_object(project, ccm=ccm)
if not project_obj.members:
objects_in_project = ccm_objects_in_project.get_objects_in_project(project, ccm=ccm, ccmpool=ccmpool)
project_obj.set_members(objects_in_project)
ccm_cache.force_cache_update_for_object(project_obj)
def populate_cache_with_project_and_members(project, ccm, ccmpool):
print "Loading object %s" % project
project_obj = ccm_cache.get_object(project, ccm=ccm)
#assuming no project.members
objects_in_project = ccm_objects_in_project.get_objects_in_project(
project, ccm=ccm, ccmpool=ccmpool, use_cache=True)
project_obj.set_members(objects_in_project)
ccm_cache.force_cache_update_for_object(project_obj)
def update_project_with_members(project, ccm, ccmpool):
print "Loading object %s" % project
project_obj = ccm_cache.get_object(project, ccm=ccm)
if not project_obj.members:
objects_in_project = ccm_objects_in_project.get_objects_in_project(
project, ccm=ccm, ccmpool=ccmpool)
project_obj.set_members(objects_in_project)
ccm_cache.force_cache_update_for_object(project_obj)
def get_project_chain(head_project, base_project, ccm):
# Do it from head to base
baseline = ccm_cache.get_object(head_project, ccm)
chain = [baseline.get_object_name()]
while baseline.get_object_name() != base_project:
predecessor = ccm_cache.get_object(baseline.predecessors[0], ccm)
baseline = ccm_cache.get_object(baseline.baseline_predecessor[0], ccm)
if baseline:
chain.append(baseline.get_object_name())
elif predecessor:
chain.append(predecessor.get_object_name())
baseline = predecessor
else:
break
# reverse the list to get the base first
chain.reverse()
return chain
def get_objects_from_graph(task, graph, objects):
objs = []
for o in graph.links(task):
for obj in objects:
if obj == o:
objs.append(obj)
break
return [ccm_cache.get_object(o) for o in objs]
def get_objects_from_graph(task, graph, objects):
objs = []
for o in graph.links(task):
for obj in objects:
if obj == o:
objs.append(obj)
break
return [ccm_cache.get_object(o) for o in objs]
def get_project_chain(head_project, base_project, ccm):
# Do it from head to base
baseline = ccm_cache.get_object(head_project, ccm)
chain = [baseline.get_object_name()]
while baseline.get_object_name() != base_project:
if baseline.predecessors:
predecessor = ccm_cache.get_object(baseline.predecessors[0], ccm)
if baseline.baseline_predecessor:
baseline = ccm_cache.get_object(baseline.baseline_predecessor[0],
ccm)
if baseline:
chain.append(baseline.get_object_name())
elif predecessor:
chain.append(predecessor.get_object_name())
baseline = predecessor
else:
break
# reverse the list to get the base first
chain.reverse()
return chain
def populate_cache_with_objects_from_project(project, ccm, ccmpool):
print "processing project %s" %project
#first try to get the object from cache
project_obj = ccm_cache.get_object(project, ccm)
if not project_obj.members:
update_project_with_members(project, ccm, ccmpool)
na_obj =[]
if project_obj.members:
num_o = len(project_obj.members.keys())
for o in project_obj.members.keys():
print "loading object: %s" % o
try:
obj = ccm_cache.get_object(o, ccm)
except ccm_cache.ObjectCacheException:
na_obj.append(o)
num_o -=1
print "objects left %d" %num_o
print "%s done, members: %d" %(project, len(project_obj.members.keys()))
print "Objects not avaliable in this db:"
print '\n'.join(na_obj)
def get_new_dirs(self, members, new_objects):
new_directories = [d for d in new_objects.keys() if ':dir:' in d]
directories = []
for dir in new_directories:
dir_obj = ccm_cache.get_object(dir, self.ccm)
new_dirs = [d for d in dir_obj.new_objects if d.endswith('/')]
for d in new_dirs:
# Get the corresponding directory four-part-name
paths = members[dir]
name = get_dir_with_path(paths, d, members)
directories.append(name)
print "new directories found:"
print '\n'.join([d + ', '.join(members[d]) for d in directories])
return directories
def find_tasks_from_objects(self, objects, project):
tasks = {}
unconfirmed_tasks = {}
if self.tag in self.history.keys():
if 'tasks' in self.history[self.tag]:
for t in self.history[self.tag]['tasks']:
logger.info("loading old task: %s" % t.get_object_name())
tasks[t.get_object_name()] = t
#Build a list of all tasks
task_list = [task for o in objects for task in ccm_cache.get_object(o, self.ccm).get_tasks()]
num_of_tasks = len(set(task_list))
logger.info("Tasks with associated objects: %i" % num_of_tasks)
task_util = TaskUtil(self.ccm)
for t in set(task_list)-set(tasks.keys()):
try:
task = ccm_cache.get_object(t, self.ccm)
except ccm_cache.ObjectCacheException:
continue
if task.status == 'completed':
if task_util.task_in_project(task, project):
tasks[task.get_object_name()] = task
else:
# try to add it anyway to see what happens...
unconfirmed_tasks[task.get_object_name()] = task
# Add objects to tasks
[t.add_object(o) for o in objects for t in tasks.values() if t.get_object_name() in ccm_cache.get_object(o, self.ccm).get_tasks()]
# Add objects to unconfirmed tasks
[t.add_object(o) for o in objects for t in unconfirmed_tasks.values() if t.get_object_name() in ccm_cache.get_object(o, self.ccm).get_tasks()]
logger.info("Sanitizing tasks")
tasks = sanitize_tasks(tasks, unconfirmed_tasks)
logger.info("No of tasks in release %s: %i" % (project.get_object_name(), len(tasks.keys())))
self.history[self.tag]['tasks'] = tasks.values()
fname = self.outputfile + '_' + self.tag + '_inc'
self.persist_data(fname, self.history[self.tag])
def find_task_from_dirs(self, dirs):
tasks = {}
for dir in dirs:
logger.info("Finding directory %s in tasks" % dir)
# Try to get the task from the tasks already found
obj = ccm_cache.get_object(dir, self.ccm)
tmp_task = obj.get_tasks()
task = list(set(tmp_task) & set([t.get_object_name() for t in self.history[self.tag]['tasks']]))
if not task:
# Use object name as task name
task = [dir]
logger.info("task found: %s" % ','.join(task))
tasks[dir]= task
return tasks
def get_new_dirs(self, members, new_objects):
new_directories = [d for d in new_objects.keys() if ':dir:' in d]
directories = []
for dir in new_directories:
dir_obj = ccm_cache.get_object(dir, self.ccm)
new_dirs = [d for d in dir_obj.new_objects if d.endswith('/')]
for d in new_dirs:
# Get the corresponding directory four-part-name
paths = members[dir]
name = get_dir_with_path(paths, d, members)
if name:
directories.append(name)
logger.info("new directories found:")
logger.info(' ,'.join([d + ', '.join(members[d])
for d in directories]))
return directories
def create_file_list(objects, lookup, ccm_types, project, empty_dirs=None, empty_dir_mark=None, all_files_for_release=False):
global object_mark_lookup
project_object = ccm_cache.get_object(project)
paths = project_object.get_members()
l = []
for o in objects:
if o.get_type() != 'dir':
perm = ccm_types[o.get_type()]
object_paths = get_object_paths(o, paths)
for p in object_paths:
l.append('M ' + perm + ' :' + str(lookup[o.get_object_name()]) + ' ' + p)
else:
#Get deleted items
deleted = o.get_deleted_objects()
if deleted:
for d in deleted:
object_paths = get_object_paths(o, paths)
for p in object_paths:
# p is the path of the directory
if d.endswith('/'):
tmp = d.rsplit('/', 1)[0]
else:
tmp = d
l.append('D ' + p + '/' + tmp)
if all_files_for_release:
# delete top level dir first:
l.append('D ' + project_object.name)
# Insert all files in the release
logger.info("Loading all objects for %s" %project_object.get_object_name())
for object, paths in project_object.members.iteritems():
if not ':dir:' in object and not ':project:' in object:
perm = ccm_types[object.split(':')[1]]
for p in paths:
l.append('M ' + perm + ' :' + str(object_mark_lookup[object]) + ' ' + p)
if empty_dirs:
for d in empty_dirs:
if empty_dir_mark:
path = d + '/.gitignore'
l.append('M 100644 :' + str(empty_dir_mark) + ' ' + path)
if not l:
return []
return '\n'.join(l)
def successor_is_released(self, predecessor, fileobject, recursion_depth):
if recursion_depth > self.max_recursion_depth:
return False
recursion_depth += 1
logger.info(
"Checking if successor is released, for %s by predecessor %s" %
(fileobject.get_object_name(), predecessor.get_object_name()))
ret_val = False
successors = predecessor.get_successors()
for s in successors:
if s in self.release_lookup.keys():
return self.release_lookup[s]
if s != fileobject.get_object_name():
try:
successor = ccm_cache.get_object(s, self.ccm)
except ccm_cache.ObjectCacheException as e:
logger.warning(e.value)
return False
logger.info("successor: %s" % successor.get_object_name())
#check releases of successor
releases = successor.get_releases()
if [r for r in releases if r in self.old_subproject_list]:
logger.info("successor: %s is released" %
successor.get_object_name())
self.release_lookup[successor.get_object_name()] = True
return True
elif [
r for r in releases
if r in self.current_subproject_list
]:
logger.info(
"successor: %s is released in current project, don't continue"
% successor.get_object_name())
self.release_lookup[successor.get_object_name()] = False
return False
else:
ret_val = self.successor_is_released(
successor, fileobject, recursion_depth)
self.release_lookup[successor.get_object_name()] = ret_val
else:
# if there is only one successor and it is the fileobject assume it to be released if the predecessor is in released state
if len(successors) == 1 and predecessor.get_status(
) == 'released':
return True
return ret_val
def create_file_list(objects, lookup, ccm_types, project, empty_dirs=None, empty_dir_mark=None, all_files_for_release=False):
global object_mark_lookup
project_object = ccm_cache.get_object(project)
paths = project_object.get_members()
l = []
for o in objects:
if o.get_type() != 'dir':
perm = ccm_types[o.get_type()]
object_paths = get_object_paths(o, paths)
for p in object_paths:
l.append('M ' + perm + ' :' + str(lookup[o.get_object_name()]) + ' ' + p)
else:
#Get deleted items
deleted = o.get_deleted_objects()
if deleted:
for d in deleted:
object_paths = get_object_paths(o, paths)
for p in object_paths:
# p is the path of the directory
if d.endswith('/'):
tmp = d.rsplit('/', 1)[0]
else:
tmp = d
l.append('D ' + p + '/' + tmp)
if all_files_for_release:
# delete top level dir first:
l.append('D ' + project_object.name)
# Insert all files in the release
logger.info("Loading all objects for %s" %project_object.get_object_name())
for object, paths in project_object.members.iteritems():
if not ':dir:' in object and not ':project:' in object:
perm = ccm_types[object.split(':')[1]]
for p in paths:
l.append('M ' + perm + ' :' + str(object_mark_lookup[object]) + ' ' + p)
if empty_dirs:
for d in empty_dirs:
if empty_dir_mark:
path = d + '/.gitignore'
l.append('M 100644 :' + str(empty_dir_mark) + ' ' + path)
if not l:
return []
return '\n'.join(l)
def get_objects_in_project_parallel(project, ccmpool=None, use_cache=False):
""" Get all the objects and paths of project with use of multiple ccm
sessions """
mgr = Manager()
free_ccm = mgr.dict()
for ccm in ccmpool.sessionArray.values():
free_ccm[ccm.getCCM_ADDR()] = {'free': True, 'database': ccm.database}
ccm_addr, database = get_and_lock_free_ccm_addr(free_ccm)
ccm = SynergySession(database, ccm_addr=ccm_addr)
ccm.keep_session_alive = True
delim = ccm.delim()
semaphore = mgr.Semaphore(ccmpool.nr_sessions)
# Starting project
if use_cache:
start_object = ccm_cache.get_object(project, ccm)
else:
start_object = SynergyObject(project, delim)
# unlock update dict entry to inform manager
entry = free_ccm[ccm_addr]
entry['free'] = True
free_ccm[ccm_addr] = entry
p_queue = mgr.Queue()
c_queue = mgr.Queue()
c_queue.put((start_object, None))
p_queue.put(start_object)
# start the produce and consumer thread
prod = Process(target=producer, args=(c_queue, p_queue, free_ccm))
cons = Process(target=consumer,
args=(c_queue, p_queue, free_ccm, semaphore, delim,
use_cache))
prod.start()
cons.start()
logger.debug("Waiting to join")
cons.join()
hierarchy = p_queue.get()
prod.join()
return hierarchy
def get_objects_in_project_parallel(project, ccmpool=None, use_cache=False):
""" Get all the objects and paths of project with use of multiple ccm
sessions """
mgr = Manager()
free_ccm = mgr.dict()
for ccm in ccmpool.sessionArray.values():
free_ccm[ccm.getCCM_ADDR()] = {'free': True, 'database': ccm.database}
ccm_addr, database = get_and_lock_free_ccm_addr(free_ccm)
ccm = SynergySession(database, ccm_addr=ccm_addr)
ccm.keep_session_alive = True
delim = ccm.delim()
semaphore = mgr.Semaphore(ccmpool.nr_sessions)
# Starting project
if use_cache:
start_object = ccm_cache.get_object(project, ccm)
else:
start_object = SynergyObject(project, delim)
# unlock update dict entry to inform manager
entry = free_ccm[ccm_addr]
entry['free'] = True
free_ccm[ccm_addr] = entry
p_queue = mgr.Queue()
c_queue = mgr.Queue()
c_queue.put((start_object, None))
p_queue.put(start_object)
# start the produce and consumer thread
prod = Process(target=producer, args=(c_queue, p_queue, free_ccm))
cons = Process(target=consumer, args=(c_queue, p_queue, free_ccm,
semaphore, delim, use_cache))
prod.start()
cons.start()
logger.debug("Waiting to join")
cons.join()
hierarchy = p_queue.get()
prod.join()
return hierarchy
def get_objects_in_project(project, ccm=None, database=None, ccmpool=None, use_cache=False):
""" Get all objects and paths in project
If use_cache is enabled all objects will stored in cache area
"""
start = time.time()
if ccmpool:
if ccmpool.nr_sessions == 1:
result = get_objects_in_project_serial(project, ccm=ccmpool[0], database=database, use_cache=use_cache)
else:
result = get_objects_in_project_parallel(project, ccmpool=ccmpool, use_cache=use_cache)
else:
result = get_objects_in_project_serial(project, ccm=ccm, database=database, use_cache=use_cache)
logger.debug("Time used fetching all objects and paths in %s: %d s.",
project, time.time() - start)
if use_cache:
ccm_object = ccm_cache.get_object(project)
ccm_object.set_members(result)
ccm_cache.force_cache_update_for_object(ccm_object)
return result
def check_successor_chain_for_object(self, fileobject, old_object, recursion_depth):
if recursion_depth > self.max_recursion_depth:
return False
recursion_depth += 1
logger.info("Checking if successor chain for %s contains %s" % (fileobject.get_object_name(), old_object.get_object_name()))
ret_val = False
successors = fileobject.get_successors()
for s in successors:
if s == old_object.get_object_name():
return True
try:
successor = ccm_cache.get_object(s, self.ccm)
except ccm_cache.ObjectCacheException:
return False
logger.info("successor: %s" %successor.get_object_name())
ret_val = self.check_successor_chain_for_object(successor, old_object, recursion_depth)
if ret_val:
break
return ret_val
def find_task_from_dirs(self, dirs):
tasks = {}
for dir in dirs:
logger.info("Finding directory %s in tasks" % dir)
# Try to get the task from the tasks already found
obj = ccm_cache.get_object(dir, self.ccm)
tmp_task = obj.get_tasks()
task = list(
set(tmp_task) & set([
t.get_object_name()
for t in self.history[self.tag]['tasks']
]))
if not task:
# Use object name as task name
task = [dir]
logger.info("task found: %s" % ','.join(task))
tasks[dir] = task
return tasks
def project_is_some_predecessor(self, project, recursion_depth):
if recursion_depth > self.max_recursion_depth:
return False
recursion_depth += 1
logger.info("Checking if %s is some predecessor of %s or %s " %(project.get_object_name(), self.current_release, self.old_release))
successors = project.get_baseline_successor()
for successor in successors:
logger.info("successor: %s" % successor)
if successor in self.old_subproject_list:
logger.info("Found %s in previous subprojects"% successor)
return True
elif successor in self.current_subproject_list:
logger.info("Found %s in current subprojects" % successor)
return True
else:
try:
successor = ccm_cache.get_object(successor, self.ccm)
except ccm_cache.ObjectCacheException:
return False
if self.project_is_some_predecessor(successor, recursion_depth):
return True
return False
def project_is_some_predecessor(self, project, recursion_depth):
if recursion_depth > self.max_recursion_depth:
return False
recursion_depth += 1
print "Checking if", project.get_object_name(), "is some predecessor of", self.current_release, "or", self.old_release, "..."
successors = project.get_baseline_successor()
for successor in successors:
print "successor:", successor
if successor in self.old_subproject_list:
print "Found", successor, "in previous subprojects"
return True
elif successor in self.current_subproject_list:
print "Found", successor, "in current subprojects"
return True
else:
try:
successor = ccm_cache.get_object(successor, self.ccm)
except ccm_cache.ObjectCacheException:
return False
if self.project_is_some_predecessor(successor, recursion_depth):
return True
return False
def update_task_in_history_with_objects(self, task, objects):
# Find the task in history if its there
found = False
for t in self.history[self.tag]['tasks']:
if t.get_object_name() == task:
if t.objects:
t.objects.extend([o for o in objects if ':project:' not in o])
t.objects = list(set(t.objects))
else:
t.objects = [o for o in objects if ':project:' not in o]
found = True
if not found:
t = ccm_cache.get_object(task, self.ccm)
if t.type == 'task':
t.objects = [o for o in objects if ':project:' not in o]
self.history[self.tag]['tasks'].append(t)
elif t.type == 'dir':
# Make a task from the directory object
task_obj = TaskObject("%s%s%s:task:%s" %(t.name, self.delim, t.version, t.instance), self.delim, t.author, t.status, t.created_time, t.tasks)
task_obj.set_attributes(t.attributes)
task_obj.complete_time = t.get_integrate_time()
task_obj.objects = [o for o in objects if ':project:' not in o]
self.history[self.tag]['tasks'].append(task_obj)
def check_successor_chain_for_object(self, fileobject, old_object,
recursion_depth):
if recursion_depth > self.max_recursion_depth:
return False
recursion_depth += 1
logger.info(
"Checking if successor chain for %s contains %s" %
(fileobject.get_object_name(), old_object.get_object_name()))
ret_val = False
successors = fileobject.get_successors()
for s in successors:
if s == old_object.get_object_name():
return True
try:
successor = ccm_cache.get_object(s, self.ccm)
except ccm_cache.ObjectCacheException:
return False
logger.info("successor: %s" % successor.get_object_name())
ret_val = self.check_successor_chain_for_object(
successor, old_object, recursion_depth)
if ret_val:
break
return ret_val
def create_object_graph(objects):
object_graph = digraph()
object_graph.add_nodes(objects)
file_objects = []
for o in objects:
file_objects.append(ccm_cache.get_object(o))
for o in file_objects:
for p in o.get_predecessors():
if not object_graph.has_node(p):
object_graph.add_node(p)
# Create relationship list
for o in file_objects:
# Bind objects to previous release
predecessors = o.get_predecessors()
if predecessors:
for p in predecessors:
object_graph.add_edge((p, o.get_object_name()))
return object_graph
def create_object_graph(objects):
object_graph = digraph()
object_graph.add_nodes(objects)
file_objects = []
for o in objects:
file_objects.append(ccm_cache.get_object(o))
for o in file_objects:
for p in o.get_predecessors():
if not object_graph.has_node(p):
object_graph.add_node(p)
# Create relationship list
for o in file_objects:
# Bind objects to previous release
predecessors = o.get_predecessors()
if predecessors:
for p in predecessors:
object_graph.add_edge((p,o.get_object_name()))
return object_graph
def do_query(next_in_queue, free_ccm, semaphore, delim, p_queue, use_cache):
(project, parent_proj) = next_in_queue
ccm_addr, database = get_and_lock_free_ccm_addr(free_ccm)
ccm = SynergySession(database, ccm_addr=ccm_addr)
ccm.keep_session_alive = True
# logger.debug('Querying: %s' % project.get_object_name())
result = get_members(project, ccm, parent_proj)
if use_cache:
objects = []
na_obj = []
for item in result:
try:
objects.append(ccm_cache.get_object(item['objectname'], ccm))
except ccm_cache.ObjectCacheException:
objects.append(SynergyObject(item['objectname'], delim))
na_obj.append(item['objectname'])
if na_obj:
logger.warning("Objects not avaliable in this db:")
logger.warning(', '.join(na_obj))
else:
objects = [SynergyObject(item['objectname'], delim)
for item in result]
# if a project is being queried it might have more than one dir with the
# same name as the project associated, find the directory that has the
# project associated as the directory's parent
if project.get_type() == 'project':
if len(objects) > 1:
objects = find_root_project(project, objects, ccm)
p_queue.put((project, objects))
entry = free_ccm[ccm_addr]
entry['free'] = True
free_ccm[ccm_addr] = entry
semaphore.release()
def populate_cache_with_objects_from_project(project, ccm, ccmpool):
print "processing project %s" % project
#first try to get the object from cache
project_obj = ccm_cache.get_object(project, ccm)
if not project_obj.members:
populate_cache_with_project_and_members(project, ccm, ccmpool)
def ccm_fast_export(releases, graphs):
global acn_ancestors
global users
users = users()
logger.basicConfig(filename='ccm_fast_export.log',level=logger.DEBUG)
commit_lookup = {}
# Get the initial release
for k, v in releases.iteritems():
if k == 'delimiter':
continue
if k == 'ccm_types':
continue
if v['previous'] is None:
release = k
break
logger.info("Starting at %s as initial release" % release)
if 'created' not in releases[release]:
initial_release_time = 0.0 # epoch for now since releases[release] has no 'created' key :(
else:
initial_release_time = time.mktime(releases[release]['created'].timetuple())
mark = 0
files = []
# Create the initial release
# get all the file objects:
file_objects = (ccm_cache.get_object(o) for o in releases[release]['objects'])
project_obj = ccm_cache.get_object(releases[release]['fourpartname'])
paths = project_obj.get_members()
for o in file_objects:
if o.get_type() != 'dir':
object_mark, mark = create_blob(o, mark)
for p in paths[o.get_object_name()]:
files.append('M ' + releases['ccm_types']['permissions'][o.get_type()] + ' :'+str(object_mark) + ' ' + p)
empty_dirs = releases[release]['empty_dirs']
logger.info("Empty dirs for release %s\n%s" %(release, empty_dirs))
mark = create_blob_for_empty_dir(get_mark(mark))
#file_list = create_file_list(objects, object_lookup, releases['ccm_types'], empty_dirs=empty_dirs, empty_dir_mark=mark)
if empty_dirs:
for d in empty_dirs:
if mark:
path = d + '/.gitignore'
files.append('M 100644 :' + str(mark) + ' ' + path)
mark = get_mark(mark)
commit_info = ['reset refs/tags/' + release, 'commit refs/tags/' + release, 'mark :' + str(mark),
'author Nokia <*****@*****.**> ' + str(int(initial_release_time)) + " +0000",
'committer Nokia <*****@*****.**> ' + str(int(initial_release_time)) + " +0000", 'data 15',
'Initial commit', '\n'.join(files), '']
print '\n'.join(commit_info)
logger.info("git-fast-import:\n%s" %('\n'.join(commit_info)))
tag_msg = 'Release: %s' %release
annotated_tag = ['tag %s' % release,
'from :%s' % str(mark),
'tagger Nokia <*****@*****.**> ' + str(int(initial_release_time)) + " +0000",
'data %s' % len(tag_msg),
tag_msg]
print '\n'.join(annotated_tag)
commit_lookup[release] = mark
# do the following releases (graphs)
release_queue = deque(releases[release]['next'])
while release_queue:
release = release_queue.popleft()
previous_release = releases[release]['previous']
logger.info("Next release: %s" % release)
commit_graph = graphs[release]['commit']
commit_graph = fix_orphan_nodes(commit_graph, previous_release)
commit_graph = ch.spaghettify_digraph(commit_graph, previous_release, release)
#htg.commit_graph_to_image(commit_graph, releases[release], graphs[release]['task'], name=releases[release]['name']+'_after' )
# Find the cutting nodes
logger.info("Finding the cutting nodes")
undirected = graph()
undirected.add_nodes(commit_graph.nodes())
[undirected.add_edge(edge) for edge in commit_graph.edges()]
cutting_nodes = cut_nodes(undirected)
del undirected
# Create the reverse commit graph
logger.info("Building the reverse commit graph")
reverse_commit_graph = commit_graph.reverse()
# Compute the accessibility matrix of the reverse commit graph
logger.info("Compute the ancestors")
ancestors = accessibility(reverse_commit_graph)
del reverse_commit_graph
logger.info("Ancestors of the release: %s" % str(ancestors[release]))
# Clean up the ancestors matrix
for k, v in ancestors.iteritems():
if k in v:
v.remove(k)
# Get the commits order
commits = topological_sorting(commit_graph)
# Fix the commits order list
commits.remove(previous_release)
commits.remove(release)
last_cutting_node = None
# Check if the release (Synergy project has changed name, if it has the
# 'base' directory name needs to be renamed
if releases.has_key('delimiter'):
delim = releases['delimiter']
else:
delim = '-'
previous_name = previous_release.split(delim)[0]
current_name = release.split(delim)[0]
if current_name != previous_name:
logger.info("Name changed: %s -> %s" %(previous_name, current_name))
from_mark = commit_lookup[previous_release]
mark, commit = rename_toplevel_dir(previous_name, current_name, release, releases, mark, from_mark)
print '\n'.join(commit)
# adjust the commit lookup
commit_lookup[previous_release] = mark
for counter, commit in enumerate(commits):
logger.info("Commit %i/%i" % (counter+1, len(commits)))
acn_ancestors = []
if last_cutting_node is not None:
acn_ancestors = ancestors[last_cutting_node]
# Create the references lists. It lists the parents of the commit
#reference = [commit_lookup[parent] for parent in ancestors[commit] if parent not in acn_ancestors]
reference = [commit_lookup[parent] for parent in commit_graph.incidents(commit)]
if len(reference) > 1:
# Merge commit
mark = create_merge_commit(commit, release, releases, mark, reference, graphs, set(ancestors[commit]) - set(acn_ancestors))
else:
# Normal commit
mark = create_commit(commit, release, releases, mark, reference, graphs)
# Update the lookup table
commit_lookup[commit] = mark
# Update the last cutting edge if necessary
if commit in cutting_nodes:
last_cutting_node = commit
if last_cutting_node is not None:
acn_ancestors = ancestors[last_cutting_node]
reference = [commit_lookup[parent] for parent in ancestors[release] if parent not in acn_ancestors]
logger.info("Reference %s" %str([parent for parent in ancestors[release] if parent not in acn_ancestors]))
if not reference:
logger.info("Reference previous %s, mark: %d" % (releases[release]['previous'], commit_lookup[releases[release]['previous']]))
reference = [commit_lookup[ releases[release]['previous'] ] ]
mark, merge_commit = create_release_merge_commit(releases, release, get_mark(mark), reference, graphs, set(ancestors[release]) - set(acn_ancestors))
print '\n'.join(merge_commit)
annotated_tag = create_annotated_tag(releases, release, mark)
print '\n'.join(annotated_tag)
commit_lookup[release] = mark
release_queue.extend(releases[release]['next'])
#release = releases[release]['next']
#release = None
#reset to master
master = get_master_tag()
reset = ['reset refs/heads/master', 'from :' + str(commit_lookup[master])]
logger.info("git-fast-import:\n%s" %('\n'.join(reset)))
print '\n'.join(reset)
def recursive_get_history(self, fileobject, recursion_depth):
""" Recursivly find the history of the file object, optionally stopping at the 'old_release' project """
next_iter = False
logger.info('Processing: %s %s' %
(fileobject.get_object_name(), fileobject.get_status()))
logger.info('Recursion depth %d' % recursion_depth)
retval = True
#Check if recursion_depth is reached
if recursion_depth > self.max_recursion_depth:
logger.warning('Giving up on %s' % fileobject.get_object_name())
return False
recursion_depth += 1
predecessors = fileobject.get_predecessors()
for p in predecessors:
try:
predecessor = ccm_cache.get_object(p, self.ccm)
except ccm_cache.ObjectCacheException:
# Object couldn't be retrived from ccm, give up on this
logger.info("Couldn't get %s from Synergy" % p)
return True
logger.info("Predecessor: %s" % predecessor.get_object_name())
# check predecessor release to see if this object should be added to the set.
if self.old_release:
# Get the release(s) for the predecessor
releases = predecessor.get_releases()
if releases:
# Check if the "old" release is the the releases for the predecessor and stop if true
if [
r for r in releases
if r in self.current_subproject_list
or r in self.old_subproject_list
]:
# Object is already released, continue with the next predecessor
logger.info("%s is already released" %
predecessor.get_object_name())
continue
logger.info(
"Couldn't find release in current_subproject_list or old_subproject_list"
)
#Check if chain of successors contains previous object - if true discard the chain
if self.successor_is_released(predecessor, fileobject, 0):
logger.info("Successor is already released %s" %
fileobject.get_object_name())
continue
#Check if projects are releated to old release. Latest first
for r in releases:
try:
project = ccm_cache.get_object(r, self.ccm)
except ccm_cache.ObjectCacheException:
continue
if self.project_is_some_predecessor(project, 0):
logger.info(
"Found Relationship between: %s and %s " %
(project.get_object_name(), self.old_release))
next_iter = True
break
if next_iter:
continue
else:
#Check if a successor is released
if self.successor_is_released(predecessor, fileobject, 0):
logger.info("Successor is already released %s " %
fileobject.get_object_name())
continue
# Check if predecessor is already added to history - if so add this as successor to fileobject, else add new predecessor to history
if not self.temp_history.has_key(predecessor.get_object_name()):
logger.info(
"Adding %s %s to history" %
(predecessor.get_object_name(), predecessor.get_status()))
self.add_to_history(predecessor)
retval &= self.recursive_get_history(predecessor,
recursion_depth)
if not retval:
# Giving up on history break out of loop
break
return retval
def get_project_history(self, latest_project, base_project):
release_chain = deque(
get_project_chain(latest_project, base_project, self.ccm))
base_project = release_chain.popleft()
baseline_project = ccm_cache.get_object(base_project, self.ccm)
self.tag = baseline_project.get_name(
) + self.delim + baseline_project.get_version()
# Initialize history
if self.tag not in self.history.keys():
self.history[self.tag] = {'objects': [], 'tasks': []}
self.history['delimiter'] = self.delim
self.history[self.tag]['previous'] = None
#if self.history[self.tag].has_key('next'):
# if not release_chain[0] in self.history[self.tag]['next']:
# self.history[self.tag]['next'].append(release_chain[0])
#else:
# self.history[self.tag]['next']= [release_chain[0]]
logger.info("getting all objects for: %s ... " % self.tag)
self.baseline_objects = None
# Do the first project as a full project
self.find_project_diff(baseline_project, None)
self.history[self.tag]['name'] = self.tag
self.history[
self.tag]['fourpartname'] = baseline_project.get_object_name()
self.history[self.tag]['created'] = baseline_project.get_created_time()
self.history[self.tag]['author'] = baseline_project.get_author()
# Add the objects and paths to the history, to be used for finding empty directories
empty_dirs = find_empty_dirs(self.baseline_objects)
self.history[self.tag]['empty_dirs'] = empty_dirs
while release_chain:
next = release_chain.popleft()
next_project = ccm_cache.get_object(next, self.ccm)
# Set next project for the current baseline_project
if self.history[self.tag].has_key('next'):
if not next_project.get_name(
) + self.delim + next_project.get_version() in self.history[
self.tag]['next']:
self.history[self.tag]['next'].append(
next_project.get_name() + self.delim +
next_project.get_version())
else:
self.history[self.tag]['next'] = [
next_project.get_name() + self.delim +
next_project.get_version()
]
# Info about baseline_project
logger.info("%s done processing, Info:" % self.tag)
logger.info("Name %s" % self.tag)
logger.info("4partname %s" %
self.history[self.tag]['fourpartname'])
logger.info("Number of: ")
logger.info(" Tasks: %i" %
len(self.history[self.tag]['tasks']))
logger.info(" Files: %i" %
len(self.history[self.tag]['objects']))
logger.info("Previous <- %s" % self.history[self.tag]['previous'])
logger.info("Next -> %s" % str(self.history[self.tag]['next']))
logger.info("")
self.tag = next_project.get_name(
) + self.delim + next_project.get_version()
logger.info("Next project: %s" % next_project.get_object_name())
if self.tag not in self.history.keys():
self.history[self.tag] = {'objects': [], 'tasks': []}
self.history[self.tag]['previous'] = baseline_project.get_name(
) + self.delim + baseline_project.get_version()
logger.info("Toplevel Project: %s " %
next_project.get_object_name())
# do the history thing
self.history[self.tag]['name'] = self.tag
self.history[
self.tag]['fourpartname'] = next_project.get_object_name()
self.find_project_diff(baseline_project, next_project)
self.history[self.tag]['created'] = next_project.get_created_time()
self.history[self.tag]['author'] = next_project.get_author()
# Add the objects and paths to the history, to be used for finding empty directories
empty_dirs = find_empty_dirs(self.project_objects)
logger.info("Empty dirs:\n%s" % '\n'.join(sorted(empty_dirs)))
self.history[self.tag]['empty_dirs'] = empty_dirs
# set new baseline project
baseline_project = next_project
# set the baseline_objects for the next iteration and
self.baseline_objects = self.project_objects
self.project_objects = []
#Store data
fname = self.outputfile + '_' + self.tag
self.persist_data(fname, self.history[self.tag])
self.history_created.append(fname)
# delete the _inc file if it exists
if os.path.isfile(fname + '_inc' + '.p'):
os.remove(fname + '_inc' + '.p')
# Info for last project:
if not self.history[self.tag].has_key('next'):
self.history[self.tag]['next'] = []
logger.info("%s done processing, Info: " % self.tag)
logger.info("Name %s" % self.tag)
logger.info("4partname %s" % self.history[self.tag]['fourpartname'])
logger.info("Number of: ")
logger.info(" Tasks: %i" % len(self.history[self.tag]['tasks']))
logger.info(" Files: %i" % len(self.history[self.tag]['objects']))
logger.info("Previous <- %s" % self.history[self.tag]['previous'])
logger.info("Next -> %s" % str(self.history[self.tag]['next']))
logger.info("")
# Get the different types in the db and their corresponding file permissions and super types
ccm_types_perms = ccm_type.get_types_and_permissions(self.ccm)
ccm_super_types = ccm_type.get_super_types(self.ccm)
if not self.history.has_key('ccm_types'):
self.history['ccm_types'] = {}
self.history['ccm_types']['permissions'] = ccm_types_perms
self.history['ccm_types']['super_types'] = ccm_super_types
return self.history
def populate_cache_with_objects_from_project(project, ccm, ccmpool):
print "processing project %s" %project
#first try to get the object from cache
project_obj = ccm_cache.get_object(project, ccm)
if not project_obj.members:
populate_cache_with_project_and_members(project, ccm, ccmpool)
def get_object(o, objects):
for obj in objects:
if obj == o:
return ccm_cache.get_object(obj)
def find_project_diff(self, baseline_project, next_project):
# Get all objects and paths for baseline_project
if not self.baseline_objects:
self.baseline_objects = baseline_project.get_members()
if self.baseline_objects is None or len(self.baseline_objects) == 1 or not isinstance(self.baseline_objects, dict):
self.baseline_objects = ccm_objects.get_objects_in_project(baseline_project.get_object_name(), ccmpool=self.ccmpool)
baseline_project.set_members(self.baseline_objects)
ccm_cache.force_cache_update_for_object(baseline_project)
if next_project:
# Get all objects and paths for next project
self.project_objects = next_project.get_members()
if self.project_objects is None or len(self.project_objects) == 1 or not isinstance(self.project_objects, dict):
self.project_objects = ccm_objects.get_objects_in_project(next_project.get_object_name(), ccmpool=self.ccmpool)
next_project.set_members(self.project_objects)
ccm_cache.force_cache_update_for_object(next_project)
# Find difference between baseline_project and next_project
new_objects, old_objects = get_changed_objects(self.baseline_objects, self.project_objects)
next_projects = [o for o in self.project_objects.keys() if ':project:' in o]
baseline_projects = [o for o in self.baseline_objects.keys() if ':project:' in o]
object_history = ObjectHistory(self.ccm, next_project.get_object_name(), old_objects=old_objects, old_release=baseline_project.get_object_name(), new_projects=next_projects, old_projects=baseline_projects)
else:
# root project, get ALL objects in release
new_objects = self.baseline_objects
old_objects = []
object_history = ObjectHistory(self.ccm, baseline_project.get_object_name())
num_of_objects = len([o for o in new_objects.keys() if ":project:" not in o])
logger.info("objects to process : %i" % num_of_objects)
objects = []
if self.tag in self.history.keys():
if 'objects' in self.history[self.tag]:
#Add all existing objects
for o in self.history[self.tag]['objects']:
objects.append(o)
#objects[o] = ccm_cache.get_object(o, self.ccm)
logger.info("no of old objects loaded %i", len(objects))
else:
self.history[self.tag] = {'objects': [], 'tasks': []}
object_names = set([o for o in new_objects.keys() if ':project:' not in o]) - set(objects)
for o in object_names:
object = ccm_cache.get_object(o, self.ccm)
if next_project:
# get the object history between releases
history = object_history.get_history(object, new_objects[object.get_object_name()])
objects.extend(history.keys())
#objects.update(object_history.get_history(object, new_objects[object.get_object_name()]))
else:
# just get all the objects in the release
logger.info('Processing: %s path: %s' %(object.get_object_name(), str(new_objects[object.get_object_name()])))
#object.set_path(new_objects[object.get_object_name()])
objects.append(o)
#objects[object.get_object_name()] = object
num_of_objects -=1
logger.info('Objects left: %i' %num_of_objects)
objects = list(set(objects))
logger.info("number of files: %i" % len(objects))
self.history[self.tag]['objects'] = objects
# Create tasks from objects, but not for initial project
if next_project:
self.find_tasks_from_objects(objects, next_project)
# Handle new projects:
if next_project:
new_created_projects = get_new_projects(old_objects, new_objects, self.delim)
dir_lookup ={}
# create lookup for path to directory-4-part-name {path : dir-name}
for k,v in new_objects.iteritems():
if ':dir:' in k:
for i in v:
dir_lookup[i] = k
project_dirs = [dir for project in new_created_projects for dir in new_objects[project]]
directories = [d for k,v in new_objects.iteritems() for d in v if ':dir:' in k]
changed_directories = set(directories).intersection(set(project_dirs))
changed_directories = remove_subdirs_under_same_path(changed_directories)
dirs = [dir_lookup[d] for d in changed_directories]
# find task and add all objects to the task, which shares the path.
project_tasks = self.find_task_from_dirs(dirs)
logger.info("Checking for new subprojects")
# check directories for new subdirectories and add their content
directories = self.get_new_dirs(self.project_objects, new_objects)
# Limit directories to only directories not already processed as a new project
directories = set(directories) - set(dirs)
# find task and add all objects to the task, which shares the path.
dir_tasks = self.find_task_from_dirs(directories)
# merge project and dir tasks
for k,v in dir_tasks.iteritems():
if not project_tasks.has_key(k):
project_tasks[k] = v
# if real synergy tasks isn't found check the path of the directories and skip possible subdirs
tasks = self.reduce_dir_tasks(project_tasks)
logger.info("Project and dir tasks reduced...")
logger.info("%s" % str(tasks))
self.update_tasks_with_directory_contens(tasks)
# remove possible duplicates from objects
self.history[self.tag]['objects'] = list(set(self.history[self.tag]['objects']))
def ccm_fast_export(releases, graphs):
global acn_ancestors
global users
users = users()
logger.basicConfig(filename='ccm_fast_export.log',level=logger.DEBUG)
commit_lookup = {}
# Get the initial release
for k, v in releases.iteritems():
if k == 'delimiter':
continue
if k == 'ccm_types':
continue
if v['previous'] is None:
release = k
break
logger.info("Starting at %s as initial release" % release)
if 'created' not in releases[release]:
initial_release_time = 0.0 # epoch for now since releases[release] has no 'created' key :(
else:
initial_release_time = time.mktime(releases[release]['created'].timetuple())
mark = 0
files = []
# Create the initial release
# get all the file objects:
file_objects = [ccm_cache.get_object(o) for o in releases[release]['objects']]
project_obj = ccm_cache.get_object(releases[release]['fourpartname'])
paths = project_obj.get_members()
for o in file_objects:
if o.get_type() != 'dir':
object_mark, mark = create_blob(o, mark)
for p in paths[o.get_object_name()]:
files.append('M ' + releases['ccm_types']['permissions'][o.get_type()] + ' :'+str(object_mark) + ' ' + p)
empty_dirs = releases[release]['empty_dirs']
logger.info("Empty dirs for release %s\n%s" %(release, empty_dirs))
mark = create_blob_for_empty_dir(get_mark(mark))
#file_list = create_file_list(objects, object_lookup, releases['ccm_types'], empty_dirs=empty_dirs, empty_dir_mark=mark)
if empty_dirs:
for d in empty_dirs:
if mark:
path = d + '/.gitignore'
files.append('M 100644 :' + str(mark) + ' ' + path)
mark = get_mark(mark)
commit_info = ['reset refs/tags/' + release, 'commit refs/tags/' + release, 'mark :' + str(mark),
'author Nokia <*****@*****.**> ' + str(int(initial_release_time)) + " +0000",
'committer Nokia <*****@*****.**> ' + str(int(initial_release_time)) + " +0000", 'data 15',
'Initial commit', '\n'.join(files), '']
print '\n'.join(commit_info)
logger.info("git-fast-import:\n%s" %('\n'.join(commit_info)))
tag_msg = 'Release: %s' %release
annotated_tag = ['tag %s' % release,
'from :%s' % str(mark),
'tagger Nokia <*****@*****.**> ' + str(int(initial_release_time)) + " +0000",
'data %s' % len(tag_msg),
tag_msg]
print '\n'.join(annotated_tag)
commit_lookup[release] = mark
# do the following releases (graphs)
release_queue = deque(releases[release]['next'])
while release_queue:
release = release_queue.popleft()
previous_release = releases[release]['previous']
logger.info("Next release: %s" % release)
commit_graph = graphs[release]['commit']
commit_graph = fix_orphan_nodes(commit_graph, previous_release)
commit_graph = ch.spaghettify_digraph(commit_graph, previous_release, release)
#htg.commit_graph_to_image(commit_graph, releases[release], graphs[release]['task'], name=releases[release]['name']+'_after' )
# Find the cutting nodes
logger.info("Finding the cutting nodes")
undirected = graph()
undirected.add_nodes(commit_graph.nodes())
[undirected.add_edge(edge) for edge in commit_graph.edges()]
cutting_nodes = cut_nodes(undirected)
del undirected
# Create the reverse commit graph
logger.info("Building the reverse commit graph")
reverse_commit_graph = commit_graph.reverse()
# Compute the accessibility matrix of the reverse commit graph
logger.info("Compute the ancestors")
ancestors = accessibility(reverse_commit_graph)
del reverse_commit_graph
logger.info("Ancestors of the release: %s" % str(ancestors[release]))
# Clean up the ancestors matrix
for k, v in ancestors.iteritems():
if k in v:
v.remove(k)
# Get the commits order
commits = topological_sorting(commit_graph)
# Fix the commits order list
commits.remove(previous_release)
commits.remove(release)
last_cutting_node = None
# Check if the release (Synergy project has changed name, if it has the
# 'base' directory name needs to be renamed
if releases.has_key('delimiter'):
delim = releases['delimiter']
else:
delim = '-'
previous_name = previous_release.split(delim)[0]
current_name = release.split(delim)[0]
if current_name != previous_name:
logger.info("Name changed: %s -> %s" %(previous_name, current_name))
from_mark = commit_lookup[previous_release]
mark, commit = rename_toplevel_dir(previous_name, current_name, release, releases, mark, from_mark)
print '\n'.join(commit)
# adjust the commit lookup
commit_lookup[previous_release] = mark
for counter, commit in enumerate(commits):
logger.info("Commit %i/%i" % (counter+1, len(commits)))
acn_ancestors = []
if last_cutting_node is not None:
acn_ancestors = ancestors[last_cutting_node]
# Create the references lists. It lists the parents of the commit
#reference = [commit_lookup[parent] for parent in ancestors[commit] if parent not in acn_ancestors]
reference = [commit_lookup[parent] for parent in commit_graph.incidents(commit)]
if len(reference) > 1:
# Merge commit
mark = create_merge_commit(commit, release, releases, mark, reference, graphs, set(ancestors[commit]) - set(acn_ancestors))
else:
# Normal commit
mark = create_commit(commit, release, releases, mark, reference, graphs)
# Update the lookup table
commit_lookup[commit] = mark
# Update the last cutting edge if necessary
if commit in cutting_nodes:
last_cutting_node = commit
if last_cutting_node is not None:
acn_ancestors = ancestors[last_cutting_node]
reference = [commit_lookup[parent] for parent in ancestors[release] if parent not in acn_ancestors]
logger.info("Reference %s" %str([parent for parent in ancestors[release] if parent not in acn_ancestors]))
if not reference:
logger.info("Reference previous %s, mark: %d" % (releases[release]['previous'], commit_lookup[releases[release]['previous']]))
reference = [commit_lookup[ releases[release]['previous'] ] ]
mark, merge_commit = create_release_merge_commit(releases, release, get_mark(mark), reference, graphs, set(ancestors[release]) - set(acn_ancestors))
print '\n'.join(merge_commit)
annotated_tag = create_annotated_tag(releases, release, mark)
print '\n'.join(annotated_tag)
commit_lookup[release] = mark
release_queue.extend(releases[release]['next'])
#release = releases[release]['next']
#release = None
#reset to master
master = get_master_tag()
reset = ['reset refs/heads/master', 'from :' + str(commit_lookup[master])]
logger.info("git-fast-import:\n%s" %('\n'.join(reset)))
print '\n'.join(reset)
def convert_history(files, tasks, releases, objects):
"""Converts the Synergy history between two releases to a Git compatible one."""
log.basicConfig(filename="convert_history.log", level=log.DEBUG)
file_objects = [ccm_cache.get_object(o) for o in objects]
log.info("Looking for cycles in the File History graph")
while find_cycle(files):
cycle = find_cycle(files)
log.info("\tA cycle was found!")
log.info("\tCycle: %s" % ", ".join(cycle))
# Find the newest file
newest = max(
cycle,
key=lambda x: [
fileobject.get_integrate_time() for fileobject in file_objects if fileobject.get_objectname() == x
][0],
)
log.info("\tObject %s is the newest in the cycle: it should not have successors!" % newest)
# Remove the outgoing link from the newest file
for successor in files.neighbors(newest):
if successor in cycle:
files.del_edge((newest, successor))
log.info("\tRemoved the %s -> %s edge" % (newest, successor))
log.info("Remove transitive edges in the File History graph")
for edge in transitive_edges(files):
if edge in files.edges():
files.del_edge(edge)
else:
log.warning("Weird, transitive edge not found!")
log.info("Sanitize tasks")
sanitized_tasks = _sanitize_tasks(tasks)
log.info("Create commits graph")
commits = create_commits_graph(files, sanitized_tasks, releases)
# Uncomment for debug... (remember import)
# hack = {'previous': releases.edges()[0]}
# htg.commit_graph_to_image(commits, hack, tasks, name='Pre-'+releases.edges()[1])
log.info("Looking for cycles in the Commits graph")
while find_cycle(commits):
log.info("Finding strictly connected components")
cycle = max(mutual_accessibility(commits).values(), key=len)
# cycle = find_cycle(commits)
log.info("\tA cycle was found!")
log.info("\tCycle: %s" % ", ".join(cycle))
log.info("Find the nodes in the cycle going from one task to another")
culpript_edges = []
for task in cycle:
for obj in tasks.links(task):
for neighbor in files.neighbors(obj):
if neighbor not in tasks.links(task) and tasks.links(neighbor)[0] in cycle:
culpript_edges.append((obj, neighbor))
log.info("\tAdding culpript edge (%s, %s)" % (obj, neighbor))
log.info("Connect the nodes found")
culpript_nodes = set()
for head, tail in culpript_edges:
culpript_nodes.add(head)
culpript_nodes.add(tail)
for head, tail in permutations(culpript_nodes, 2):
if tasks.links(head)[0] == tasks.links(tail)[0] and (head, tail) not in culpript_edges:
log.info("\tAdding edge (%s, %s)" % (head, tail))
culpript_edges.append((head, tail))
reduced_digraph = digraph()
reduced_digraph.add_nodes(culpript_nodes)
[reduced_digraph.add_edge(edge) for edge in culpript_edges]
shortest_cycle = max(mutual_accessibility(reduced_digraph).values(), key=len)
log.info("Cycle in objects: %s" % shortest_cycle)
candidate_cuts = []
# Find the tasks
t = set()
for node in shortest_cycle:
t.add(tasks.links(node)[0])
log.info("T: %s" % str(t))
for i in t:
log.info("Cuts for task %s" % i)
# Find the objects in the cycle belonging to task i
obj_in_task = set(tasks.links(i)) & set(shortest_cycle)
log.info("Objects in cycle and task: %s" % obj_in_task)
if len(obj_in_task) < 15:
if len(obj_in_task) > 1:
for j in range(1, len(obj_in_task) / 2 + 1):
candidate_cuts.extend([k for k in combinations(obj_in_task, j)])
else:
log.info("Cycle too long...")
pass
log.info("Candidate_cuts: %s" % str(candidate_cuts))
# Find the cut to break the cycle
cut = _find_cut(candidate_cuts, cycle, tasks, files, releases)
if not cut:
# Make a qualified guess of a cut with the shortest walk of files in the tasks
walk, node = _find_shortest_incident_or_neighbor_walk(shortest_cycle, cycle, files, tasks)
new_cut = walk
new_cut.append(node)
candidate_cuts.insert(0, tuple(new_cut))
log.info("Candidate cuts: %s", candidate_cuts)
cut = _find_cut(candidate_cuts, cycle, tasks, files, releases)
if not cut:
# Error! This should not happen
log.info("Cut not found.")
log.shutdown()
raise Exception("Cut not found")
tasks, task, task_name = _apply_cut(cut, tasks)
commits = create_commits_graph(files, tasks, releases)
else:
log.info("No cycles found")
log.shutdown()
return commits
def get_project_history(self, latest_project, base_project):
release_chain = deque(get_project_chain(latest_project, base_project, self.ccm))
base_project = release_chain.popleft()
baseline_project = ccm_cache.get_object(base_project, self.ccm)
self.tag = baseline_project.get_name() + self.delim + baseline_project.get_version()
# Initialize history
if self.tag not in self.history.keys():
self.history[self.tag] = {'objects': [], 'tasks': []}
self.history['delimiter'] = self.delim
self.history[self.tag]['previous'] = None
#if self.history[self.tag].has_key('next'):
# if not release_chain[0] in self.history[self.tag]['next']:
# self.history[self.tag]['next'].append(release_chain[0])
#else:
# self.history[self.tag]['next']= [release_chain[0]]
logger.info("getting all objects for: %s ... " % self.tag)
self.baseline_objects = None
# Do the first project as a full project
self.find_project_diff(baseline_project, None)
self.history[self.tag]['name'] = self.tag
self.history[self.tag]['fourpartname'] = baseline_project.get_object_name()
self.history[self.tag]['created'] = baseline_project.get_created_time()
self.history[self.tag]['author'] = baseline_project.get_author()
# Add the objects and paths to the history, to be used for finding empty directories
empty_dirs = find_empty_dirs(self.baseline_objects)
self.history[self.tag]['empty_dirs'] = empty_dirs
while release_chain:
next = release_chain.popleft()
next_project = ccm_cache.get_object(next, self.ccm)
# Set next project for the current baseline_project
if self.history[self.tag].has_key('next'):
if not next_project.get_name() + self.delim + next_project.get_version() in self.history[self.tag]['next']:
self.history[self.tag]['next'].append(next_project.get_name() + self.delim + next_project.get_version())
else:
self.history[self.tag]['next'] = [next_project.get_name() + self.delim + next_project.get_version()]
# Info about baseline_project
logger.info("%s done processing, Info:" % self.tag)
logger.info("Name %s" % self.tag)
logger.info("4partname %s" % self.history[self.tag]['fourpartname'])
logger.info("Number of: ")
logger.info(" Tasks: %i" % len(self.history[self.tag]['tasks']))
logger.info(" Files: %i" % len(self.history[self.tag]['objects']))
logger.info("Previous <- %s" % self.history[self.tag]['previous'])
logger.info("Next -> %s" % str(self.history[self.tag]['next']))
logger.info("")
self.tag = next_project.get_name() + self.delim + next_project.get_version()
logger.info("Next project: %s" % next_project.get_object_name())
if self.tag not in self.history.keys():
self.history[self.tag] = {'objects': [], 'tasks': []}
self.history[self.tag]['previous'] = baseline_project.get_name() + self.delim + baseline_project.get_version()
logger.info("Toplevel Project: %s " % next_project.get_object_name())
# do the history thing
self.history[self.tag]['name'] = self.tag
self.history[self.tag]['fourpartname'] = next_project.get_object_name()
self.find_project_diff(baseline_project, next_project)
self.history[self.tag]['created'] = next_project.get_created_time()
self.history[self.tag]['author'] = next_project.get_author()
# Add the objects and paths to the history, to be used for finding empty directories
empty_dirs = find_empty_dirs(self.project_objects)
logger.info("Empty dirs:\n%s" % '\n'.join(sorted(empty_dirs)))
self.history[self.tag]['empty_dirs'] = empty_dirs
# set new baseline project
baseline_project = next_project
# set the baseline_objects for the next iteration and
self.baseline_objects = self.project_objects
self.project_objects = []
#Store data
fname = self.outputfile + '_' + self.tag
self.persist_data(fname, self.history[self.tag])
self.history_created.append(fname)
# delete the _inc file if it exists
if os.path.isfile(fname + '_inc' + '.p'):
os.remove(fname + '_inc' + '.p')
# Info for last project:
if not self.history[self.tag].has_key('next'):
self.history[self.tag]['next'] = []
logger.info("%s done processing, Info: " %self.tag)
logger.info("Name %s" % self.tag)
logger.info("4partname %s" % self.history[self.tag]['fourpartname'])
logger.info("Number of: ")
logger.info(" Tasks: %i" % len(self.history[self.tag]['tasks']))
logger.info(" Files: %i" % len(self.history[self.tag]['objects']))
logger.info("Previous <- %s" % self.history[self.tag]['previous'])
logger.info("Next -> %s" % str(self.history[self.tag]['next']))
logger.info("")
# Get the different types in the db and their corresponding file permissions and super types
ccm_types_perms = ccm_type.get_types_and_permissions(self.ccm)
ccm_super_types = ccm_type.get_super_types(self.ccm)
if not self.history.has_key('ccm_types'):
self.history['ccm_types'] = {}
self.history['ccm_types']['permissions'] = ccm_types_perms
self.history['ccm_types']['super_types'] = ccm_super_types
return self.history
def recursive_get_history(self, fileobject, recursion_depth):
""" Recursivly find the history of the file object, optionally stopping at the 'old_release' project """
next_iter = False
logger.info('Processing: %s %s' % (fileobject.get_object_name(), fileobject.get_status()))
logger.info('Recursion depth %d' % recursion_depth)
retval = True
#Check if recursion_depth is reached
if recursion_depth > self.max_recursion_depth:
logger.warning('Giving up on %s' % fileobject.get_object_name())
return False
recursion_depth += 1
predecessors = fileobject.get_predecessors()
for p in predecessors:
try:
predecessor = ccm_cache.get_object(p, self.ccm)
except ccm_cache.ObjectCacheException:
# Object couldn't be retrived from ccm, give up on this
logger.info("Couldn't get %s from Synergy" % p)
return True
logger.info("Predecessor: %s" % predecessor.get_object_name())
# check predecessor release to see if this object should be added to the set.
if self.old_release:
# Get the release(s) for the predecessor
releases = predecessor.get_releases()
if releases:
# Check if the "old" release is the the releases for the predecessor and stop if true
if [r for r in releases if r in self.current_subproject_list or r in self.old_subproject_list]:
# Object is already released, continue with the next predecessor
logger.info("%s is already released" %predecessor.get_object_name())
continue
logger.info("Couldn't find release in current_subproject_list or old_subproject_list")
#Check if chain of successors contains previous object - if true discard the chain
if self.successor_is_released(predecessor, fileobject, 0):
logger.info("Successor is already released %s" % fileobject.get_object_name())
continue
#Check if projects are releated to old release. Latest first
for r in releases:
try:
project = ccm_cache.get_object(r, self.ccm)
except ccm_cache.ObjectCacheException:
continue
if self.project_is_some_predecessor(project, 0):
logger.info("Found Relationship between: %s and %s " %(project.get_object_name(), self.old_release))
next_iter = True
break
if next_iter:
continue
else:
#Check if a successor is released
if self.successor_is_released(predecessor, fileobject, 0):
logger.info("Successor is already released %s " % fileobject.get_object_name())
continue
# Check if predecessor is already added to history - if so add this as successor to fileobject, else add new predecessor to history
if not self.temp_history.has_key(predecessor.get_object_name()):
logger.info("Adding %s %s to history" % (predecessor.get_object_name(), predecessor.get_status()) )
self.add_to_history(predecessor)
retval &= self.recursive_get_history(predecessor, recursion_depth)
if not retval:
# Giving up on history break out of loop
break
return retval
def get_objects_in_project_serial(project,
ccm=None,
database=None,
use_cache=False):
""" Get all objects and paths of a project """
if not ccm:
if not database:
raise SynergyException("No ccm instance nor database given\n" +
"Cannot start ccm session!\n")
ccm = SynergySession(database)
else:
logger.debug("ccm instance: %s" % ccm.environment['CCM_ADDR'])
delim = ccm.delim()
if use_cache:
start_object = ccm_cache.get_object(project, ccm)
else:
start_object = SynergyObject(project, delim)
queue = deque([start_object])
hierarchy = {}
dir_structure = {}
proj_lookup = {}
cwd = ''
count = 1
hierarchy[start_object.get_object_name()] = [start_object.name]
dir_structure[start_object.get_object_name()] = ''
while queue:
obj = queue.popleft()
#logger.debug('Processing: %s' % obj.get_object_name())
parent_proj = None
if obj.get_type() == 'dir' or obj.get_type() == 'project':
# Processing a dir set 'working dir'
cwd = dir_structure[obj.get_object_name()]
if obj.get_type() == 'dir':
parent_proj = proj_lookup[obj.get_object_name()]
result = get_members(obj, ccm, parent_proj)
if use_cache:
objects = []
na_obj = []
for item in result:
try:
objects.append(
ccm_cache.get_object(item['objectname'], ccm))
except ccm_cache.ObjectCacheException:
objects.append(
SynergyObject(item['objectname'], ccm.delim()))
na_obj.append(item['objectname'])
if na_obj:
logger.warning("Objects not avaliable in this db:")
logger.warning(', '.join(na_obj))
else:
objects = [
SynergyObject(item['objectname'], delim) for item in result
]
# if a project is being queried it might have more than one dir with the
# same name as the project associated, find the directory that has the
# project associated as the directory's parent
if obj.get_type() == 'project':
if len(objects) > 1:
objects = find_root_project(obj, objects, ccm)
for synergy_object in objects:
count += 1
if synergy_object.get_type() == 'dir':
# add the directory to the queue and record its parent project
queue.append(synergy_object)
#logger.debug("object: %s child %s cwd %s" % (obj
# .get_object_name(), o.get_object_name(), cwd))
dir_structure[synergy_object.get_object_name()] = \
'%s%s/' % (cwd, synergy_object.get_name())
if obj.get_type() == 'project':
proj_lookup[synergy_object.get_object_name()] = \
obj.get_object_name()
elif obj.get_type() == 'dir':
proj_lookup[synergy_object.get_object_name()] = \
proj_lookup[obj.get_object_name()]
# Also add the directory to the Hierachy to get empty dirs
if synergy_object.get_object_name() in hierarchy.keys():
hierarchy[synergy_object.get_object_name()].append(
'%s%s' % (cwd, synergy_object.get_name()))
else:
hierarchy[synergy_object.get_object_name()] = \
['%s%s' % (cwd, synergy_object.get_name())]
elif synergy_object.get_type() == 'project':
dir_structure[synergy_object.get_object_name()] = cwd
# Add the project to the queue
queue.append(synergy_object)
#logger.debug("object: %s child %s cwd %s" % (obj
# .get_object_name(), o.get_object_name(), cwd))
# Add the project to the hierarchy,
# so the subprojects for the release/project is known
if synergy_object.get_object_name() in hierarchy.keys():
hierarchy[synergy_object.get_object_name()].append(
'%s%s' % (cwd, synergy_object.get_name()))
else:
hierarchy[synergy_object.get_object_name()] = \
['%s%s' % (cwd, synergy_object.get_name())]
else:
# Add the object to the hierarchy
if obj.get_type() == 'dir':
if synergy_object.get_object_name() in hierarchy.keys():
hierarchy[synergy_object.get_object_name()].append(
'%s%s' % (cwd, synergy_object.get_name()))
else:
hierarchy[synergy_object.get_object_name()] = \
['%s%s' % (cwd, synergy_object.get_name())]
#logger.debug("Object: %s has path %s%s" % (o.get_object_name(), cwd, o.get_name()))
logger.debug("Object count: %6d" % count)
return hierarchy
def get_objects_in_project_serial(project, ccm=None, database=None, use_cache=False):
""" Get all objects and paths of a project """
if not ccm:
if not database:
raise SynergyException("No ccm instance nor database given\n" +
"Cannot start ccm session!\n")
ccm = SynergySession(database)
else:
logger.debug("ccm instance: %s" % ccm.environment['CCM_ADDR'])
delim = ccm.delim()
if use_cache:
start_object = ccm_cache.get_object(project, ccm)
else:
start_object = SynergyObject(project, delim)
queue = deque([start_object])
hierarchy = {}
dir_structure = {}
proj_lookup = {}
cwd = ''
count = 1
hierarchy[start_object.get_object_name()] = [start_object.name]
dir_structure[start_object.get_object_name()] = ''
while queue:
obj = queue.popleft()
#logger.debug('Processing: %s' % obj.get_object_name())
parent_proj = None
if obj.get_type() == 'dir' or obj.get_type() == 'project':
# Processing a dir set 'working dir'
cwd = dir_structure[obj.get_object_name()]
if obj.get_type() == 'dir':
parent_proj = proj_lookup[obj.get_object_name()]
result = get_members(obj, ccm, parent_proj)
if use_cache:
objects = []
na_obj = []
for item in result:
try:
objects.append(ccm_cache.get_object(item['objectname'], ccm))
except ccm_cache.ObjectCacheException:
objects.append(SynergyObject(item['objectname'], ccm.delim()))
na_obj.append(item['objectname'])
if na_obj:
logger.warning("Objects not avaliable in this db:")
logger.warning(', '.join(na_obj))
else:
objects = [SynergyObject(item['objectname'], delim) for item in result]
# if a project is being queried it might have more than one dir with the
# same name as the project associated, find the directory that has the
# project associated as the directory's parent
if obj.get_type() == 'project':
if len(objects) > 1:
objects = find_root_project(obj, objects, ccm)
for synergy_object in objects:
count += 1
if synergy_object.get_type() == 'dir':
# add the directory to the queue and record its parent project
queue.append(synergy_object)
#logger.debug("object: %s child %s cwd %s" % (obj
# .get_object_name(), o.get_object_name(), cwd))
dir_structure[synergy_object.get_object_name()] = \
'%s%s/' % (cwd, synergy_object.get_name())
if obj.get_type() == 'project':
proj_lookup[synergy_object.get_object_name()] = \
obj.get_object_name()
elif obj.get_type() == 'dir':
proj_lookup[synergy_object.get_object_name()] = \
proj_lookup[obj.get_object_name()]
# Also add the directory to the Hierachy to get empty dirs
if synergy_object.get_object_name() in hierarchy.keys():
hierarchy[synergy_object.get_object_name()].append(
'%s%s' % (cwd, synergy_object.get_name()))
else:
hierarchy[synergy_object.get_object_name()] = \
['%s%s' % (cwd, synergy_object.get_name())]
elif synergy_object.get_type() == 'project':
dir_structure[synergy_object.get_object_name()] = cwd
# Add the project to the queue
queue.append(synergy_object)
#logger.debug("object: %s child %s cwd %s" % (obj
# .get_object_name(), o.get_object_name(), cwd))
# Add the project to the hierarchy,
# so the subprojects for the release/project is known
if synergy_object.get_object_name() in hierarchy.keys():
hierarchy[synergy_object.get_object_name()].append(
'%s%s' % (cwd, synergy_object.get_name()))
else:
hierarchy[synergy_object.get_object_name()] = \
['%s%s' % (cwd, synergy_object.get_name())]
else:
# Add the object to the hierarchy
if obj.get_type() == 'dir':
if synergy_object.get_object_name() in hierarchy.keys():
hierarchy[synergy_object.get_object_name()].append(
'%s%s' % (cwd, synergy_object.get_name()))
else:
hierarchy[synergy_object.get_object_name()] = \
['%s%s' % (cwd, synergy_object.get_name())]
#logger.debug("Object: %s has path %s%s" % (o.get_object_name(), cwd, o.get_name()))
logger.debug("Object count: %6d" % count)
return hierarchy
def convert_history(files, tasks, releases, objects):
"""Converts the Synergy history between two releases to a Git compatible one."""
log.basicConfig(filename='convert_history.log', level=log.DEBUG)
file_objects = [ccm_cache.get_object(o) for o in objects]
log.info("Looking for cycles in the File History graph")
while find_cycle(files):
cycle = find_cycle(files)
log.info("\tA cycle was found!")
log.info("\tCycle: %s" % ", ".join(cycle))
# Find the newest file
newest = max(cycle,
key=lambda x: [
fileobject.get_integrate_time()
for fileobject in file_objects
if fileobject.get_objectname() == x
][0])
log.info(
"\tObject %s is the newest in the cycle: it should not have successors!"
% newest)
# Remove the outgoing link from the newest file
for successor in files.neighbors(newest):
if successor in cycle:
files.del_edge((newest, successor))
log.info("\tRemoved the %s -> %s edge" % (newest, successor))
log.info("Remove transitive edges in the File History graph")
for edge in transitive_edges(files):
if edge in files.edges():
files.del_edge(edge)
else:
log.warning("Weird, transitive edge not found!")
log.info("Sanitize tasks")
sanitized_tasks = _sanitize_tasks(tasks)
log.info("Create commits graph")
commits = create_commits_graph(files, sanitized_tasks, releases)
# Uncomment for debug... (remember import)
#hack = {'previous': releases.edges()[0]}
#htg.commit_graph_to_image(commits, hack, tasks, name='Pre-'+releases.edges()[1])
log.info("Looking for cycles in the Commits graph")
while find_cycle(commits):
log.info("Finding strictly connected components")
cycle = max(mutual_accessibility(commits).values(), key=len)
#cycle = find_cycle(commits)
log.info("\tA cycle was found!")
log.info("\tCycle: %s" % ", ".join(cycle))
log.info("Find the nodes in the cycle going from one task to another")
culpript_edges = []
for task in cycle:
for obj in tasks.links(task):
for neighbor in files.neighbors(obj):
if neighbor not in tasks.links(task) and tasks.links(
neighbor)[0] in cycle:
culpript_edges.append((obj, neighbor))
log.info("\tAdding culpript edge (%s, %s)" %
(obj, neighbor))
log.info("Connect the nodes found")
culpript_nodes = set()
for head, tail in culpript_edges:
culpript_nodes.add(head)
culpript_nodes.add(tail)
for head, tail in permutations(culpript_nodes, 2):
if tasks.links(head)[0] == tasks.links(tail)[0] and (
head, tail) not in culpript_edges:
log.info("\tAdding edge (%s, %s)" % (head, tail))
culpript_edges.append((head, tail))
reduced_digraph = digraph()
reduced_digraph.add_nodes(culpript_nodes)
[reduced_digraph.add_edge(edge) for edge in culpript_edges]
shortest_cycle = max(mutual_accessibility(reduced_digraph).values(),
key=len)
log.info("Cycle in objects: %s" % shortest_cycle)
candidate_cuts = []
# Find the tasks
t = set()
for node in shortest_cycle:
t.add(tasks.links(node)[0])
log.info("T: %s" % str(t))
for i in t:
log.info("Cuts for task %s" % i)
# Find the objects in the cycle belonging to task i
obj_in_task = set(tasks.links(i)) & set(shortest_cycle)
log.info("Objects in cycle and task: %s" % obj_in_task)
if len(obj_in_task) < 15:
if len(obj_in_task) > 1:
for j in range(1, len(obj_in_task) / 2 + 1):
candidate_cuts.extend(
[k for k in combinations(obj_in_task, j)])
else:
log.info("Cycle too long...")
pass
if len(candidate_cuts) < 50:
log.info("Candidate_cuts: %s" % str(candidate_cuts))
# Find the cut to break the cycle
cut = _find_cut(candidate_cuts, cycle, tasks, files, releases)
if not cut:
log.info("Shortest cycle cut didn't work, next option...")
# Make a qualified guess of a cut with the shortest walk of files in the tasks
walk, node = _find_shortest_incident_or_neighbor_walk(
shortest_cycle, cycle, files, tasks)
log.info("Shortest incident or neighbor walk from {0}: {1}".format(
node, walk))
new_cut = walk
new_cut.append(node)
candidate_cuts.insert(0, tuple(new_cut))
log.info("Candidate cuts: %s", candidate_cuts)
cut = _find_cut(candidate_cuts, cycle, tasks, files, releases)
if not cut:
# Error! This should not happen
log.info("Cut not found.")
log.shutdown()
raise Exception("Cut not found")
tasks, task, task_name = _apply_cut(cut, tasks)
commits = create_commits_graph(files, tasks, releases)
else:
log.info("No cycles found")
log.shutdown()
return commits
def get_object(o, objects):
for obj in objects:
if obj == o:
return ccm_cache.get_object(obj)
def find_project_diff(self, baseline_project, next_project):
# Get all objects and paths for baseline_project
if not self.baseline_objects:
self.baseline_objects = baseline_project.get_members()
if self.baseline_objects is None or len(
self.baseline_objects) == 1 or not isinstance(
self.baseline_objects, dict):
self.baseline_objects = ccm_objects.get_objects_in_project(
baseline_project.get_object_name(), ccmpool=self.ccmpool)
baseline_project.set_members(self.baseline_objects)
ccm_cache.force_cache_update_for_object(baseline_project)
if next_project:
# Get all objects and paths for next project
self.project_objects = next_project.get_members()
if self.project_objects is None or len(
self.project_objects) == 1 or not isinstance(
self.project_objects, dict):
self.project_objects = ccm_objects.get_objects_in_project(
next_project.get_object_name(), ccmpool=self.ccmpool)
next_project.set_members(self.project_objects)
ccm_cache.force_cache_update_for_object(next_project)
# Find difference between baseline_project and next_project
new_objects, old_objects = get_changed_objects(
self.baseline_objects, self.project_objects)
next_projects = [
o for o in self.project_objects.keys() if ':project:' in o
]
baseline_projects = [
o for o in self.baseline_objects.keys() if ':project:' in o
]
object_history = ObjectHistory(
self.ccm,
next_project.get_object_name(),
old_objects=old_objects,
old_release=baseline_project.get_object_name(),
new_projects=next_projects,
old_projects=baseline_projects)
else:
# root project, get ALL objects in release
new_objects = self.baseline_objects
old_objects = []
object_history = ObjectHistory(self.ccm,
baseline_project.get_object_name())
num_of_objects = len(
[o for o in new_objects.keys() if ":project:" not in o])
logger.info("objects to process : %i" % num_of_objects)
objects = []
if self.tag in self.history.keys():
if 'objects' in self.history[self.tag]:
#Add all existing objects
for o in self.history[self.tag]['objects']:
objects.append(o)
#objects[o] = ccm_cache.get_object(o, self.ccm)
logger.info("no of old objects loaded %i", len(objects))
else:
self.history[self.tag] = {'objects': [], 'tasks': []}
object_names = set([
o for o in new_objects.keys() if ':project:' not in o
]) - set(objects)
for o in object_names:
object = ccm_cache.get_object(o, self.ccm)
if next_project:
# get the object history between releases
history = object_history.get_history(
object, new_objects[object.get_object_name()])
objects.extend(history.keys())
#objects.update(object_history.get_history(object, new_objects[object.get_object_name()]))
else:
# just get all the objects in the release
logger.info('Processing: %s path: %s' %
(object.get_object_name(),
str(new_objects[object.get_object_name()])))
#object.set_path(new_objects[object.get_object_name()])
objects.append(o)
#objects[object.get_object_name()] = object
num_of_objects -= 1
logger.info('Objects left: %i' % num_of_objects)
objects = list(set(objects))
logger.info("number of files: %i" % len(objects))
self.history[self.tag]['objects'] = objects
# Create tasks from objects, but not for initial project
if next_project:
self.find_tasks_from_objects(objects, next_project)
# Handle new projects:
if next_project:
new_created_projects = get_new_projects(old_objects, new_objects,
self.delim)
dir_lookup = {}
# create lookup for path to directory-4-part-name {path : dir-name}
for k, v in new_objects.iteritems():
if ':dir:' in k:
for i in v:
dir_lookup[i] = k
project_dirs = [
dir for project in new_created_projects
for dir in new_objects[project]
]
directories = [
d for k, v in new_objects.iteritems() for d in v
if ':dir:' in k
]
changed_directories = set(directories).intersection(
set(project_dirs))
changed_directories = remove_subdirs_under_same_path(
changed_directories)
dirs = [dir_lookup[d] for d in changed_directories]
# find task and add all objects to the task, which shares the path.
project_tasks = self.find_task_from_dirs(dirs)
logger.info("Checking for new subprojects")
# check directories for new subdirectories and add their content
directories = self.get_new_dirs(self.project_objects, new_objects)
# Limit directories to only directories not already processed as a new project
directories = set(directories) - set(dirs)
# find task and add all objects to the task, which shares the path.
dir_tasks = self.find_task_from_dirs(directories)
# merge project and dir tasks
for k, v in dir_tasks.iteritems():
if not project_tasks.has_key(k):
project_tasks[k] = v
# if real synergy tasks isn't found check the path of the directories and skip possible subdirs
tasks = self.reduce_dir_tasks(project_tasks)
logger.info("Project and dir tasks reduced...")
logger.info("%s" % str(tasks))
self.update_tasks_with_directory_contens(tasks)
# remove possible duplicates from objects
self.history[self.tag]['objects'] = list(
set(self.history[self.tag]['objects']))
