def find_tasks_from_objects(self, objects, project): tasks = {} unconfirmed_tasks = {} if self.tag in self.history.keys(): if 'tasks' in self.history[self.tag]: for t in self.history[self.tag]['tasks']: logger.info("loading old task: %s" % t.get_object_name()) tasks[t.get_object_name()] = t #Build a list of all tasks task_list = [ task for o in objects for task in ccm_cache.get_object(o, self.ccm).get_tasks() ] num_of_tasks = len(set(task_list)) logger.info("Tasks with associated objects: %i" % num_of_tasks) task_util = TaskUtil(self.ccm) for t in set(task_list) - set(tasks.keys()): try: task = ccm_cache.get_object(t, self.ccm) except ccm_cache.ObjectCacheException: continue if task.status == 'completed': if task_util.task_in_project(task, project): tasks[task.get_object_name()] = task else: # try to add it anyway to see what happens... unconfirmed_tasks[task.get_object_name()] = task # Add objects to tasks [ t.add_object(o) for o in objects for t in tasks.values() if t.get_object_name() in ccm_cache.get_object( o, self.ccm).get_tasks() ] # Add objects to unconfirmed tasks [ t.add_object(o) for o in objects for t in unconfirmed_tasks.values() if t.get_object_name() in ccm_cache.get_object(o, self.ccm).get_tasks() ] logger.info("Sanitizing tasks") tasks = sanitize_tasks(tasks, unconfirmed_tasks) logger.info("No of tasks in release %s: %i" % (project.get_object_name(), len(tasks.keys()))) self.history[self.tag]['tasks'] = tasks.values() fname = self.outputfile + '_' + self.tag + '_inc' self.persist_data(fname, self.history[self.tag])
def get_objects_in_project(project, ccm=None, database=None, ccmpool=None, use_cache=False): """ Get all objects and paths in project If use_cache is enabled all objects will stored in cache area """ start = time.time() if ccmpool: if ccmpool.nr_sessions == 1: result = get_objects_in_project_serial(project, ccm=ccmpool[0], database=database, use_cache=use_cache) else: result = get_objects_in_project_parallel(project, ccmpool=ccmpool, use_cache=use_cache) else: result = get_objects_in_project_serial(project, ccm=ccm, database=database, use_cache=use_cache) logger.debug("Time used fetching all objects and paths in %s: %d s.", project, time.time() - start) if use_cache: ccm_object = ccm_cache.get_object(project) ccm_object.set_members(result) ccm_cache.force_cache_update_for_object(ccm_object) return result
def get_master_tag(): f = open('config.p', 'rb') config = cPickle.load(f) f.close() object = ccm_cache.get_object(config['master']) tag = object.name + object.separator + object.version return tag
def do_query(next_in_queue, free_ccm, semaphore, delim, p_queue, use_cache): (project, parent_proj) = next_in_queue ccm_addr, database = get_and_lock_free_ccm_addr(free_ccm) ccm = SynergySession(database, ccm_addr=ccm_addr) ccm.keep_session_alive = True # logger.debug('Querying: %s' % project.get_object_name()) result = get_members(project, ccm, parent_proj) if use_cache: objects = [] na_obj = [] for item in result: try: objects.append(ccm_cache.get_object(item['objectname'], ccm)) except ccm_cache.ObjectCacheException: objects.append(SynergyObject(item['objectname'], delim)) na_obj.append(item['objectname']) if na_obj: logger.warning("Objects not avaliable in this db:") logger.warning(', '.join(na_obj)) else: objects = [SynergyObject(item['objectname'], delim) for item in result] # if a project is being queried it might have more than one dir with the # same name as the project associated, find the directory that has the # project associated as the directory's parent if project.get_type() == 'project': if len(objects) > 1: objects = find_root_project(project, objects, ccm) p_queue.put((project, objects)) entry = free_ccm[ccm_addr] entry['free'] = True free_ccm[ccm_addr] = entry semaphore.release()
def successor_is_released(self, predecessor, fileobject, recursion_depth): if recursion_depth > self.max_recursion_depth: return False recursion_depth += 1 logger.info("Checking if successor is released, for %s by predecessor %s" %(fileobject.get_object_name(), predecessor.get_object_name())) ret_val = False successors = predecessor.get_successors() for s in successors: if s in self.release_lookup.keys(): return self.release_lookup[s] if s != fileobject.get_object_name(): try: successor = ccm_cache.get_object(s, self.ccm) except ccm_cache.ObjectCacheException as e: logger.warning(e.value) return False logger.info("successor: %s" % successor.get_object_name()) #check releases of successor releases = successor.get_releases() if [r for r in releases if r in self.old_subproject_list]: logger.info("successor: %s is released" % successor.get_object_name()) self.release_lookup[successor.get_object_name()] = True return True elif [r for r in releases if r in self.current_subproject_list]: logger.info("successor: %s is released in current project, don't continue" % successor.get_object_name()) self.release_lookup[successor.get_object_name()] = False return False else: ret_val = self.successor_is_released(successor, fileobject, recursion_depth) self.release_lookup[successor.get_object_name()] = ret_val else: # if there is only one successor and it is the fileobject assume it to be released if the predecessor is in released state if len(successors) == 1 and predecessor.get_status() == 'released': return True return ret_val
def create_release_graph(objects, release, previous): release_graph = hypergraph() release_graph.add_nodes(objects) release_graph.add_edges([release, previous]) file_objects = [ccm_cache.get_object(o) for o in objects] for o in file_objects: link = True # Bind objects to this release successors = o.get_successors() if successors: for s in successors: if s not in release_graph.nodes(): link &= True else: link &= False if link: if o.get_object_name() not in release_graph.links(release): release_graph.link(o.get_object_name(), release) # Bind objects to previous release predecessors = o.get_predecessors() if predecessors is not None: for p in predecessors: if p not in objects: if not release_graph.has_node(p): release_graph.add_node(p) #print "linking", p, "to release", previous release_graph.link(p, previous) return release_graph
def update_task_in_history_with_objects(self, task, objects): # Find the task in history if its there found = False for t in self.history[self.tag]['tasks']: if t.get_object_name() == task: if t.objects: t.objects.extend( [o for o in objects if ':project:' not in o]) t.objects = list(set(t.objects)) else: t.objects = [o for o in objects if ':project:' not in o] found = True if not found: t = ccm_cache.get_object(task, self.ccm) if t.type == 'task': t.objects = [o for o in objects if ':project:' not in o] self.history[self.tag]['tasks'].append(t) elif t.type == 'dir': # Make a task from the directory object task_obj = TaskObject( "%s%s%s:task:%s" % (t.name, self.delim, t.version, t.instance), self.delim, t.author, t.status, t.created_time, t.tasks) task_obj.set_attributes(t.attributes) task_obj.complete_time = t.get_integrate_time() task_obj.objects = [o for o in objects if ':project:' not in o] self.history[self.tag]['tasks'].append(task_obj)
def create_release_graph(objects, release, previous): release_graph = hypergraph() release_graph.add_nodes(objects) release_graph.add_edges([release, previous]) file_objects = [ccm_cache.get_object(o) for o in objects] for o in file_objects: link = True # Bind objects to this release successors = o.get_successors() if successors: for s in successors: if s not in release_graph.nodes(): link &= True else: link &=False if link: if o.get_object_name() not in release_graph.links(release): release_graph.link(o.get_object_name(), release) # Bind objects to previous release predecessors = o.get_predecessors() if predecessors is not None: for p in predecessors: if p not in objects: if not release_graph.has_node(p): release_graph.add_node(p) #print "linking", p, "to release", previous release_graph.link(p, previous) return release_graph
def populate_cache_with_project_and_members(project, ccm, ccmpool): print "Loading object %s" % project project_obj = ccm_cache.get_object(project, ccm=ccm) #assuming no project.members objects_in_project = ccm_objects_in_project.get_objects_in_project(project, ccm=ccm, ccmpool=ccmpool, use_cache=True) project_obj.set_members(objects_in_project) ccm_cache.force_cache_update_for_object(project_obj)
def project_is_some_predecessor(self, project, recursion_depth): if recursion_depth > self.max_recursion_depth: return False recursion_depth += 1 logger.info("Checking if %s is some predecessor of %s or %s " % (project.get_object_name(), self.current_release, self.old_release)) successors = project.get_baseline_successor() for successor in successors: logger.info("successor: %s" % successor) if successor in self.old_subproject_list: logger.info("Found %s in previous subprojects" % successor) return True elif successor in self.current_subproject_list: logger.info("Found %s in current subprojects" % successor) return True else: try: successor = ccm_cache.get_object(successor, self.ccm) except ccm_cache.ObjectCacheException: return False if self.project_is_some_predecessor(successor, recursion_depth): return True return False
def update_project_with_members(project, ccm, ccmpool): print "Loading object %s" % project project_obj = ccm_cache.get_object(project, ccm=ccm) if not project_obj.members: objects_in_project = ccm_objects_in_project.get_objects_in_project(project, ccm=ccm, ccmpool=ccmpool) project_obj.set_members(objects_in_project) ccm_cache.force_cache_update_for_object(project_obj)
def populate_cache_with_project_and_members(project, ccm, ccmpool): print "Loading object %s" % project project_obj = ccm_cache.get_object(project, ccm=ccm) #assuming no project.members objects_in_project = ccm_objects_in_project.get_objects_in_project( project, ccm=ccm, ccmpool=ccmpool, use_cache=True) project_obj.set_members(objects_in_project) ccm_cache.force_cache_update_for_object(project_obj)
def update_project_with_members(project, ccm, ccmpool): print "Loading object %s" % project project_obj = ccm_cache.get_object(project, ccm=ccm) if not project_obj.members: objects_in_project = ccm_objects_in_project.get_objects_in_project( project, ccm=ccm, ccmpool=ccmpool) project_obj.set_members(objects_in_project) ccm_cache.force_cache_update_for_object(project_obj)
def get_project_chain(head_project, base_project, ccm): # Do it from head to base baseline = ccm_cache.get_object(head_project, ccm) chain = [baseline.get_object_name()] while baseline.get_object_name() != base_project: predecessor = ccm_cache.get_object(baseline.predecessors[0], ccm) baseline = ccm_cache.get_object(baseline.baseline_predecessor[0], ccm) if baseline: chain.append(baseline.get_object_name()) elif predecessor: chain.append(predecessor.get_object_name()) baseline = predecessor else: break # reverse the list to get the base first chain.reverse() return chain
def get_objects_from_graph(task, graph, objects): objs = [] for o in graph.links(task): for obj in objects: if obj == o: objs.append(obj) break return [ccm_cache.get_object(o) for o in objs]
def get_project_chain(head_project, base_project, ccm): # Do it from head to base baseline = ccm_cache.get_object(head_project, ccm) chain = [baseline.get_object_name()] while baseline.get_object_name() != base_project: if baseline.predecessors: predecessor = ccm_cache.get_object(baseline.predecessors[0], ccm) if baseline.baseline_predecessor: baseline = ccm_cache.get_object(baseline.baseline_predecessor[0], ccm) if baseline: chain.append(baseline.get_object_name()) elif predecessor: chain.append(predecessor.get_object_name()) baseline = predecessor else: break # reverse the list to get the base first chain.reverse() return chain
def populate_cache_with_objects_from_project(project, ccm, ccmpool): print "processing project %s" %project #first try to get the object from cache project_obj = ccm_cache.get_object(project, ccm) if not project_obj.members: update_project_with_members(project, ccm, ccmpool) na_obj =[] if project_obj.members: num_o = len(project_obj.members.keys()) for o in project_obj.members.keys(): print "loading object: %s" % o try: obj = ccm_cache.get_object(o, ccm) except ccm_cache.ObjectCacheException: na_obj.append(o) num_o -=1 print "objects left %d" %num_o print "%s done, members: %d" %(project, len(project_obj.members.keys())) print "Objects not avaliable in this db:" print '\n'.join(na_obj)
def get_new_dirs(self, members, new_objects): new_directories = [d for d in new_objects.keys() if ':dir:' in d] directories = [] for dir in new_directories: dir_obj = ccm_cache.get_object(dir, self.ccm) new_dirs = [d for d in dir_obj.new_objects if d.endswith('/')] for d in new_dirs: # Get the corresponding directory four-part-name paths = members[dir] name = get_dir_with_path(paths, d, members) directories.append(name) print "new directories found:" print '\n'.join([d + ', '.join(members[d]) for d in directories]) return directories
def find_tasks_from_objects(self, objects, project): tasks = {} unconfirmed_tasks = {} if self.tag in self.history.keys(): if 'tasks' in self.history[self.tag]: for t in self.history[self.tag]['tasks']: logger.info("loading old task: %s" % t.get_object_name()) tasks[t.get_object_name()] = t #Build a list of all tasks task_list = [task for o in objects for task in ccm_cache.get_object(o, self.ccm).get_tasks()] num_of_tasks = len(set(task_list)) logger.info("Tasks with associated objects: %i" % num_of_tasks) task_util = TaskUtil(self.ccm) for t in set(task_list)-set(tasks.keys()): try: task = ccm_cache.get_object(t, self.ccm) except ccm_cache.ObjectCacheException: continue if task.status == 'completed': if task_util.task_in_project(task, project): tasks[task.get_object_name()] = task else: # try to add it anyway to see what happens... unconfirmed_tasks[task.get_object_name()] = task # Add objects to tasks [t.add_object(o) for o in objects for t in tasks.values() if t.get_object_name() in ccm_cache.get_object(o, self.ccm).get_tasks()] # Add objects to unconfirmed tasks [t.add_object(o) for o in objects for t in unconfirmed_tasks.values() if t.get_object_name() in ccm_cache.get_object(o, self.ccm).get_tasks()] logger.info("Sanitizing tasks") tasks = sanitize_tasks(tasks, unconfirmed_tasks) logger.info("No of tasks in release %s: %i" % (project.get_object_name(), len(tasks.keys()))) self.history[self.tag]['tasks'] = tasks.values() fname = self.outputfile + '_' + self.tag + '_inc' self.persist_data(fname, self.history[self.tag])
def find_task_from_dirs(self, dirs): tasks = {} for dir in dirs: logger.info("Finding directory %s in tasks" % dir) # Try to get the task from the tasks already found obj = ccm_cache.get_object(dir, self.ccm) tmp_task = obj.get_tasks() task = list(set(tmp_task) & set([t.get_object_name() for t in self.history[self.tag]['tasks']])) if not task: # Use object name as task name task = [dir] logger.info("task found: %s" % ','.join(task)) tasks[dir]= task return tasks
def get_new_dirs(self, members, new_objects): new_directories = [d for d in new_objects.keys() if ':dir:' in d] directories = [] for dir in new_directories: dir_obj = ccm_cache.get_object(dir, self.ccm) new_dirs = [d for d in dir_obj.new_objects if d.endswith('/')] for d in new_dirs: # Get the corresponding directory four-part-name paths = members[dir] name = get_dir_with_path(paths, d, members) if name: directories.append(name) logger.info("new directories found:") logger.info(' ,'.join([d + ', '.join(members[d]) for d in directories])) return directories
def create_file_list(objects, lookup, ccm_types, project, empty_dirs=None, empty_dir_mark=None, all_files_for_release=False): global object_mark_lookup project_object = ccm_cache.get_object(project) paths = project_object.get_members() l = [] for o in objects: if o.get_type() != 'dir': perm = ccm_types[o.get_type()] object_paths = get_object_paths(o, paths) for p in object_paths: l.append('M ' + perm + ' :' + str(lookup[o.get_object_name()]) + ' ' + p) else: #Get deleted items deleted = o.get_deleted_objects() if deleted: for d in deleted: object_paths = get_object_paths(o, paths) for p in object_paths: # p is the path of the directory if d.endswith('/'): tmp = d.rsplit('/', 1)[0] else: tmp = d l.append('D ' + p + '/' + tmp) if all_files_for_release: # delete top level dir first: l.append('D ' + project_object.name) # Insert all files in the release logger.info("Loading all objects for %s" %project_object.get_object_name()) for object, paths in project_object.members.iteritems(): if not ':dir:' in object and not ':project:' in object: perm = ccm_types[object.split(':')[1]] for p in paths: l.append('M ' + perm + ' :' + str(object_mark_lookup[object]) + ' ' + p) if empty_dirs: for d in empty_dirs: if empty_dir_mark: path = d + '/.gitignore' l.append('M 100644 :' + str(empty_dir_mark) + ' ' + path) if not l: return [] return '\n'.join(l)
def successor_is_released(self, predecessor, fileobject, recursion_depth): if recursion_depth > self.max_recursion_depth: return False recursion_depth += 1 logger.info( "Checking if successor is released, for %s by predecessor %s" % (fileobject.get_object_name(), predecessor.get_object_name())) ret_val = False successors = predecessor.get_successors() for s in successors: if s in self.release_lookup.keys(): return self.release_lookup[s] if s != fileobject.get_object_name(): try: successor = ccm_cache.get_object(s, self.ccm) except ccm_cache.ObjectCacheException as e: logger.warning(e.value) return False logger.info("successor: %s" % successor.get_object_name()) #check releases of successor releases = successor.get_releases() if [r for r in releases if r in self.old_subproject_list]: logger.info("successor: %s is released" % successor.get_object_name()) self.release_lookup[successor.get_object_name()] = True return True elif [ r for r in releases if r in self.current_subproject_list ]: logger.info( "successor: %s is released in current project, don't continue" % successor.get_object_name()) self.release_lookup[successor.get_object_name()] = False return False else: ret_val = self.successor_is_released( successor, fileobject, recursion_depth) self.release_lookup[successor.get_object_name()] = ret_val else: # if there is only one successor and it is the fileobject assume it to be released if the predecessor is in released state if len(successors) == 1 and predecessor.get_status( ) == 'released': return True return ret_val
def get_objects_in_project_parallel(project, ccmpool=None, use_cache=False): """ Get all the objects and paths of project with use of multiple ccm sessions """ mgr = Manager() free_ccm = mgr.dict() for ccm in ccmpool.sessionArray.values(): free_ccm[ccm.getCCM_ADDR()] = {'free': True, 'database': ccm.database} ccm_addr, database = get_and_lock_free_ccm_addr(free_ccm) ccm = SynergySession(database, ccm_addr=ccm_addr) ccm.keep_session_alive = True delim = ccm.delim() semaphore = mgr.Semaphore(ccmpool.nr_sessions) # Starting project if use_cache: start_object = ccm_cache.get_object(project, ccm) else: start_object = SynergyObject(project, delim) # unlock update dict entry to inform manager entry = free_ccm[ccm_addr] entry['free'] = True free_ccm[ccm_addr] = entry p_queue = mgr.Queue() c_queue = mgr.Queue() c_queue.put((start_object, None)) p_queue.put(start_object) # start the produce and consumer thread prod = Process(target=producer, args=(c_queue, p_queue, free_ccm)) cons = Process(target=consumer, args=(c_queue, p_queue, free_ccm, semaphore, delim, use_cache)) prod.start() cons.start() logger.debug("Waiting to join") cons.join() hierarchy = p_queue.get() prod.join() return hierarchy
def check_successor_chain_for_object(self, fileobject, old_object, recursion_depth): if recursion_depth > self.max_recursion_depth: return False recursion_depth += 1 logger.info("Checking if successor chain for %s contains %s" % (fileobject.get_object_name(), old_object.get_object_name())) ret_val = False successors = fileobject.get_successors() for s in successors: if s == old_object.get_object_name(): return True try: successor = ccm_cache.get_object(s, self.ccm) except ccm_cache.ObjectCacheException: return False logger.info("successor: %s" %successor.get_object_name()) ret_val = self.check_successor_chain_for_object(successor, old_object, recursion_depth) if ret_val: break return ret_val
def find_task_from_dirs(self, dirs): tasks = {} for dir in dirs: logger.info("Finding directory %s in tasks" % dir) # Try to get the task from the tasks already found obj = ccm_cache.get_object(dir, self.ccm) tmp_task = obj.get_tasks() task = list( set(tmp_task) & set([ t.get_object_name() for t in self.history[self.tag]['tasks'] ])) if not task: # Use object name as task name task = [dir] logger.info("task found: %s" % ','.join(task)) tasks[dir] = task return tasks
def project_is_some_predecessor(self, project, recursion_depth): if recursion_depth > self.max_recursion_depth: return False recursion_depth += 1 logger.info("Checking if %s is some predecessor of %s or %s " %(project.get_object_name(), self.current_release, self.old_release)) successors = project.get_baseline_successor() for successor in successors: logger.info("successor: %s" % successor) if successor in self.old_subproject_list: logger.info("Found %s in previous subprojects"% successor) return True elif successor in self.current_subproject_list: logger.info("Found %s in current subprojects" % successor) return True else: try: successor = ccm_cache.get_object(successor, self.ccm) except ccm_cache.ObjectCacheException: return False if self.project_is_some_predecessor(successor, recursion_depth): return True return False
def project_is_some_predecessor(self, project, recursion_depth): if recursion_depth > self.max_recursion_depth: return False recursion_depth += 1 print "Checking if", project.get_object_name(), "is some predecessor of", self.current_release, "or", self.old_release, "..." successors = project.get_baseline_successor() for successor in successors: print "successor:", successor if successor in self.old_subproject_list: print "Found", successor, "in previous subprojects" return True elif successor in self.current_subproject_list: print "Found", successor, "in current subprojects" return True else: try: successor = ccm_cache.get_object(successor, self.ccm) except ccm_cache.ObjectCacheException: return False if self.project_is_some_predecessor(successor, recursion_depth): return True return False
def update_task_in_history_with_objects(self, task, objects): # Find the task in history if its there found = False for t in self.history[self.tag]['tasks']: if t.get_object_name() == task: if t.objects: t.objects.extend([o for o in objects if ':project:' not in o]) t.objects = list(set(t.objects)) else: t.objects = [o for o in objects if ':project:' not in o] found = True if not found: t = ccm_cache.get_object(task, self.ccm) if t.type == 'task': t.objects = [o for o in objects if ':project:' not in o] self.history[self.tag]['tasks'].append(t) elif t.type == 'dir': # Make a task from the directory object task_obj = TaskObject("%s%s%s:task:%s" %(t.name, self.delim, t.version, t.instance), self.delim, t.author, t.status, t.created_time, t.tasks) task_obj.set_attributes(t.attributes) task_obj.complete_time = t.get_integrate_time() task_obj.objects = [o for o in objects if ':project:' not in o] self.history[self.tag]['tasks'].append(task_obj)
def check_successor_chain_for_object(self, fileobject, old_object, recursion_depth): if recursion_depth > self.max_recursion_depth: return False recursion_depth += 1 logger.info( "Checking if successor chain for %s contains %s" % (fileobject.get_object_name(), old_object.get_object_name())) ret_val = False successors = fileobject.get_successors() for s in successors: if s == old_object.get_object_name(): return True try: successor = ccm_cache.get_object(s, self.ccm) except ccm_cache.ObjectCacheException: return False logger.info("successor: %s" % successor.get_object_name()) ret_val = self.check_successor_chain_for_object( successor, old_object, recursion_depth) if ret_val: break return ret_val
def create_object_graph(objects): object_graph = digraph() object_graph.add_nodes(objects) file_objects = [] for o in objects: file_objects.append(ccm_cache.get_object(o)) for o in file_objects: for p in o.get_predecessors(): if not object_graph.has_node(p): object_graph.add_node(p) # Create relationship list for o in file_objects: # Bind objects to previous release predecessors = o.get_predecessors() if predecessors: for p in predecessors: object_graph.add_edge((p, o.get_object_name())) return object_graph
def create_object_graph(objects): object_graph = digraph() object_graph.add_nodes(objects) file_objects = [] for o in objects: file_objects.append(ccm_cache.get_object(o)) for o in file_objects: for p in o.get_predecessors(): if not object_graph.has_node(p): object_graph.add_node(p) # Create relationship list for o in file_objects: # Bind objects to previous release predecessors = o.get_predecessors() if predecessors: for p in predecessors: object_graph.add_edge((p,o.get_object_name())) return object_graph
def populate_cache_with_objects_from_project(project, ccm, ccmpool): print "processing project %s" % project #first try to get the object from cache project_obj = ccm_cache.get_object(project, ccm) if not project_obj.members: populate_cache_with_project_and_members(project, ccm, ccmpool)
def ccm_fast_export(releases, graphs): global acn_ancestors global users users = users() logger.basicConfig(filename='ccm_fast_export.log',level=logger.DEBUG) commit_lookup = {} # Get the initial release for k, v in releases.iteritems(): if k == 'delimiter': continue if k == 'ccm_types': continue if v['previous'] is None: release = k break logger.info("Starting at %s as initial release" % release) if 'created' not in releases[release]: initial_release_time = 0.0 # epoch for now since releases[release] has no 'created' key :( else: initial_release_time = time.mktime(releases[release]['created'].timetuple()) mark = 0 files = [] # Create the initial release # get all the file objects: file_objects = (ccm_cache.get_object(o) for o in releases[release]['objects']) project_obj = ccm_cache.get_object(releases[release]['fourpartname']) paths = project_obj.get_members() for o in file_objects: if o.get_type() != 'dir': object_mark, mark = create_blob(o, mark) for p in paths[o.get_object_name()]: files.append('M ' + releases['ccm_types']['permissions'][o.get_type()] + ' :'+str(object_mark) + ' ' + p) empty_dirs = releases[release]['empty_dirs'] logger.info("Empty dirs for release %s\n%s" %(release, empty_dirs)) mark = create_blob_for_empty_dir(get_mark(mark)) #file_list = create_file_list(objects, object_lookup, releases['ccm_types'], empty_dirs=empty_dirs, empty_dir_mark=mark) if empty_dirs: for d in empty_dirs: if mark: path = d + '/.gitignore' files.append('M 100644 :' + str(mark) + ' ' + path) mark = get_mark(mark) commit_info = ['reset refs/tags/' + release, 'commit refs/tags/' + release, 'mark :' + str(mark), 'author Nokia <*****@*****.**> ' + str(int(initial_release_time)) + " +0000", 'committer Nokia <*****@*****.**> ' + str(int(initial_release_time)) + " +0000", 'data 15', 'Initial commit', '\n'.join(files), ''] print '\n'.join(commit_info) logger.info("git-fast-import:\n%s" %('\n'.join(commit_info))) tag_msg = 'Release: %s' %release annotated_tag = ['tag %s' % release, 'from :%s' % str(mark), 'tagger Nokia <*****@*****.**> ' + str(int(initial_release_time)) + " +0000", 'data %s' % len(tag_msg), tag_msg] print '\n'.join(annotated_tag) commit_lookup[release] = mark # do the following releases (graphs) release_queue = deque(releases[release]['next']) while release_queue: release = release_queue.popleft() previous_release = releases[release]['previous'] logger.info("Next release: %s" % release) commit_graph = graphs[release]['commit'] commit_graph = fix_orphan_nodes(commit_graph, previous_release) commit_graph = ch.spaghettify_digraph(commit_graph, previous_release, release) #htg.commit_graph_to_image(commit_graph, releases[release], graphs[release]['task'], name=releases[release]['name']+'_after' ) # Find the cutting nodes logger.info("Finding the cutting nodes") undirected = graph() undirected.add_nodes(commit_graph.nodes()) [undirected.add_edge(edge) for edge in commit_graph.edges()] cutting_nodes = cut_nodes(undirected) del undirected # Create the reverse commit graph logger.info("Building the reverse commit graph") reverse_commit_graph = commit_graph.reverse() # Compute the accessibility matrix of the reverse commit graph logger.info("Compute the ancestors") ancestors = accessibility(reverse_commit_graph) del reverse_commit_graph logger.info("Ancestors of the release: %s" % str(ancestors[release])) # Clean up the ancestors matrix for k, v in ancestors.iteritems(): if k in v: v.remove(k) # Get the commits order commits = topological_sorting(commit_graph) # Fix the commits order list commits.remove(previous_release) commits.remove(release) last_cutting_node = None # Check if the release (Synergy project has changed name, if it has the # 'base' directory name needs to be renamed if releases.has_key('delimiter'): delim = releases['delimiter'] else: delim = '-' previous_name = previous_release.split(delim)[0] current_name = release.split(delim)[0] if current_name != previous_name: logger.info("Name changed: %s -> %s" %(previous_name, current_name)) from_mark = commit_lookup[previous_release] mark, commit = rename_toplevel_dir(previous_name, current_name, release, releases, mark, from_mark) print '\n'.join(commit) # adjust the commit lookup commit_lookup[previous_release] = mark for counter, commit in enumerate(commits): logger.info("Commit %i/%i" % (counter+1, len(commits))) acn_ancestors = [] if last_cutting_node is not None: acn_ancestors = ancestors[last_cutting_node] # Create the references lists. It lists the parents of the commit #reference = [commit_lookup[parent] for parent in ancestors[commit] if parent not in acn_ancestors] reference = [commit_lookup[parent] for parent in commit_graph.incidents(commit)] if len(reference) > 1: # Merge commit mark = create_merge_commit(commit, release, releases, mark, reference, graphs, set(ancestors[commit]) - set(acn_ancestors)) else: # Normal commit mark = create_commit(commit, release, releases, mark, reference, graphs) # Update the lookup table commit_lookup[commit] = mark # Update the last cutting edge if necessary if commit in cutting_nodes: last_cutting_node = commit if last_cutting_node is not None: acn_ancestors = ancestors[last_cutting_node] reference = [commit_lookup[parent] for parent in ancestors[release] if parent not in acn_ancestors] logger.info("Reference %s" %str([parent for parent in ancestors[release] if parent not in acn_ancestors])) if not reference: logger.info("Reference previous %s, mark: %d" % (releases[release]['previous'], commit_lookup[releases[release]['previous']])) reference = [commit_lookup[ releases[release]['previous'] ] ] mark, merge_commit = create_release_merge_commit(releases, release, get_mark(mark), reference, graphs, set(ancestors[release]) - set(acn_ancestors)) print '\n'.join(merge_commit) annotated_tag = create_annotated_tag(releases, release, mark) print '\n'.join(annotated_tag) commit_lookup[release] = mark release_queue.extend(releases[release]['next']) #release = releases[release]['next'] #release = None #reset to master master = get_master_tag() reset = ['reset refs/heads/master', 'from :' + str(commit_lookup[master])] logger.info("git-fast-import:\n%s" %('\n'.join(reset))) print '\n'.join(reset)
def recursive_get_history(self, fileobject, recursion_depth): """ Recursivly find the history of the file object, optionally stopping at the 'old_release' project """ next_iter = False logger.info('Processing: %s %s' % (fileobject.get_object_name(), fileobject.get_status())) logger.info('Recursion depth %d' % recursion_depth) retval = True #Check if recursion_depth is reached if recursion_depth > self.max_recursion_depth: logger.warning('Giving up on %s' % fileobject.get_object_name()) return False recursion_depth += 1 predecessors = fileobject.get_predecessors() for p in predecessors: try: predecessor = ccm_cache.get_object(p, self.ccm) except ccm_cache.ObjectCacheException: # Object couldn't be retrived from ccm, give up on this logger.info("Couldn't get %s from Synergy" % p) return True logger.info("Predecessor: %s" % predecessor.get_object_name()) # check predecessor release to see if this object should be added to the set. if self.old_release: # Get the release(s) for the predecessor releases = predecessor.get_releases() if releases: # Check if the "old" release is the the releases for the predecessor and stop if true if [ r for r in releases if r in self.current_subproject_list or r in self.old_subproject_list ]: # Object is already released, continue with the next predecessor logger.info("%s is already released" % predecessor.get_object_name()) continue logger.info( "Couldn't find release in current_subproject_list or old_subproject_list" ) #Check if chain of successors contains previous object - if true discard the chain if self.successor_is_released(predecessor, fileobject, 0): logger.info("Successor is already released %s" % fileobject.get_object_name()) continue #Check if projects are releated to old release. Latest first for r in releases: try: project = ccm_cache.get_object(r, self.ccm) except ccm_cache.ObjectCacheException: continue if self.project_is_some_predecessor(project, 0): logger.info( "Found Relationship between: %s and %s " % (project.get_object_name(), self.old_release)) next_iter = True break if next_iter: continue else: #Check if a successor is released if self.successor_is_released(predecessor, fileobject, 0): logger.info("Successor is already released %s " % fileobject.get_object_name()) continue # Check if predecessor is already added to history - if so add this as successor to fileobject, else add new predecessor to history if not self.temp_history.has_key(predecessor.get_object_name()): logger.info( "Adding %s %s to history" % (predecessor.get_object_name(), predecessor.get_status())) self.add_to_history(predecessor) retval &= self.recursive_get_history(predecessor, recursion_depth) if not retval: # Giving up on history break out of loop break return retval
def get_project_history(self, latest_project, base_project): release_chain = deque( get_project_chain(latest_project, base_project, self.ccm)) base_project = release_chain.popleft() baseline_project = ccm_cache.get_object(base_project, self.ccm) self.tag = baseline_project.get_name( ) + self.delim + baseline_project.get_version() # Initialize history if self.tag not in self.history.keys(): self.history[self.tag] = {'objects': [], 'tasks': []} self.history['delimiter'] = self.delim self.history[self.tag]['previous'] = None #if self.history[self.tag].has_key('next'): # if not release_chain[0] in self.history[self.tag]['next']: # self.history[self.tag]['next'].append(release_chain[0]) #else: # self.history[self.tag]['next']= [release_chain[0]] logger.info("getting all objects for: %s ... " % self.tag) self.baseline_objects = None # Do the first project as a full project self.find_project_diff(baseline_project, None) self.history[self.tag]['name'] = self.tag self.history[ self.tag]['fourpartname'] = baseline_project.get_object_name() self.history[self.tag]['created'] = baseline_project.get_created_time() self.history[self.tag]['author'] = baseline_project.get_author() # Add the objects and paths to the history, to be used for finding empty directories empty_dirs = find_empty_dirs(self.baseline_objects) self.history[self.tag]['empty_dirs'] = empty_dirs while release_chain: next = release_chain.popleft() next_project = ccm_cache.get_object(next, self.ccm) # Set next project for the current baseline_project if self.history[self.tag].has_key('next'): if not next_project.get_name( ) + self.delim + next_project.get_version() in self.history[ self.tag]['next']: self.history[self.tag]['next'].append( next_project.get_name() + self.delim + next_project.get_version()) else: self.history[self.tag]['next'] = [ next_project.get_name() + self.delim + next_project.get_version() ] # Info about baseline_project logger.info("%s done processing, Info:" % self.tag) logger.info("Name %s" % self.tag) logger.info("4partname %s" % self.history[self.tag]['fourpartname']) logger.info("Number of: ") logger.info(" Tasks: %i" % len(self.history[self.tag]['tasks'])) logger.info(" Files: %i" % len(self.history[self.tag]['objects'])) logger.info("Previous <- %s" % self.history[self.tag]['previous']) logger.info("Next -> %s" % str(self.history[self.tag]['next'])) logger.info("") self.tag = next_project.get_name( ) + self.delim + next_project.get_version() logger.info("Next project: %s" % next_project.get_object_name()) if self.tag not in self.history.keys(): self.history[self.tag] = {'objects': [], 'tasks': []} self.history[self.tag]['previous'] = baseline_project.get_name( ) + self.delim + baseline_project.get_version() logger.info("Toplevel Project: %s " % next_project.get_object_name()) # do the history thing self.history[self.tag]['name'] = self.tag self.history[ self.tag]['fourpartname'] = next_project.get_object_name() self.find_project_diff(baseline_project, next_project) self.history[self.tag]['created'] = next_project.get_created_time() self.history[self.tag]['author'] = next_project.get_author() # Add the objects and paths to the history, to be used for finding empty directories empty_dirs = find_empty_dirs(self.project_objects) logger.info("Empty dirs:\n%s" % '\n'.join(sorted(empty_dirs))) self.history[self.tag]['empty_dirs'] = empty_dirs # set new baseline project baseline_project = next_project # set the baseline_objects for the next iteration and self.baseline_objects = self.project_objects self.project_objects = [] #Store data fname = self.outputfile + '_' + self.tag self.persist_data(fname, self.history[self.tag]) self.history_created.append(fname) # delete the _inc file if it exists if os.path.isfile(fname + '_inc' + '.p'): os.remove(fname + '_inc' + '.p') # Info for last project: if not self.history[self.tag].has_key('next'): self.history[self.tag]['next'] = [] logger.info("%s done processing, Info: " % self.tag) logger.info("Name %s" % self.tag) logger.info("4partname %s" % self.history[self.tag]['fourpartname']) logger.info("Number of: ") logger.info(" Tasks: %i" % len(self.history[self.tag]['tasks'])) logger.info(" Files: %i" % len(self.history[self.tag]['objects'])) logger.info("Previous <- %s" % self.history[self.tag]['previous']) logger.info("Next -> %s" % str(self.history[self.tag]['next'])) logger.info("") # Get the different types in the db and their corresponding file permissions and super types ccm_types_perms = ccm_type.get_types_and_permissions(self.ccm) ccm_super_types = ccm_type.get_super_types(self.ccm) if not self.history.has_key('ccm_types'): self.history['ccm_types'] = {} self.history['ccm_types']['permissions'] = ccm_types_perms self.history['ccm_types']['super_types'] = ccm_super_types return self.history
def populate_cache_with_objects_from_project(project, ccm, ccmpool): print "processing project %s" %project #first try to get the object from cache project_obj = ccm_cache.get_object(project, ccm) if not project_obj.members: populate_cache_with_project_and_members(project, ccm, ccmpool)
def get_object(o, objects): for obj in objects: if obj == o: return ccm_cache.get_object(obj)
def find_project_diff(self, baseline_project, next_project): # Get all objects and paths for baseline_project if not self.baseline_objects: self.baseline_objects = baseline_project.get_members() if self.baseline_objects is None or len(self.baseline_objects) == 1 or not isinstance(self.baseline_objects, dict): self.baseline_objects = ccm_objects.get_objects_in_project(baseline_project.get_object_name(), ccmpool=self.ccmpool) baseline_project.set_members(self.baseline_objects) ccm_cache.force_cache_update_for_object(baseline_project) if next_project: # Get all objects and paths for next project self.project_objects = next_project.get_members() if self.project_objects is None or len(self.project_objects) == 1 or not isinstance(self.project_objects, dict): self.project_objects = ccm_objects.get_objects_in_project(next_project.get_object_name(), ccmpool=self.ccmpool) next_project.set_members(self.project_objects) ccm_cache.force_cache_update_for_object(next_project) # Find difference between baseline_project and next_project new_objects, old_objects = get_changed_objects(self.baseline_objects, self.project_objects) next_projects = [o for o in self.project_objects.keys() if ':project:' in o] baseline_projects = [o for o in self.baseline_objects.keys() if ':project:' in o] object_history = ObjectHistory(self.ccm, next_project.get_object_name(), old_objects=old_objects, old_release=baseline_project.get_object_name(), new_projects=next_projects, old_projects=baseline_projects) else: # root project, get ALL objects in release new_objects = self.baseline_objects old_objects = [] object_history = ObjectHistory(self.ccm, baseline_project.get_object_name()) num_of_objects = len([o for o in new_objects.keys() if ":project:" not in o]) logger.info("objects to process : %i" % num_of_objects) objects = [] if self.tag in self.history.keys(): if 'objects' in self.history[self.tag]: #Add all existing objects for o in self.history[self.tag]['objects']: objects.append(o) #objects[o] = ccm_cache.get_object(o, self.ccm) logger.info("no of old objects loaded %i", len(objects)) else: self.history[self.tag] = {'objects': [], 'tasks': []} object_names = set([o for o in new_objects.keys() if ':project:' not in o]) - set(objects) for o in object_names: object = ccm_cache.get_object(o, self.ccm) if next_project: # get the object history between releases history = object_history.get_history(object, new_objects[object.get_object_name()]) objects.extend(history.keys()) #objects.update(object_history.get_history(object, new_objects[object.get_object_name()])) else: # just get all the objects in the release logger.info('Processing: %s path: %s' %(object.get_object_name(), str(new_objects[object.get_object_name()]))) #object.set_path(new_objects[object.get_object_name()]) objects.append(o) #objects[object.get_object_name()] = object num_of_objects -=1 logger.info('Objects left: %i' %num_of_objects) objects = list(set(objects)) logger.info("number of files: %i" % len(objects)) self.history[self.tag]['objects'] = objects # Create tasks from objects, but not for initial project if next_project: self.find_tasks_from_objects(objects, next_project) # Handle new projects: if next_project: new_created_projects = get_new_projects(old_objects, new_objects, self.delim) dir_lookup ={} # create lookup for path to directory-4-part-name {path : dir-name} for k,v in new_objects.iteritems(): if ':dir:' in k: for i in v: dir_lookup[i] = k project_dirs = [dir for project in new_created_projects for dir in new_objects[project]] directories = [d for k,v in new_objects.iteritems() for d in v if ':dir:' in k] changed_directories = set(directories).intersection(set(project_dirs)) changed_directories = remove_subdirs_under_same_path(changed_directories) dirs = [dir_lookup[d] for d in changed_directories] # find task and add all objects to the task, which shares the path. project_tasks = self.find_task_from_dirs(dirs) logger.info("Checking for new subprojects") # check directories for new subdirectories and add their content directories = self.get_new_dirs(self.project_objects, new_objects) # Limit directories to only directories not already processed as a new project directories = set(directories) - set(dirs) # find task and add all objects to the task, which shares the path. dir_tasks = self.find_task_from_dirs(directories) # merge project and dir tasks for k,v in dir_tasks.iteritems(): if not project_tasks.has_key(k): project_tasks[k] = v # if real synergy tasks isn't found check the path of the directories and skip possible subdirs tasks = self.reduce_dir_tasks(project_tasks) logger.info("Project and dir tasks reduced...") logger.info("%s" % str(tasks)) self.update_tasks_with_directory_contens(tasks) # remove possible duplicates from objects self.history[self.tag]['objects'] = list(set(self.history[self.tag]['objects']))
def ccm_fast_export(releases, graphs): global acn_ancestors global users users = users() logger.basicConfig(filename='ccm_fast_export.log',level=logger.DEBUG) commit_lookup = {} # Get the initial release for k, v in releases.iteritems(): if k == 'delimiter': continue if k == 'ccm_types': continue if v['previous'] is None: release = k break logger.info("Starting at %s as initial release" % release) if 'created' not in releases[release]: initial_release_time = 0.0 # epoch for now since releases[release] has no 'created' key :( else: initial_release_time = time.mktime(releases[release]['created'].timetuple()) mark = 0 files = [] # Create the initial release # get all the file objects: file_objects = [ccm_cache.get_object(o) for o in releases[release]['objects']] project_obj = ccm_cache.get_object(releases[release]['fourpartname']) paths = project_obj.get_members() for o in file_objects: if o.get_type() != 'dir': object_mark, mark = create_blob(o, mark) for p in paths[o.get_object_name()]: files.append('M ' + releases['ccm_types']['permissions'][o.get_type()] + ' :'+str(object_mark) + ' ' + p) empty_dirs = releases[release]['empty_dirs'] logger.info("Empty dirs for release %s\n%s" %(release, empty_dirs)) mark = create_blob_for_empty_dir(get_mark(mark)) #file_list = create_file_list(objects, object_lookup, releases['ccm_types'], empty_dirs=empty_dirs, empty_dir_mark=mark) if empty_dirs: for d in empty_dirs: if mark: path = d + '/.gitignore' files.append('M 100644 :' + str(mark) + ' ' + path) mark = get_mark(mark) commit_info = ['reset refs/tags/' + release, 'commit refs/tags/' + release, 'mark :' + str(mark), 'author Nokia <*****@*****.**> ' + str(int(initial_release_time)) + " +0000", 'committer Nokia <*****@*****.**> ' + str(int(initial_release_time)) + " +0000", 'data 15', 'Initial commit', '\n'.join(files), ''] print '\n'.join(commit_info) logger.info("git-fast-import:\n%s" %('\n'.join(commit_info))) tag_msg = 'Release: %s' %release annotated_tag = ['tag %s' % release, 'from :%s' % str(mark), 'tagger Nokia <*****@*****.**> ' + str(int(initial_release_time)) + " +0000", 'data %s' % len(tag_msg), tag_msg] print '\n'.join(annotated_tag) commit_lookup[release] = mark # do the following releases (graphs) release_queue = deque(releases[release]['next']) while release_queue: release = release_queue.popleft() previous_release = releases[release]['previous'] logger.info("Next release: %s" % release) commit_graph = graphs[release]['commit'] commit_graph = fix_orphan_nodes(commit_graph, previous_release) commit_graph = ch.spaghettify_digraph(commit_graph, previous_release, release) #htg.commit_graph_to_image(commit_graph, releases[release], graphs[release]['task'], name=releases[release]['name']+'_after' ) # Find the cutting nodes logger.info("Finding the cutting nodes") undirected = graph() undirected.add_nodes(commit_graph.nodes()) [undirected.add_edge(edge) for edge in commit_graph.edges()] cutting_nodes = cut_nodes(undirected) del undirected # Create the reverse commit graph logger.info("Building the reverse commit graph") reverse_commit_graph = commit_graph.reverse() # Compute the accessibility matrix of the reverse commit graph logger.info("Compute the ancestors") ancestors = accessibility(reverse_commit_graph) del reverse_commit_graph logger.info("Ancestors of the release: %s" % str(ancestors[release])) # Clean up the ancestors matrix for k, v in ancestors.iteritems(): if k in v: v.remove(k) # Get the commits order commits = topological_sorting(commit_graph) # Fix the commits order list commits.remove(previous_release) commits.remove(release) last_cutting_node = None # Check if the release (Synergy project has changed name, if it has the # 'base' directory name needs to be renamed if releases.has_key('delimiter'): delim = releases['delimiter'] else: delim = '-' previous_name = previous_release.split(delim)[0] current_name = release.split(delim)[0] if current_name != previous_name: logger.info("Name changed: %s -> %s" %(previous_name, current_name)) from_mark = commit_lookup[previous_release] mark, commit = rename_toplevel_dir(previous_name, current_name, release, releases, mark, from_mark) print '\n'.join(commit) # adjust the commit lookup commit_lookup[previous_release] = mark for counter, commit in enumerate(commits): logger.info("Commit %i/%i" % (counter+1, len(commits))) acn_ancestors = [] if last_cutting_node is not None: acn_ancestors = ancestors[last_cutting_node] # Create the references lists. It lists the parents of the commit #reference = [commit_lookup[parent] for parent in ancestors[commit] if parent not in acn_ancestors] reference = [commit_lookup[parent] for parent in commit_graph.incidents(commit)] if len(reference) > 1: # Merge commit mark = create_merge_commit(commit, release, releases, mark, reference, graphs, set(ancestors[commit]) - set(acn_ancestors)) else: # Normal commit mark = create_commit(commit, release, releases, mark, reference, graphs) # Update the lookup table commit_lookup[commit] = mark # Update the last cutting edge if necessary if commit in cutting_nodes: last_cutting_node = commit if last_cutting_node is not None: acn_ancestors = ancestors[last_cutting_node] reference = [commit_lookup[parent] for parent in ancestors[release] if parent not in acn_ancestors] logger.info("Reference %s" %str([parent for parent in ancestors[release] if parent not in acn_ancestors])) if not reference: logger.info("Reference previous %s, mark: %d" % (releases[release]['previous'], commit_lookup[releases[release]['previous']])) reference = [commit_lookup[ releases[release]['previous'] ] ] mark, merge_commit = create_release_merge_commit(releases, release, get_mark(mark), reference, graphs, set(ancestors[release]) - set(acn_ancestors)) print '\n'.join(merge_commit) annotated_tag = create_annotated_tag(releases, release, mark) print '\n'.join(annotated_tag) commit_lookup[release] = mark release_queue.extend(releases[release]['next']) #release = releases[release]['next'] #release = None #reset to master master = get_master_tag() reset = ['reset refs/heads/master', 'from :' + str(commit_lookup[master])] logger.info("git-fast-import:\n%s" %('\n'.join(reset))) print '\n'.join(reset)
def convert_history(files, tasks, releases, objects): """Converts the Synergy history between two releases to a Git compatible one.""" log.basicConfig(filename="convert_history.log", level=log.DEBUG) file_objects = [ccm_cache.get_object(o) for o in objects] log.info("Looking for cycles in the File History graph") while find_cycle(files): cycle = find_cycle(files) log.info("\tA cycle was found!") log.info("\tCycle: %s" % ", ".join(cycle)) # Find the newest file newest = max( cycle, key=lambda x: [ fileobject.get_integrate_time() for fileobject in file_objects if fileobject.get_objectname() == x ][0], ) log.info("\tObject %s is the newest in the cycle: it should not have successors!" % newest) # Remove the outgoing link from the newest file for successor in files.neighbors(newest): if successor in cycle: files.del_edge((newest, successor)) log.info("\tRemoved the %s -> %s edge" % (newest, successor)) log.info("Remove transitive edges in the File History graph") for edge in transitive_edges(files): if edge in files.edges(): files.del_edge(edge) else: log.warning("Weird, transitive edge not found!") log.info("Sanitize tasks") sanitized_tasks = _sanitize_tasks(tasks) log.info("Create commits graph") commits = create_commits_graph(files, sanitized_tasks, releases) # Uncomment for debug... (remember import) # hack = {'previous': releases.edges()[0]} # htg.commit_graph_to_image(commits, hack, tasks, name='Pre-'+releases.edges()[1]) log.info("Looking for cycles in the Commits graph") while find_cycle(commits): log.info("Finding strictly connected components") cycle = max(mutual_accessibility(commits).values(), key=len) # cycle = find_cycle(commits) log.info("\tA cycle was found!") log.info("\tCycle: %s" % ", ".join(cycle)) log.info("Find the nodes in the cycle going from one task to another") culpript_edges = [] for task in cycle: for obj in tasks.links(task): for neighbor in files.neighbors(obj): if neighbor not in tasks.links(task) and tasks.links(neighbor)[0] in cycle: culpript_edges.append((obj, neighbor)) log.info("\tAdding culpript edge (%s, %s)" % (obj, neighbor)) log.info("Connect the nodes found") culpript_nodes = set() for head, tail in culpript_edges: culpript_nodes.add(head) culpript_nodes.add(tail) for head, tail in permutations(culpript_nodes, 2): if tasks.links(head)[0] == tasks.links(tail)[0] and (head, tail) not in culpript_edges: log.info("\tAdding edge (%s, %s)" % (head, tail)) culpript_edges.append((head, tail)) reduced_digraph = digraph() reduced_digraph.add_nodes(culpript_nodes) [reduced_digraph.add_edge(edge) for edge in culpript_edges] shortest_cycle = max(mutual_accessibility(reduced_digraph).values(), key=len) log.info("Cycle in objects: %s" % shortest_cycle) candidate_cuts = [] # Find the tasks t = set() for node in shortest_cycle: t.add(tasks.links(node)[0]) log.info("T: %s" % str(t)) for i in t: log.info("Cuts for task %s" % i) # Find the objects in the cycle belonging to task i obj_in_task = set(tasks.links(i)) & set(shortest_cycle) log.info("Objects in cycle and task: %s" % obj_in_task) if len(obj_in_task) < 15: if len(obj_in_task) > 1: for j in range(1, len(obj_in_task) / 2 + 1): candidate_cuts.extend([k for k in combinations(obj_in_task, j)]) else: log.info("Cycle too long...") pass log.info("Candidate_cuts: %s" % str(candidate_cuts)) # Find the cut to break the cycle cut = _find_cut(candidate_cuts, cycle, tasks, files, releases) if not cut: # Make a qualified guess of a cut with the shortest walk of files in the tasks walk, node = _find_shortest_incident_or_neighbor_walk(shortest_cycle, cycle, files, tasks) new_cut = walk new_cut.append(node) candidate_cuts.insert(0, tuple(new_cut)) log.info("Candidate cuts: %s", candidate_cuts) cut = _find_cut(candidate_cuts, cycle, tasks, files, releases) if not cut: # Error! This should not happen log.info("Cut not found.") log.shutdown() raise Exception("Cut not found") tasks, task, task_name = _apply_cut(cut, tasks) commits = create_commits_graph(files, tasks, releases) else: log.info("No cycles found") log.shutdown() return commits
def get_project_history(self, latest_project, base_project): release_chain = deque(get_project_chain(latest_project, base_project, self.ccm)) base_project = release_chain.popleft() baseline_project = ccm_cache.get_object(base_project, self.ccm) self.tag = baseline_project.get_name() + self.delim + baseline_project.get_version() # Initialize history if self.tag not in self.history.keys(): self.history[self.tag] = {'objects': [], 'tasks': []} self.history['delimiter'] = self.delim self.history[self.tag]['previous'] = None #if self.history[self.tag].has_key('next'): # if not release_chain[0] in self.history[self.tag]['next']: # self.history[self.tag]['next'].append(release_chain[0]) #else: # self.history[self.tag]['next']= [release_chain[0]] logger.info("getting all objects for: %s ... " % self.tag) self.baseline_objects = None # Do the first project as a full project self.find_project_diff(baseline_project, None) self.history[self.tag]['name'] = self.tag self.history[self.tag]['fourpartname'] = baseline_project.get_object_name() self.history[self.tag]['created'] = baseline_project.get_created_time() self.history[self.tag]['author'] = baseline_project.get_author() # Add the objects and paths to the history, to be used for finding empty directories empty_dirs = find_empty_dirs(self.baseline_objects) self.history[self.tag]['empty_dirs'] = empty_dirs while release_chain: next = release_chain.popleft() next_project = ccm_cache.get_object(next, self.ccm) # Set next project for the current baseline_project if self.history[self.tag].has_key('next'): if not next_project.get_name() + self.delim + next_project.get_version() in self.history[self.tag]['next']: self.history[self.tag]['next'].append(next_project.get_name() + self.delim + next_project.get_version()) else: self.history[self.tag]['next'] = [next_project.get_name() + self.delim + next_project.get_version()] # Info about baseline_project logger.info("%s done processing, Info:" % self.tag) logger.info("Name %s" % self.tag) logger.info("4partname %s" % self.history[self.tag]['fourpartname']) logger.info("Number of: ") logger.info(" Tasks: %i" % len(self.history[self.tag]['tasks'])) logger.info(" Files: %i" % len(self.history[self.tag]['objects'])) logger.info("Previous <- %s" % self.history[self.tag]['previous']) logger.info("Next -> %s" % str(self.history[self.tag]['next'])) logger.info("") self.tag = next_project.get_name() + self.delim + next_project.get_version() logger.info("Next project: %s" % next_project.get_object_name()) if self.tag not in self.history.keys(): self.history[self.tag] = {'objects': [], 'tasks': []} self.history[self.tag]['previous'] = baseline_project.get_name() + self.delim + baseline_project.get_version() logger.info("Toplevel Project: %s " % next_project.get_object_name()) # do the history thing self.history[self.tag]['name'] = self.tag self.history[self.tag]['fourpartname'] = next_project.get_object_name() self.find_project_diff(baseline_project, next_project) self.history[self.tag]['created'] = next_project.get_created_time() self.history[self.tag]['author'] = next_project.get_author() # Add the objects and paths to the history, to be used for finding empty directories empty_dirs = find_empty_dirs(self.project_objects) logger.info("Empty dirs:\n%s" % '\n'.join(sorted(empty_dirs))) self.history[self.tag]['empty_dirs'] = empty_dirs # set new baseline project baseline_project = next_project # set the baseline_objects for the next iteration and self.baseline_objects = self.project_objects self.project_objects = [] #Store data fname = self.outputfile + '_' + self.tag self.persist_data(fname, self.history[self.tag]) self.history_created.append(fname) # delete the _inc file if it exists if os.path.isfile(fname + '_inc' + '.p'): os.remove(fname + '_inc' + '.p') # Info for last project: if not self.history[self.tag].has_key('next'): self.history[self.tag]['next'] = [] logger.info("%s done processing, Info: " %self.tag) logger.info("Name %s" % self.tag) logger.info("4partname %s" % self.history[self.tag]['fourpartname']) logger.info("Number of: ") logger.info(" Tasks: %i" % len(self.history[self.tag]['tasks'])) logger.info(" Files: %i" % len(self.history[self.tag]['objects'])) logger.info("Previous <- %s" % self.history[self.tag]['previous']) logger.info("Next -> %s" % str(self.history[self.tag]['next'])) logger.info("") # Get the different types in the db and their corresponding file permissions and super types ccm_types_perms = ccm_type.get_types_and_permissions(self.ccm) ccm_super_types = ccm_type.get_super_types(self.ccm) if not self.history.has_key('ccm_types'): self.history['ccm_types'] = {} self.history['ccm_types']['permissions'] = ccm_types_perms self.history['ccm_types']['super_types'] = ccm_super_types return self.history
def recursive_get_history(self, fileobject, recursion_depth): """ Recursivly find the history of the file object, optionally stopping at the 'old_release' project """ next_iter = False logger.info('Processing: %s %s' % (fileobject.get_object_name(), fileobject.get_status())) logger.info('Recursion depth %d' % recursion_depth) retval = True #Check if recursion_depth is reached if recursion_depth > self.max_recursion_depth: logger.warning('Giving up on %s' % fileobject.get_object_name()) return False recursion_depth += 1 predecessors = fileobject.get_predecessors() for p in predecessors: try: predecessor = ccm_cache.get_object(p, self.ccm) except ccm_cache.ObjectCacheException: # Object couldn't be retrived from ccm, give up on this logger.info("Couldn't get %s from Synergy" % p) return True logger.info("Predecessor: %s" % predecessor.get_object_name()) # check predecessor release to see if this object should be added to the set. if self.old_release: # Get the release(s) for the predecessor releases = predecessor.get_releases() if releases: # Check if the "old" release is the the releases for the predecessor and stop if true if [r for r in releases if r in self.current_subproject_list or r in self.old_subproject_list]: # Object is already released, continue with the next predecessor logger.info("%s is already released" %predecessor.get_object_name()) continue logger.info("Couldn't find release in current_subproject_list or old_subproject_list") #Check if chain of successors contains previous object - if true discard the chain if self.successor_is_released(predecessor, fileobject, 0): logger.info("Successor is already released %s" % fileobject.get_object_name()) continue #Check if projects are releated to old release. Latest first for r in releases: try: project = ccm_cache.get_object(r, self.ccm) except ccm_cache.ObjectCacheException: continue if self.project_is_some_predecessor(project, 0): logger.info("Found Relationship between: %s and %s " %(project.get_object_name(), self.old_release)) next_iter = True break if next_iter: continue else: #Check if a successor is released if self.successor_is_released(predecessor, fileobject, 0): logger.info("Successor is already released %s " % fileobject.get_object_name()) continue # Check if predecessor is already added to history - if so add this as successor to fileobject, else add new predecessor to history if not self.temp_history.has_key(predecessor.get_object_name()): logger.info("Adding %s %s to history" % (predecessor.get_object_name(), predecessor.get_status()) ) self.add_to_history(predecessor) retval &= self.recursive_get_history(predecessor, recursion_depth) if not retval: # Giving up on history break out of loop break return retval
def get_objects_in_project_serial(project, ccm=None, database=None, use_cache=False): """ Get all objects and paths of a project """ if not ccm: if not database: raise SynergyException("No ccm instance nor database given\n" + "Cannot start ccm session!\n") ccm = SynergySession(database) else: logger.debug("ccm instance: %s" % ccm.environment['CCM_ADDR']) delim = ccm.delim() if use_cache: start_object = ccm_cache.get_object(project, ccm) else: start_object = SynergyObject(project, delim) queue = deque([start_object]) hierarchy = {} dir_structure = {} proj_lookup = {} cwd = '' count = 1 hierarchy[start_object.get_object_name()] = [start_object.name] dir_structure[start_object.get_object_name()] = '' while queue: obj = queue.popleft() #logger.debug('Processing: %s' % obj.get_object_name()) parent_proj = None if obj.get_type() == 'dir' or obj.get_type() == 'project': # Processing a dir set 'working dir' cwd = dir_structure[obj.get_object_name()] if obj.get_type() == 'dir': parent_proj = proj_lookup[obj.get_object_name()] result = get_members(obj, ccm, parent_proj) if use_cache: objects = [] na_obj = [] for item in result: try: objects.append( ccm_cache.get_object(item['objectname'], ccm)) except ccm_cache.ObjectCacheException: objects.append( SynergyObject(item['objectname'], ccm.delim())) na_obj.append(item['objectname']) if na_obj: logger.warning("Objects not avaliable in this db:") logger.warning(', '.join(na_obj)) else: objects = [ SynergyObject(item['objectname'], delim) for item in result ] # if a project is being queried it might have more than one dir with the # same name as the project associated, find the directory that has the # project associated as the directory's parent if obj.get_type() == 'project': if len(objects) > 1: objects = find_root_project(obj, objects, ccm) for synergy_object in objects: count += 1 if synergy_object.get_type() == 'dir': # add the directory to the queue and record its parent project queue.append(synergy_object) #logger.debug("object: %s child %s cwd %s" % (obj # .get_object_name(), o.get_object_name(), cwd)) dir_structure[synergy_object.get_object_name()] = \ '%s%s/' % (cwd, synergy_object.get_name()) if obj.get_type() == 'project': proj_lookup[synergy_object.get_object_name()] = \ obj.get_object_name() elif obj.get_type() == 'dir': proj_lookup[synergy_object.get_object_name()] = \ proj_lookup[obj.get_object_name()] # Also add the directory to the Hierachy to get empty dirs if synergy_object.get_object_name() in hierarchy.keys(): hierarchy[synergy_object.get_object_name()].append( '%s%s' % (cwd, synergy_object.get_name())) else: hierarchy[synergy_object.get_object_name()] = \ ['%s%s' % (cwd, synergy_object.get_name())] elif synergy_object.get_type() == 'project': dir_structure[synergy_object.get_object_name()] = cwd # Add the project to the queue queue.append(synergy_object) #logger.debug("object: %s child %s cwd %s" % (obj # .get_object_name(), o.get_object_name(), cwd)) # Add the project to the hierarchy, # so the subprojects for the release/project is known if synergy_object.get_object_name() in hierarchy.keys(): hierarchy[synergy_object.get_object_name()].append( '%s%s' % (cwd, synergy_object.get_name())) else: hierarchy[synergy_object.get_object_name()] = \ ['%s%s' % (cwd, synergy_object.get_name())] else: # Add the object to the hierarchy if obj.get_type() == 'dir': if synergy_object.get_object_name() in hierarchy.keys(): hierarchy[synergy_object.get_object_name()].append( '%s%s' % (cwd, synergy_object.get_name())) else: hierarchy[synergy_object.get_object_name()] = \ ['%s%s' % (cwd, synergy_object.get_name())] #logger.debug("Object: %s has path %s%s" % (o.get_object_name(), cwd, o.get_name())) logger.debug("Object count: %6d" % count) return hierarchy
def get_objects_in_project_serial(project, ccm=None, database=None, use_cache=False): """ Get all objects and paths of a project """ if not ccm: if not database: raise SynergyException("No ccm instance nor database given\n" + "Cannot start ccm session!\n") ccm = SynergySession(database) else: logger.debug("ccm instance: %s" % ccm.environment['CCM_ADDR']) delim = ccm.delim() if use_cache: start_object = ccm_cache.get_object(project, ccm) else: start_object = SynergyObject(project, delim) queue = deque([start_object]) hierarchy = {} dir_structure = {} proj_lookup = {} cwd = '' count = 1 hierarchy[start_object.get_object_name()] = [start_object.name] dir_structure[start_object.get_object_name()] = '' while queue: obj = queue.popleft() #logger.debug('Processing: %s' % obj.get_object_name()) parent_proj = None if obj.get_type() == 'dir' or obj.get_type() == 'project': # Processing a dir set 'working dir' cwd = dir_structure[obj.get_object_name()] if obj.get_type() == 'dir': parent_proj = proj_lookup[obj.get_object_name()] result = get_members(obj, ccm, parent_proj) if use_cache: objects = [] na_obj = [] for item in result: try: objects.append(ccm_cache.get_object(item['objectname'], ccm)) except ccm_cache.ObjectCacheException: objects.append(SynergyObject(item['objectname'], ccm.delim())) na_obj.append(item['objectname']) if na_obj: logger.warning("Objects not avaliable in this db:") logger.warning(', '.join(na_obj)) else: objects = [SynergyObject(item['objectname'], delim) for item in result] # if a project is being queried it might have more than one dir with the # same name as the project associated, find the directory that has the # project associated as the directory's parent if obj.get_type() == 'project': if len(objects) > 1: objects = find_root_project(obj, objects, ccm) for synergy_object in objects: count += 1 if synergy_object.get_type() == 'dir': # add the directory to the queue and record its parent project queue.append(synergy_object) #logger.debug("object: %s child %s cwd %s" % (obj # .get_object_name(), o.get_object_name(), cwd)) dir_structure[synergy_object.get_object_name()] = \ '%s%s/' % (cwd, synergy_object.get_name()) if obj.get_type() == 'project': proj_lookup[synergy_object.get_object_name()] = \ obj.get_object_name() elif obj.get_type() == 'dir': proj_lookup[synergy_object.get_object_name()] = \ proj_lookup[obj.get_object_name()] # Also add the directory to the Hierachy to get empty dirs if synergy_object.get_object_name() in hierarchy.keys(): hierarchy[synergy_object.get_object_name()].append( '%s%s' % (cwd, synergy_object.get_name())) else: hierarchy[synergy_object.get_object_name()] = \ ['%s%s' % (cwd, synergy_object.get_name())] elif synergy_object.get_type() == 'project': dir_structure[synergy_object.get_object_name()] = cwd # Add the project to the queue queue.append(synergy_object) #logger.debug("object: %s child %s cwd %s" % (obj # .get_object_name(), o.get_object_name(), cwd)) # Add the project to the hierarchy, # so the subprojects for the release/project is known if synergy_object.get_object_name() in hierarchy.keys(): hierarchy[synergy_object.get_object_name()].append( '%s%s' % (cwd, synergy_object.get_name())) else: hierarchy[synergy_object.get_object_name()] = \ ['%s%s' % (cwd, synergy_object.get_name())] else: # Add the object to the hierarchy if obj.get_type() == 'dir': if synergy_object.get_object_name() in hierarchy.keys(): hierarchy[synergy_object.get_object_name()].append( '%s%s' % (cwd, synergy_object.get_name())) else: hierarchy[synergy_object.get_object_name()] = \ ['%s%s' % (cwd, synergy_object.get_name())] #logger.debug("Object: %s has path %s%s" % (o.get_object_name(), cwd, o.get_name())) logger.debug("Object count: %6d" % count) return hierarchy
def convert_history(files, tasks, releases, objects): """Converts the Synergy history between two releases to a Git compatible one.""" log.basicConfig(filename='convert_history.log', level=log.DEBUG) file_objects = [ccm_cache.get_object(o) for o in objects] log.info("Looking for cycles in the File History graph") while find_cycle(files): cycle = find_cycle(files) log.info("\tA cycle was found!") log.info("\tCycle: %s" % ", ".join(cycle)) # Find the newest file newest = max(cycle, key=lambda x: [ fileobject.get_integrate_time() for fileobject in file_objects if fileobject.get_objectname() == x ][0]) log.info( "\tObject %s is the newest in the cycle: it should not have successors!" % newest) # Remove the outgoing link from the newest file for successor in files.neighbors(newest): if successor in cycle: files.del_edge((newest, successor)) log.info("\tRemoved the %s -> %s edge" % (newest, successor)) log.info("Remove transitive edges in the File History graph") for edge in transitive_edges(files): if edge in files.edges(): files.del_edge(edge) else: log.warning("Weird, transitive edge not found!") log.info("Sanitize tasks") sanitized_tasks = _sanitize_tasks(tasks) log.info("Create commits graph") commits = create_commits_graph(files, sanitized_tasks, releases) # Uncomment for debug... (remember import) #hack = {'previous': releases.edges()[0]} #htg.commit_graph_to_image(commits, hack, tasks, name='Pre-'+releases.edges()[1]) log.info("Looking for cycles in the Commits graph") while find_cycle(commits): log.info("Finding strictly connected components") cycle = max(mutual_accessibility(commits).values(), key=len) #cycle = find_cycle(commits) log.info("\tA cycle was found!") log.info("\tCycle: %s" % ", ".join(cycle)) log.info("Find the nodes in the cycle going from one task to another") culpript_edges = [] for task in cycle: for obj in tasks.links(task): for neighbor in files.neighbors(obj): if neighbor not in tasks.links(task) and tasks.links( neighbor)[0] in cycle: culpript_edges.append((obj, neighbor)) log.info("\tAdding culpript edge (%s, %s)" % (obj, neighbor)) log.info("Connect the nodes found") culpript_nodes = set() for head, tail in culpript_edges: culpript_nodes.add(head) culpript_nodes.add(tail) for head, tail in permutations(culpript_nodes, 2): if tasks.links(head)[0] == tasks.links(tail)[0] and ( head, tail) not in culpript_edges: log.info("\tAdding edge (%s, %s)" % (head, tail)) culpript_edges.append((head, tail)) reduced_digraph = digraph() reduced_digraph.add_nodes(culpript_nodes) [reduced_digraph.add_edge(edge) for edge in culpript_edges] shortest_cycle = max(mutual_accessibility(reduced_digraph).values(), key=len) log.info("Cycle in objects: %s" % shortest_cycle) candidate_cuts = [] # Find the tasks t = set() for node in shortest_cycle: t.add(tasks.links(node)[0]) log.info("T: %s" % str(t)) for i in t: log.info("Cuts for task %s" % i) # Find the objects in the cycle belonging to task i obj_in_task = set(tasks.links(i)) & set(shortest_cycle) log.info("Objects in cycle and task: %s" % obj_in_task) if len(obj_in_task) < 15: if len(obj_in_task) > 1: for j in range(1, len(obj_in_task) / 2 + 1): candidate_cuts.extend( [k for k in combinations(obj_in_task, j)]) else: log.info("Cycle too long...") pass if len(candidate_cuts) < 50: log.info("Candidate_cuts: %s" % str(candidate_cuts)) # Find the cut to break the cycle cut = _find_cut(candidate_cuts, cycle, tasks, files, releases) if not cut: log.info("Shortest cycle cut didn't work, next option...") # Make a qualified guess of a cut with the shortest walk of files in the tasks walk, node = _find_shortest_incident_or_neighbor_walk( shortest_cycle, cycle, files, tasks) log.info("Shortest incident or neighbor walk from {0}: {1}".format( node, walk)) new_cut = walk new_cut.append(node) candidate_cuts.insert(0, tuple(new_cut)) log.info("Candidate cuts: %s", candidate_cuts) cut = _find_cut(candidate_cuts, cycle, tasks, files, releases) if not cut: # Error! This should not happen log.info("Cut not found.") log.shutdown() raise Exception("Cut not found") tasks, task, task_name = _apply_cut(cut, tasks) commits = create_commits_graph(files, tasks, releases) else: log.info("No cycles found") log.shutdown() return commits
def find_project_diff(self, baseline_project, next_project): # Get all objects and paths for baseline_project if not self.baseline_objects: self.baseline_objects = baseline_project.get_members() if self.baseline_objects is None or len( self.baseline_objects) == 1 or not isinstance( self.baseline_objects, dict): self.baseline_objects = ccm_objects.get_objects_in_project( baseline_project.get_object_name(), ccmpool=self.ccmpool) baseline_project.set_members(self.baseline_objects) ccm_cache.force_cache_update_for_object(baseline_project) if next_project: # Get all objects and paths for next project self.project_objects = next_project.get_members() if self.project_objects is None or len( self.project_objects) == 1 or not isinstance( self.project_objects, dict): self.project_objects = ccm_objects.get_objects_in_project( next_project.get_object_name(), ccmpool=self.ccmpool) next_project.set_members(self.project_objects) ccm_cache.force_cache_update_for_object(next_project) # Find difference between baseline_project and next_project new_objects, old_objects = get_changed_objects( self.baseline_objects, self.project_objects) next_projects = [ o for o in self.project_objects.keys() if ':project:' in o ] baseline_projects = [ o for o in self.baseline_objects.keys() if ':project:' in o ] object_history = ObjectHistory( self.ccm, next_project.get_object_name(), old_objects=old_objects, old_release=baseline_project.get_object_name(), new_projects=next_projects, old_projects=baseline_projects) else: # root project, get ALL objects in release new_objects = self.baseline_objects old_objects = [] object_history = ObjectHistory(self.ccm, baseline_project.get_object_name()) num_of_objects = len( [o for o in new_objects.keys() if ":project:" not in o]) logger.info("objects to process : %i" % num_of_objects) objects = [] if self.tag in self.history.keys(): if 'objects' in self.history[self.tag]: #Add all existing objects for o in self.history[self.tag]['objects']: objects.append(o) #objects[o] = ccm_cache.get_object(o, self.ccm) logger.info("no of old objects loaded %i", len(objects)) else: self.history[self.tag] = {'objects': [], 'tasks': []} object_names = set([ o for o in new_objects.keys() if ':project:' not in o ]) - set(objects) for o in object_names: object = ccm_cache.get_object(o, self.ccm) if next_project: # get the object history between releases history = object_history.get_history( object, new_objects[object.get_object_name()]) objects.extend(history.keys()) #objects.update(object_history.get_history(object, new_objects[object.get_object_name()])) else: # just get all the objects in the release logger.info('Processing: %s path: %s' % (object.get_object_name(), str(new_objects[object.get_object_name()]))) #object.set_path(new_objects[object.get_object_name()]) objects.append(o) #objects[object.get_object_name()] = object num_of_objects -= 1 logger.info('Objects left: %i' % num_of_objects) objects = list(set(objects)) logger.info("number of files: %i" % len(objects)) self.history[self.tag]['objects'] = objects # Create tasks from objects, but not for initial project if next_project: self.find_tasks_from_objects(objects, next_project) # Handle new projects: if next_project: new_created_projects = get_new_projects(old_objects, new_objects, self.delim) dir_lookup = {} # create lookup for path to directory-4-part-name {path : dir-name} for k, v in new_objects.iteritems(): if ':dir:' in k: for i in v: dir_lookup[i] = k project_dirs = [ dir for project in new_created_projects for dir in new_objects[project] ] directories = [ d for k, v in new_objects.iteritems() for d in v if ':dir:' in k ] changed_directories = set(directories).intersection( set(project_dirs)) changed_directories = remove_subdirs_under_same_path( changed_directories) dirs = [dir_lookup[d] for d in changed_directories] # find task and add all objects to the task, which shares the path. project_tasks = self.find_task_from_dirs(dirs) logger.info("Checking for new subprojects") # check directories for new subdirectories and add their content directories = self.get_new_dirs(self.project_objects, new_objects) # Limit directories to only directories not already processed as a new project directories = set(directories) - set(dirs) # find task and add all objects to the task, which shares the path. dir_tasks = self.find_task_from_dirs(directories) # merge project and dir tasks for k, v in dir_tasks.iteritems(): if not project_tasks.has_key(k): project_tasks[k] = v # if real synergy tasks isn't found check the path of the directories and skip possible subdirs tasks = self.reduce_dir_tasks(project_tasks) logger.info("Project and dir tasks reduced...") logger.info("%s" % str(tasks)) self.update_tasks_with_directory_contens(tasks) # remove possible duplicates from objects self.history[self.tag]['objects'] = list( set(self.history[self.tag]['objects']))