def do_query(next_in_queue, free_ccm, semaphore, delim, p_queue, use_cache): (project, parent_proj) = next_in_queue ccm_addr, database = get_and_lock_free_ccm_addr(free_ccm) ccm = SynergySession(database, ccm_addr=ccm_addr) ccm.keep_session_alive = True # logger.debug('Querying: %s' % project.get_object_name()) result = get_members(project, ccm, parent_proj) if use_cache: objects = [] na_obj = [] for item in result: try: objects.append(ccm_cache.get_object(item['objectname'], ccm)) except ccm_cache.ObjectCacheException: objects.append(SynergyObject(item['objectname'], delim)) na_obj.append(item['objectname']) if na_obj: logger.warning("Objects not avaliable in this db:") logger.warning(', '.join(na_obj)) else: objects = [SynergyObject(item['objectname'], delim) for item in result] # if a project is being queried it might have more than one dir with the # same name as the project associated, find the directory that has the # project associated as the directory's parent if project.get_type() == 'project': if len(objects) > 1: objects = find_root_project(project, objects, ccm) p_queue.put((project, objects)) entry = free_ccm[ccm_addr] entry['free'] = True free_ccm[ccm_addr] = entry semaphore.release()
def get_objects_in_project_parallel(project, ccmpool=None): mgr = Manager() free_ccm = mgr.dict() for ccm in ccmpool.sessionArray.values(): free_ccm[ccm.getCCM_ADDR()] = True ccm_addr = get_and_lock_free_ccm_addr(free_ccm) ccm = SynergySession(None, ccm_addr=ccm_addr) ccm.keep_session_alive = True delim = ccm.delim() # unlock free_ccm[ccm_addr] = True semaphore = mgr.Semaphore(ccmpool.nr_sessions) so = SynergyObject(project, delim) p_queue = mgr.Queue() c_queue = mgr.Queue() c_queue.put((so, None)) p_queue.put(so) # start the produce and consumer thread prod = Process(target=producer, args=(c_queue, p_queue, free_ccm)) cons = Process(target=consumer, args=(c_queue, p_queue, free_ccm, semaphore, delim)) prod.start() cons.start() print "Waiting to join" cons.join() hierarchy = p_queue.get() prod.join() return hierarchy
def create_ccm_session_from_config(): f = open('config.p', 'rb') config = cPickle.load(f) f.close() ccm = SynergySession(config['database']) return ccm
def get_objects_in_project_parallel(project, ccmpool=None, use_cache=False): """ Get all the objects and paths of project with use of multiple ccm sessions """ mgr = Manager() free_ccm = mgr.dict() for ccm in ccmpool.sessionArray.values(): free_ccm[ccm.getCCM_ADDR()] = {'free': True, 'database': ccm.database} ccm_addr, database = get_and_lock_free_ccm_addr(free_ccm) ccm = SynergySession(database, ccm_addr=ccm_addr) ccm.keep_session_alive = True delim = ccm.delim() semaphore = mgr.Semaphore(ccmpool.nr_sessions) # Starting project if use_cache: start_object = ccm_cache.get_object(project, ccm) else: start_object = SynergyObject(project, delim) # unlock update dict entry to inform manager entry = free_ccm[ccm_addr] entry['free'] = True free_ccm[ccm_addr] = entry p_queue = mgr.Queue() c_queue = mgr.Queue() c_queue.put((start_object, None)) p_queue.put(start_object) # start the produce and consumer thread prod = Process(target=producer, args=(c_queue, p_queue, free_ccm)) cons = Process(target=consumer, args=(c_queue, p_queue, free_ccm, semaphore, delim, use_cache)) prod.start() cons.start() logger.debug("Waiting to join") cons.join() hierarchy = p_queue.get() prod.join() return hierarchy
def do_query(next, free_ccm, semaphore, delim, p_queue): (project, parent_proj) = next #print 'Querying:', project.get_object_name() ccm_addr = get_and_lock_free_ccm_addr(free_ccm) ccm = SynergySession(None, ccm_addr=ccm_addr) ccm.keep_session_alive = True result = get_members(project, ccm, parent_proj) objects = [SynergyObject(o['objectname'], delim) for o in result] # if a project is being queried it might have more than one dir with the # same name as the project associated, find the directory that has the # project associated as the directory's parent if project.get_type() == 'project': if len(objects) > 1: objects = find_root_project(project, objects, ccm) p_queue.put((project, objects)) free_ccm[ccm_addr] = True semaphore.release()
def get_objects_in_project_serial(project, ccm=None, database=None, use_cache=False): """ Get all objects and paths of a project """ if not ccm: if not database: raise SynergyException("No ccm instance nor database given\n" + "Cannot start ccm session!\n") ccm = SynergySession(database) else: logger.debug("ccm instance: %s" % ccm.environment['CCM_ADDR']) delim = ccm.delim() if use_cache: start_object = ccm_cache.get_object(project, ccm) else: start_object = SynergyObject(project, delim) queue = deque([start_object]) hierarchy = {} dir_structure = {} proj_lookup = {} cwd = '' count = 1 hierarchy[start_object.get_object_name()] = [start_object.name] dir_structure[start_object.get_object_name()] = '' while queue: obj = queue.popleft() #logger.debug('Processing: %s' % obj.get_object_name()) parent_proj = None if obj.get_type() == 'dir' or obj.get_type() == 'project': # Processing a dir set 'working dir' cwd = dir_structure[obj.get_object_name()] if obj.get_type() == 'dir': parent_proj = proj_lookup[obj.get_object_name()] result = get_members(obj, ccm, parent_proj) if use_cache: objects = [] na_obj = [] for item in result: try: objects.append(ccm_cache.get_object(item['objectname'], ccm)) except ccm_cache.ObjectCacheException: objects.append(SynergyObject(item['objectname'], ccm.delim())) na_obj.append(item['objectname']) if na_obj: logger.warning("Objects not avaliable in this db:") logger.warning(', '.join(na_obj)) else: objects = [SynergyObject(item['objectname'], delim) for item in result] # if a project is being queried it might have more than one dir with the # same name as the project associated, find the directory that has the # project associated as the directory's parent if obj.get_type() == 'project': if len(objects) > 1: objects = find_root_project(obj, objects, ccm) for synergy_object in objects: count += 1 if synergy_object.get_type() == 'dir': # add the directory to the queue and record its parent project queue.append(synergy_object) #logger.debug("object: %s child %s cwd %s" % (obj # .get_object_name(), o.get_object_name(), cwd)) dir_structure[synergy_object.get_object_name()] = \ '%s%s/' % (cwd, synergy_object.get_name()) if obj.get_type() == 'project': proj_lookup[synergy_object.get_object_name()] = \ obj.get_object_name() elif obj.get_type() == 'dir': proj_lookup[synergy_object.get_object_name()] = \ proj_lookup[obj.get_object_name()] # Also add the directory to the Hierachy to get empty dirs if synergy_object.get_object_name() in hierarchy.keys(): hierarchy[synergy_object.get_object_name()].append( '%s%s' % (cwd, synergy_object.get_name())) else: hierarchy[synergy_object.get_object_name()] = \ ['%s%s' % (cwd, synergy_object.get_name())] elif synergy_object.get_type() == 'project': dir_structure[synergy_object.get_object_name()] = cwd # Add the project to the queue queue.append(synergy_object) #logger.debug("object: %s child %s cwd %s" % (obj # .get_object_name(), o.get_object_name(), cwd)) # Add the project to the hierarchy, # so the subprojects for the release/project is known if synergy_object.get_object_name() in hierarchy.keys(): hierarchy[synergy_object.get_object_name()].append( '%s%s' % (cwd, synergy_object.get_name())) else: hierarchy[synergy_object.get_object_name()] = \ ['%s%s' % (cwd, synergy_object.get_name())] else: # Add the object to the hierarchy if obj.get_type() == 'dir': if synergy_object.get_object_name() in hierarchy.keys(): hierarchy[synergy_object.get_object_name()].append( '%s%s' % (cwd, synergy_object.get_name())) else: hierarchy[synergy_object.get_object_name()] = \ ['%s%s' % (cwd, synergy_object.get_name())] #logger.debug("Object: %s has path %s%s" % (o.get_object_name(), cwd, o.get_name())) logger.debug("Object count: %6d" % count) return hierarchy
def create_session(database, engine, command_name, ccm_ui_path, ccm_eng_path, offline, i): ccm = SynergySession(database, engine, command_name, ccm_ui_path, ccm_eng_path, offline) ccm.keep_session_alive = True ccm.sessionID = i return (i,ccm)
def get_objects_in_project_serial(project, ccm=None, database=None, use_cache=False): """ Get all objects and paths of a project """ if not ccm: if not database: raise SynergyException("No ccm instance nor database given\n" + "Cannot start ccm session!\n") ccm = SynergySession(database) else: logger.debug("ccm instance: %s" % ccm.environment['CCM_ADDR']) delim = ccm.delim() if use_cache: start_object = ccm_cache.get_object(project, ccm) else: start_object = SynergyObject(project, delim) queue = deque([start_object]) hierarchy = {} dir_structure = {} proj_lookup = {} cwd = '' count = 1 hierarchy[start_object.get_object_name()] = [start_object.name] dir_structure[start_object.get_object_name()] = '' while queue: obj = queue.popleft() #logger.debug('Processing: %s' % obj.get_object_name()) parent_proj = None if obj.get_type() == 'dir' or obj.get_type() == 'project': # Processing a dir set 'working dir' cwd = dir_structure[obj.get_object_name()] if obj.get_type() == 'dir': parent_proj = proj_lookup[obj.get_object_name()] result = get_members(obj, ccm, parent_proj) if use_cache: objects = [] na_obj = [] for item in result: try: objects.append( ccm_cache.get_object(item['objectname'], ccm)) except ccm_cache.ObjectCacheException: objects.append( SynergyObject(item['objectname'], ccm.delim())) na_obj.append(item['objectname']) if na_obj: logger.warning("Objects not avaliable in this db:") logger.warning(', '.join(na_obj)) else: objects = [ SynergyObject(item['objectname'], delim) for item in result ] # if a project is being queried it might have more than one dir with the # same name as the project associated, find the directory that has the # project associated as the directory's parent if obj.get_type() == 'project': if len(objects) > 1: objects = find_root_project(obj, objects, ccm) for synergy_object in objects: count += 1 if synergy_object.get_type() == 'dir': # add the directory to the queue and record its parent project queue.append(synergy_object) #logger.debug("object: %s child %s cwd %s" % (obj # .get_object_name(), o.get_object_name(), cwd)) dir_structure[synergy_object.get_object_name()] = \ '%s%s/' % (cwd, synergy_object.get_name()) if obj.get_type() == 'project': proj_lookup[synergy_object.get_object_name()] = \ obj.get_object_name() elif obj.get_type() == 'dir': proj_lookup[synergy_object.get_object_name()] = \ proj_lookup[obj.get_object_name()] # Also add the directory to the Hierachy to get empty dirs if synergy_object.get_object_name() in hierarchy.keys(): hierarchy[synergy_object.get_object_name()].append( '%s%s' % (cwd, synergy_object.get_name())) else: hierarchy[synergy_object.get_object_name()] = \ ['%s%s' % (cwd, synergy_object.get_name())] elif synergy_object.get_type() == 'project': dir_structure[synergy_object.get_object_name()] = cwd # Add the project to the queue queue.append(synergy_object) #logger.debug("object: %s child %s cwd %s" % (obj # .get_object_name(), o.get_object_name(), cwd)) # Add the project to the hierarchy, # so the subprojects for the release/project is known if synergy_object.get_object_name() in hierarchy.keys(): hierarchy[synergy_object.get_object_name()].append( '%s%s' % (cwd, synergy_object.get_name())) else: hierarchy[synergy_object.get_object_name()] = \ ['%s%s' % (cwd, synergy_object.get_name())] else: # Add the object to the hierarchy if obj.get_type() == 'dir': if synergy_object.get_object_name() in hierarchy.keys(): hierarchy[synergy_object.get_object_name()].append( '%s%s' % (cwd, synergy_object.get_name())) else: hierarchy[synergy_object.get_object_name()] = \ ['%s%s' % (cwd, synergy_object.get_name())] #logger.debug("Object: %s has path %s%s" % (o.get_object_name(), cwd, o.get_name())) logger.debug("Object count: %6d" % count) return hierarchy
def add_project(project, db, commit_msg, parent=None, branch_name=None, path=None): print "Starting Synergy..." ccm = SynergySession(db) if not project_ok(project, ccm): raise Exception("Error no such Synergy project") if not git_project_ok(): raise Exception("Git project not ok, cwd needs to be git project root") cwd = os.getcwd() if parent: # if not on the branch already check it out if not parent == get_current_branch(): command = ['git', 'checkout', '-q', parent] run_command(command) else: #check out first commit and create branch from here first_commit = run_command(['git', 'log', '--format=%H']).splitlines()[-1] command = ['git', 'checkout', '-q', first_commit] run_command(command) # setup new branch if branch_name: if branch_name != get_current_branch(): run_command(['git', 'checkout', '-q', '-b', branch_name]) # setup path to import project into if path: if not os.path.exists(path): os.makedirs(path) os.chdir(path) cwd = os.getcwd() else: os.chdir(path) cwd = os.getcwd() try: run_command(['git', 'rm', '-q', '-rf', cwd]) except GitException as ex: if not 'did not match any files' in ex.value: raise Exception("Error deleting files in {0}".format(cwd)) else: # delete everything try: run_command(['git', 'rm', '-q', '-rf', cwd]) except GitException as ex: if not 'did not match any files' in ex.value: raise Exception("Error deleting files in {0}".format(cwd)) print "Creating snapshot of project" get_snapshot(project, ccm, cwd) # add everything to git print "Adding project to git" run_command(['git', 'add', '.']) print "Committing..." out = run_command(['git', 'commit', '-m', '\n'.join(commit_msg)]) print out if '(no branch)' in get_current_branch(): # detached head state current_commit = get_current_commit() #find which branch contains parent commit if not parent: parent = get_parent_commit(current_commit) branches = run_command(['git', 'branch', '--contains', parent]).splitlines() branch = None for branch in branches: if not '(no branch)' in branch: break # Checkout branch and fast-forward out = run_command(['git', 'checkout', '-q', branch.strip()]) print out out = run_command(['git', 'merge', current_commit]) print out print "Done"
def start_sessions(config): ccm = SynergySession(config['database']) ccm_pool = SynergySessions(database=config['database'], nr_sessions=config['max_sessions']) return ccm, ccm_pool
def get_objects_in_project_serial(project, ccm=None, database=None): if not ccm: if not database: raise SynergyException('No ccm instance nor database given\nCannot start ccm session!\n') ccm = SynergySession(database) else: print "ccm instance:", ccm.environment['CCM_ADDR'] delim = ccm.delim() so = SynergyObject(project, delim) queue = deque([so]) hierarchy = {} dir_structure = {} proj_lookup = {} cwd = '' count = 1 hierarchy[so.get_object_name()] = [so.name] dir_structure[so.get_object_name()] = '' while queue: obj = queue.popleft() #print 'Processing:', obj.get_object_name() parent_proj = None if obj.get_type() == 'dir': # Processing a dir set 'working dir' cwd = dir_structure[obj.get_object_name()] parent_proj = proj_lookup[obj.get_object_name()] result = get_members(obj, ccm, parent_proj) objects = [SynergyObject(o['objectname'], delim) for o in result] # if a project is being queried it might have more than one dir with the # same name as the project associated, find the directory that has the # project associated as the directory's parent if obj.get_type() == 'project': if len(objects) > 1: objects = find_root_project(obj, objects, ccm) for o in objects: count +=1 if o.get_type() == 'dir': # add the directory to the queue and record its parent project queue.append(o) dir_structure[o.get_object_name()] = '%s%s/' % (cwd, o.get_name()) if obj.get_type() == 'project': proj_lookup[o.get_object_name()] = obj.get_object_name() elif obj.get_type() == 'dir': proj_lookup[o.get_object_name()] = proj_lookup[obj.get_object_name()] # Also add the directory to the Hierachy to get empty dirs if o.get_object_name() in hierarchy.keys(): hierarchy[o.get_object_name()].append('%s%s' % (cwd, o.get_name())) else: hierarchy[o.get_object_name()] = ['%s%s' % (cwd, o.get_name())] elif o.get_type() == 'project': # Add the project to the queue queue.append(o) # Add the project to the hierarchy, so the subprojects for the release/project is known if o.get_object_name() in hierarchy.keys(): hierarchy[o.get_object_name()].append('%s%s' % (cwd, o.get_name())) else: hierarchy[o.get_object_name()] = ['%s%s' % (cwd, o.get_name())] else: # Add the object to the hierarchy if obj.get_type() == 'dir': if o.get_object_name() in hierarchy.keys(): hierarchy[o.get_object_name()].append('%s%s' % (cwd, o.get_name())) else: hierarchy[o.get_object_name()] = ['%s%s' % (cwd, o.get_name())] #print "Object:", o.get_object_name(), 'has path:' #print '%s%s' % (cwd, o.get_name()) print "Object count: %6d" % count return hierarchy