def archive(repo_path, archive_path, repo_id, commit, filter_extensions=True): # Determine if we can access the path where the archive should be if not _determine_access(archive_path): logger.error("Failed to save to archive %s" % archive_path) return False temp_archive = os.path.join(repo_path, "%s-temp" % commit) temp_archive_compress_file_no_ext = os.path.join(temp_archive, commit) temp_archive_compress_file = "%s.tar.gz" % temp_archive_compress_file_no_ext archive_repo_path = os.path.join(archive_path, repo_id) archive_compress_file = "%s.tar.gz" % os.path.join(archive_repo_path, commit) _clear_archive(temp_archive, archive_compress_file) target_directories = _identify_target_directories(repo_path) _clone_files_in_targets(repo_path, temp_archive, target_directories, filter_extensions=filter_extensions) _compress_files(temp_archive, temp_archive_compress_file_no_ext) _move_compress_file_to_archive(archive_repo_path, temp_archive_compress_file) # Delete the temporary folder _clear_archive_temp(temp_archive) return True
def get_arg_db(self): """Retrieve info from the table laptop of the db for the specific laptop""" try: get_query = config.QUERY_GET_ARG self.cur.execute(get_query, (self.name,)) db_output = [item for item in self.cur.fetchall()] return db_output except Exception as e: logger.error(f'Selecting arguments from laptop table: {e} ')
def update_db(self): """Update the Laptop in the table laptop of the db""" try: query = config.QUERY_UPDATE_LAPTOP self.cur.execute(query, (self.price, self.rating, self.reviews, datetime.now(), self.name)) self.con.commit() logger.info('Table laptop: updated -> ' + self.name) except Exception as e: logger.error(f'Updating table laptop: {e} ')
def add_to_db(self): """Add the Laptop to the table laptop of the db""" try: query = config.QUERY_INSERT_LAPTOP records = (self.name, self.price, self.rating, self.reviews, self.link, datetime.now(), None, self.valid) self.cur.execute(query, records) self.con.commit() logger.info('Table laptop: added -> ' + self.name) except Exception as e: logger.error(f'Adding record to table laptop: {e} ')
def add_to_db(self, laptop_id): """Add the Review to the table reviews of the db""" try: query = config.QUERY_INSERT_REVIEWS records = (laptop_id, self.user_id, self.username, self.location, self.date, self.rank, self.profile, self.content, datetime.now()) self.cur.execute(query, records) self.con.commit() logger.info('Reviews added for laptop:-> ' + str(laptop_id)) except Exception as e: logger.error(f'Adding record to the table Review: {e}')
def update_db(self): """Update the Profile in the table profile of the db""" try: query = config.QUERY_UPDATE_PROFILE records = (self.ranking, self.review, self.votes, datetime.now(), self.username) self.cur.execute(query, records) self.con.commit() logger.info('Table profile: updated -> ' + self.username) except Exception as e: logger.error(f'Updating table Profile {e}')
def unloadAll(self): if not self.unloaded: for pluginName, pluginObject in self.pluginObjects.items(): try: pluginObject.unload() logger.info("Unloaded plugin: " + pluginName) except: exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() logger.error("Uncaught exception occurred while unloading plugin: " + traceback.format_exc()) self.unloaded = True
def if_exist(self): """Check if the Profile already exists in the table profile of the db""" try: query = config.QUERY_PROFILE_EXIST self.cur.execute(query, (self.username, )) if self.cur.fetchone()[0] != 0: return True else: return False except Exception as e: logger.error(f'Checking existence: {e}')
def add_to_db(self): """Add the Profile to the table profile of the db""" try: query = config.QUERY_INSERT_PROFILE records = (self.username, self.ranking, self.review, self.votes, datetime.now(), None, self.valid) self.cur.execute(query, records) self.con.commit() logger.info('Table profile: added -> ' + self.username) except Exception as e: logger.error(f'Adding to Profile table: {e}')
def if_exist(self): """Check if the Review already exists in the table reviews of the db""" try: query = config.QUERY_REVIEW_EXIST self.cur.execute(query, (self.username, self.location, self.date)) result = self.cur.fetchone()[0] if result != 0: return True else: return False except Exception as e: logger.error(f'Checking existence: {e}')
def read_csv(): # Reads the csv and returns the file object logger.info(" reading the dependency file") try: fname = test_orch_object.data_path + test_orch_object.file_name file = open(fname, 'ra') csv_f = csv.reader(file, delimiter=',') return csv_f except: logger.error("Error in reading the input dependency file") return None
def if_exist(self): """Check if the Laptop already exists in the table laptop of the db""" try: query = config.QUERY_LAPTOP_EXIST self.cur.execute(query, (self.name,)) result = self.cur.fetchone()[0] if result != 0: return True else: return False except Exception as e: logger.error(f'Checking existence: {e} ')
def loadPlugin(self, symbolicName, dependency, dependList): #logger.debug("trying to load plugin: " + symbolicName) """Wrapper to __loadPlugin to provide nested exception handline for all loading of plugins""" try: self.__loadPlugin(symbolicName, dependency, dependList) except UnsatisfiedDependency as e: logger.error("Ignore plugin: " + symbolicName + " -- unsatisfied dependency: " + str(e)) except UnavailableResource as e: logger.error("Failed plugin: " + symbolicName + " -- unsatisfied dependency: " + str(e)) except MalformedPlugin as e: logger.error("Failed plugin: " + symbolicName + " -- failed loading dependency: " + str(e)) except FailedDependency as e: logger.error("Failed plugin: " + symbolicName + " -- dependency previously failed to load: " + str(e)) except InvalidResourceComponent as e: logger.error("Failed plugin: " + symbolicName + " Required resource: " + e.resource + " did not provide a valid " + e.componentType + e.component)
def add_to_db(self, laptop_id, link): """Add the Features to the table laptop_features of the db""" try: query = config.QUERY_INSERT_FEATURES records = (laptop_id, link, self.screen_size, self.max_screen_resolution, self.brand, self.card_description, self.brand_name, self.item_weight, self.operating_system, self.computer_memory_type, self.batteries, self.date, datetime.now(), self.valid) self.cur.execute(query, records) self.con.commit() logger.info('Table features laptop: added -> ' + str(laptop_id)) except Exception as e: logger.error(f'Adding Laptop features: {e}')
def scanManifests(self, pluginPath): self.manifests = {} pluginDirectories = os.listdir(pluginPath) for directory in pluginDirectories: pluginDirectory = pluginPath + '/' + directory manifestPath = pluginDirectory + '/' + manifestFilename if os.path.isdir(pluginDirectory) and os.path.exists(manifestPath): try: manifest = Manifest(pluginDirectory, manifestFilename) self.processManifest(manifest) except MalformedManifest: logger.error("Malformed plugin manifest: " + manifestPath) else: logger.error("Cannot read plugin: " + pluginDirectory)
def __init__(self): logger.info("Initializing the orchestrator") parser = SafeConfigParser() parser.read(CONFIG_FILE) try: global thread_list self.file_name = parser.get(CONFIG_SECTION, "fileName") self.data_path = parser.get(CONFIG_SECTION, "data") self.test_dir = parser.get(CONFIG_SECTION, "test_dir") self.num_threads = parser.get(CONFIG_SECTION, "num_threads") self.test_execution_order = parser.get(CONFIG_SECTION, "test_order_file") # Initializ the thread list as per the number of threads in the configuration for index in range(0, int(self.num_threads), 1): thread_list.append("Thread-" + str(index)) #print thread_list except: traceback.print_exc() logger.error("Error in reading configuration file")
def sequential_test_set_generator(self, label_map): # This finds the test cases that needs to be executed sequentially if they are not in the dependency file # Take label map as input and then compare logger.info("inside the sequential test case executor") reverse_dict = {} reverse_dict = {value: key for key, value in label_map.iteritems()} file_name = self.test_dir + self.test_execution_order try: file_object = open(file_name, 'r') sequence_list = [ ] # To find the test cases that needs to be executed in sequence for line in file_object: test_case = line.strip("\n") if test_case not in reverse_dict: sequence_list.append(test_case) return sequence_list # Returns the sequence of test case that needs to be executed except Exception as e: logger.error("Error in executing test cases in sequence ", e)
def create_tables(): db = connect_to_db() conn = db.cursor(buffered=True) try: conn.execute(config.TABLE1) db.commit() conn.execute(config.TABLE2) db.commit() conn.execute(config.TABLE3) db.commit() conn.execute(config.TABLE4) db.commit() conn.execute(config.KEY_TABLE1) conn.execute(config.KEY_TABLE2) db.commit() logger.info("\n*** Created tables successfully ***\n") except Exception as e: logger.error(f'{e} ') finally: db.close()
def setupLink(linkPath, targetPath): log.debug("\"{}\" -> \"{}\"".format(linkPath, targetPath)) if os.path.islink(linkPath): log.debug("Skipping \"{}\" because it is already a symlink".format(linkPath)) return False if not pathlib.Path(linkPath).exists(): log.debug("Skipping \"{}\" because it doesn't exist".format(linkPath)) return False copyPath = targetPath if pathlib.Path(targetPath).exists(): remotePathBak = targetPath i=1 while pathlib.Path(remotePathBak).exists(): remotePathBak = "{}-bak-{}".format(targetPath, i) i+=1 log.info("\"{}\" already exists. Will backup local files to \"{}\"".format(targetPath, remotePathBak)) copyPath = remotePathBak parentPath = pathlib.Path(copyPath).parent.absolute() if not parentPath.exists(): os.makedirs(parentPath) if os.path.isdir(linkPath): log.debug("\"{}\" is directory".format(linkPath)) shutil.copytree(linkPath, copyPath) shutil.rmtree(linkPath) log.info("Linking \"{}\" -> \"{}\"".format(linkPath, targetPath)) mklinkDir(linkPath, targetPath) elif os.path.isfile(linkPath): log.debug("\"{}\" is file".format(linkPath)) shutil.copy(linkPath, copyPath) os.remove(linkPath) log.info("Linking \"{}\" -> \"{}\"".format(linkPath, targetPath)) mklink(linkPath, targetPath) else: log.error("\"{}\" is neither symlink, directory, nor file".format(linkPath)) return False return True
def loadRequests(self, manifest, dependList): for request in manifest.Requests: for provider in self.getProviderManifests(request.requestName): #check if we have a cycle forming if provider.SymbolicName in dependList: #append the name here for showing the cycle in the exception dependList.append(provider.SymbolicName) self.failed.append(manifest.SymbolicName) raise DependencyCycle(str("->".join(dependList))) #skip ones that have previously failed to load if provider.SymbolicName in self.failed: logger.error(provider.Name + " a " + provider.Provides + " provider previously failed to load.") #else load the provider else: try: self.loadPlugin(provider.SymbolicName, request, dependList) except UnsatisfiedDependency: logger.error(provider.Name + " a " + provider.Provides + " provider failed: missing dependencies.") except MalformedPlugin: logger.error(provider.Name + " a " + provider.Provides + " provider failed: malformed plugin.")
def __loadPlugin(self, symbolicName, dependency, dependList): #dependList holds the list of dependencies along the depth-first cross section of the tree. Used to find cycles. dependList = dependList[:] dependList.append(symbolicName) #get the manifest from the manifest list try: manifest = self.manifests[symbolicName] except KeyError: self.failed.append(symbolicName) raise UnsatisfiedDependency(symbolicName + ":" + dependency.dependencyString) #to check whether the dependency can actually be satisfied by loading this plugin if dependency != None: if dependency.satisfied(manifest.SymbolicName, manifest.Version): pass #dependency is satisfied else: self.failed.append(manifest.SymbolicName) raise UnsatisfiedDependency(symbolicName + ":" + dependency.dependencyString + ". Version present is: " + manifest.Version) #preliminary checks done. Start actually loading the plugin now if not manifest.SymbolicName in self.plugins.keys(): #load the dependencies self.loadDependencies(manifest, dependList) #load the requests self.loadRequests(manifest, dependList) #import the plugin try: pluginModule = __import__(manifest.SymbolicName) except ImportError: exceptionType, exceptionValue, exceptionTraceback = sys.exc_info() logger.error('Uncaught exception occured in command handler.') logger.error(traceback.format_exc()) raise MalformedPlugin(manifest.SymbolicName + ": failed to import.") #get the plugin class from the module try: #pluginObjectClass = pluginModule.__getattribute__(manifest.SymbolicName) pluginObjectClass = pluginModule.__getattribute__("Plugin") except AttributeError: self.failed.append(manifest.SymbolicName) raise MalformedPlugin(manifest.SymbolicName + ": class is not present.") #check that the plugin class is a subclass of Plugin if not issubclass(pluginObjectClass, Plugin): self.failed.append(manifest.SymbolicName) raise MalformedPlugin(manifest.SymbolicName + ": is not derived from Plugin.") #add the plugin object and plugin module to the correct dictionaries self.pluginObjects[manifest.SymbolicName] = pluginObjectClass() self.plugins[manifest.SymbolicName] = pluginModule #load the actual plugin self.pluginObjects[manifest.SymbolicName].load() logger.info("Loaded plugin: " + manifest.Name) self.reloadOrder.append(manifest.SymbolicName) else: pass