def archive(repo_path, archive_path, repo_id, commit, filter_extensions=True):
# Determine if we can access the path where the archive should be
if not _determine_access(archive_path):
logger.error("Failed to save to archive %s" % archive_path)
return False
temp_archive = os.path.join(repo_path, "%s-temp" % commit)
temp_archive_compress_file_no_ext = os.path.join(temp_archive, commit)
temp_archive_compress_file = "%s.tar.gz" % temp_archive_compress_file_no_ext
archive_repo_path = os.path.join(archive_path, repo_id)
archive_compress_file = "%s.tar.gz" % os.path.join(archive_repo_path, commit)
_clear_archive(temp_archive, archive_compress_file)
target_directories = _identify_target_directories(repo_path)
_clone_files_in_targets(repo_path, temp_archive, target_directories, filter_extensions=filter_extensions)
_compress_files(temp_archive, temp_archive_compress_file_no_ext)
_move_compress_file_to_archive(archive_repo_path, temp_archive_compress_file)
# Delete the temporary folder
_clear_archive_temp(temp_archive)
return True
def get_arg_db(self):
"""Retrieve info from the table laptop of the db for the specific laptop"""
try:
get_query = config.QUERY_GET_ARG
self.cur.execute(get_query, (self.name,))
db_output = [item for item in self.cur.fetchall()]
return db_output
except Exception as e:
logger.error(f'Selecting arguments from laptop table: {e} ')
def update_db(self):
"""Update the Laptop in the table laptop of the db"""
try:
query = config.QUERY_UPDATE_LAPTOP
self.cur.execute(query, (self.price, self.rating, self.reviews, datetime.now(), self.name))
self.con.commit()
logger.info('Table laptop: updated -> ' + self.name)
except Exception as e:
logger.error(f'Updating table laptop: {e} ')
def add_to_db(self):
"""Add the Laptop to the table laptop of the db"""
try:
query = config.QUERY_INSERT_LAPTOP
records = (self.name, self.price, self.rating, self.reviews, self.link, datetime.now(), None, self.valid)
self.cur.execute(query, records)
self.con.commit()
logger.info('Table laptop: added -> ' + self.name)
except Exception as e:
logger.error(f'Adding record to table laptop: {e} ')
def add_to_db(self, laptop_id):
"""Add the Review to the table reviews of the db"""
try:
query = config.QUERY_INSERT_REVIEWS
records = (laptop_id, self.user_id, self.username, self.location, self.date, self.rank, self.profile, self.content, datetime.now())
self.cur.execute(query, records)
self.con.commit()
logger.info('Reviews added for laptop:-> ' + str(laptop_id))
except Exception as e:
logger.error(f'Adding record to the table Review: {e}')
def update_db(self):
"""Update the Profile in the table profile of the db"""
try:
query = config.QUERY_UPDATE_PROFILE
records = (self.ranking, self.review, self.votes, datetime.now(),
self.username)
self.cur.execute(query, records)
self.con.commit()
logger.info('Table profile: updated -> ' + self.username)
except Exception as e:
logger.error(f'Updating table Profile {e}')
def unloadAll(self):
if not self.unloaded:
for pluginName, pluginObject in self.pluginObjects.items():
try:
pluginObject.unload()
logger.info("Unloaded plugin: " + pluginName)
except:
exceptionType, exceptionValue, exceptionTraceback = sys.exc_info()
logger.error("Uncaught exception occurred while unloading plugin: " + traceback.format_exc())
self.unloaded = True
def if_exist(self):
"""Check if the Profile already exists in the table profile of the db"""
try:
query = config.QUERY_PROFILE_EXIST
self.cur.execute(query, (self.username, ))
if self.cur.fetchone()[0] != 0:
return True
else:
return False
except Exception as e:
logger.error(f'Checking existence: {e}')
def add_to_db(self):
"""Add the Profile to the table profile of the db"""
try:
query = config.QUERY_INSERT_PROFILE
records = (self.username, self.ranking, self.review, self.votes,
datetime.now(), None, self.valid)
self.cur.execute(query, records)
self.con.commit()
logger.info('Table profile: added -> ' + self.username)
except Exception as e:
logger.error(f'Adding to Profile table: {e}')
def if_exist(self):
"""Check if the Review already exists in the table reviews of the db"""
try:
query = config.QUERY_REVIEW_EXIST
self.cur.execute(query, (self.username, self.location, self.date))
result = self.cur.fetchone()[0]
if result != 0:
return True
else:
return False
except Exception as e:
logger.error(f'Checking existence: {e}')
def read_csv():
# Reads the csv and returns the file object
logger.info(" reading the dependency file")
try:
fname = test_orch_object.data_path + test_orch_object.file_name
file = open(fname, 'ra')
csv_f = csv.reader(file, delimiter=',')
return csv_f
except:
logger.error("Error in reading the input dependency file")
return None
def if_exist(self):
"""Check if the Laptop already exists in the table laptop of the db"""
try:
query = config.QUERY_LAPTOP_EXIST
self.cur.execute(query, (self.name,))
result = self.cur.fetchone()[0]
if result != 0:
return True
else:
return False
except Exception as e:
logger.error(f'Checking existence: {e} ')
def loadPlugin(self, symbolicName, dependency, dependList):
#logger.debug("trying to load plugin: " + symbolicName)
"""Wrapper to __loadPlugin to provide nested exception handline for all loading of plugins"""
try:
self.__loadPlugin(symbolicName, dependency, dependList)
except UnsatisfiedDependency as e:
logger.error("Ignore plugin: " + symbolicName + " -- unsatisfied dependency: " + str(e))
except UnavailableResource as e:
logger.error("Failed plugin: " + symbolicName + " -- unsatisfied dependency: " + str(e))
except MalformedPlugin as e:
logger.error("Failed plugin: " + symbolicName + " -- failed loading dependency: " + str(e))
except FailedDependency as e:
logger.error("Failed plugin: " + symbolicName + " -- dependency previously failed to load: " + str(e))
except InvalidResourceComponent as e:
logger.error("Failed plugin: " + symbolicName + " Required resource: " + e.resource + " did not provide a valid " + e.componentType + e.component)
def add_to_db(self, laptop_id, link):
"""Add the Features to the table laptop_features of the db"""
try:
query = config.QUERY_INSERT_FEATURES
records = (laptop_id, link, self.screen_size,
self.max_screen_resolution, self.brand,
self.card_description, self.brand_name,
self.item_weight, self.operating_system,
self.computer_memory_type, self.batteries, self.date,
datetime.now(), self.valid)
self.cur.execute(query, records)
self.con.commit()
logger.info('Table features laptop: added -> ' + str(laptop_id))
except Exception as e:
logger.error(f'Adding Laptop features: {e}')
def scanManifests(self, pluginPath):
self.manifests = {}
pluginDirectories = os.listdir(pluginPath)
for directory in pluginDirectories:
pluginDirectory = pluginPath + '/' + directory
manifestPath = pluginDirectory + '/' + manifestFilename
if os.path.isdir(pluginDirectory) and os.path.exists(manifestPath):
try:
manifest = Manifest(pluginDirectory, manifestFilename)
self.processManifest(manifest)
except MalformedManifest:
logger.error("Malformed plugin manifest: " + manifestPath)
else:
logger.error("Cannot read plugin: " + pluginDirectory)
def __init__(self):
logger.info("Initializing the orchestrator")
parser = SafeConfigParser()
parser.read(CONFIG_FILE)
try:
global thread_list
self.file_name = parser.get(CONFIG_SECTION, "fileName")
self.data_path = parser.get(CONFIG_SECTION, "data")
self.test_dir = parser.get(CONFIG_SECTION, "test_dir")
self.num_threads = parser.get(CONFIG_SECTION, "num_threads")
self.test_execution_order = parser.get(CONFIG_SECTION,
"test_order_file")
# Initializ the thread list as per the number of threads in the configuration
for index in range(0, int(self.num_threads), 1):
thread_list.append("Thread-" + str(index))
#print thread_list
except:
traceback.print_exc()
logger.error("Error in reading configuration file")
def sequential_test_set_generator(self, label_map):
# This finds the test cases that needs to be executed sequentially if they are not in the dependency file
# Take label map as input and then compare
logger.info("inside the sequential test case executor")
reverse_dict = {}
reverse_dict = {value: key for key, value in label_map.iteritems()}
file_name = self.test_dir + self.test_execution_order
try:
file_object = open(file_name, 'r')
sequence_list = [
] # To find the test cases that needs to be executed in sequence
for line in file_object:
test_case = line.strip("\n")
if test_case not in reverse_dict:
sequence_list.append(test_case)
return sequence_list # Returns the sequence of test case that needs to be executed
except Exception as e:
logger.error("Error in executing test cases in sequence ", e)
def create_tables():
db = connect_to_db()
conn = db.cursor(buffered=True)
try:
conn.execute(config.TABLE1)
db.commit()
conn.execute(config.TABLE2)
db.commit()
conn.execute(config.TABLE3)
db.commit()
conn.execute(config.TABLE4)
db.commit()
conn.execute(config.KEY_TABLE1)
conn.execute(config.KEY_TABLE2)
db.commit()
logger.info("\n*** Created tables successfully ***\n")
except Exception as e:
logger.error(f'{e} ')
finally:
db.close()
def setupLink(linkPath, targetPath):
log.debug("\"{}\" -> \"{}\"".format(linkPath, targetPath))
if os.path.islink(linkPath):
log.debug("Skipping \"{}\" because it is already a symlink".format(linkPath))
return False
if not pathlib.Path(linkPath).exists():
log.debug("Skipping \"{}\" because it doesn't exist".format(linkPath))
return False
copyPath = targetPath
if pathlib.Path(targetPath).exists():
remotePathBak = targetPath
i=1
while pathlib.Path(remotePathBak).exists():
remotePathBak = "{}-bak-{}".format(targetPath, i)
i+=1
log.info("\"{}\" already exists. Will backup local files to \"{}\"".format(targetPath, remotePathBak))
copyPath = remotePathBak
parentPath = pathlib.Path(copyPath).parent.absolute()
if not parentPath.exists():
os.makedirs(parentPath)
if os.path.isdir(linkPath):
log.debug("\"{}\" is directory".format(linkPath))
shutil.copytree(linkPath, copyPath)
shutil.rmtree(linkPath)
log.info("Linking \"{}\" -> \"{}\"".format(linkPath, targetPath))
mklinkDir(linkPath, targetPath)
elif os.path.isfile(linkPath):
log.debug("\"{}\" is file".format(linkPath))
shutil.copy(linkPath, copyPath)
os.remove(linkPath)
log.info("Linking \"{}\" -> \"{}\"".format(linkPath, targetPath))
mklink(linkPath, targetPath)
else:
log.error("\"{}\" is neither symlink, directory, nor file".format(linkPath))
return False
return True
def loadRequests(self, manifest, dependList):
for request in manifest.Requests:
for provider in self.getProviderManifests(request.requestName):
#check if we have a cycle forming
if provider.SymbolicName in dependList:
#append the name here for showing the cycle in the exception
dependList.append(provider.SymbolicName)
self.failed.append(manifest.SymbolicName)
raise DependencyCycle(str("->".join(dependList)))
#skip ones that have previously failed to load
if provider.SymbolicName in self.failed:
logger.error(provider.Name + " a " + provider.Provides + " provider previously failed to load.")
#else load the provider
else:
try:
self.loadPlugin(provider.SymbolicName, request, dependList)
except UnsatisfiedDependency:
logger.error(provider.Name + " a " + provider.Provides + " provider failed: missing dependencies.")
except MalformedPlugin:
logger.error(provider.Name + " a " + provider.Provides + " provider failed: malformed plugin.")
def __loadPlugin(self, symbolicName, dependency, dependList):
#dependList holds the list of dependencies along the depth-first cross section of the tree. Used to find cycles.
dependList = dependList[:]
dependList.append(symbolicName)
#get the manifest from the manifest list
try:
manifest = self.manifests[symbolicName]
except KeyError:
self.failed.append(symbolicName)
raise UnsatisfiedDependency(symbolicName + ":" + dependency.dependencyString)
#to check whether the dependency can actually be satisfied by loading this plugin
if dependency != None:
if dependency.satisfied(manifest.SymbolicName, manifest.Version):
pass #dependency is satisfied
else:
self.failed.append(manifest.SymbolicName)
raise UnsatisfiedDependency(symbolicName + ":" + dependency.dependencyString + ". Version present is: " + manifest.Version)
#preliminary checks done. Start actually loading the plugin now
if not manifest.SymbolicName in self.plugins.keys():
#load the dependencies
self.loadDependencies(manifest, dependList)
#load the requests
self.loadRequests(manifest, dependList)
#import the plugin
try:
pluginModule = __import__(manifest.SymbolicName)
except ImportError:
exceptionType, exceptionValue, exceptionTraceback = sys.exc_info()
logger.error('Uncaught exception occured in command handler.')
logger.error(traceback.format_exc())
raise MalformedPlugin(manifest.SymbolicName + ": failed to import.")
#get the plugin class from the module
try:
#pluginObjectClass = pluginModule.__getattribute__(manifest.SymbolicName)
pluginObjectClass = pluginModule.__getattribute__("Plugin")
except AttributeError:
self.failed.append(manifest.SymbolicName)
raise MalformedPlugin(manifest.SymbolicName + ": class is not present.")
#check that the plugin class is a subclass of Plugin
if not issubclass(pluginObjectClass, Plugin):
self.failed.append(manifest.SymbolicName)
raise MalformedPlugin(manifest.SymbolicName + ": is not derived from Plugin.")
#add the plugin object and plugin module to the correct dictionaries
self.pluginObjects[manifest.SymbolicName] = pluginObjectClass()
self.plugins[manifest.SymbolicName] = pluginModule
#load the actual plugin
self.pluginObjects[manifest.SymbolicName].load()
logger.info("Loaded plugin: " + manifest.Name)
self.reloadOrder.append(manifest.SymbolicName)
else:
pass
