def get_class(kls):
"""Construct an object from the name of the class in argument.
https://stackoverflow.com/questions/452969/does-python-have-an-equivalent-to-java-class-forname
Args:
kls (str): String with the full name of the class to instanciate (example: beholder.classes.graphRepresentation.strategyNodes.ColorStrategy.ColorNodeStrategy4).
Returns:
bool: An object instance from the class name specify in parameter.
"""
# Example of parameter kls: beholder.classes.graphRepresentation.strategyNodes.ColorStrategy.ColorNodeStrategy4
# We split kls with the "." separator to obtain parts = [beholder, classes, graphRepresentation, strategyNodes, ColorStrategy, ColorNodeStrategy4]
parts = kls.split('.')
# Compute the module. For that, select all elements of parts except the last one. So: module = beholder.classes.graphRepresentation.strategyNodes.ColorStrategy
module = ".".join(parts[:-1])
try:
# Import the module previously computed. So import beholder.classes.graphRepresentation.strategyNodes.ColorStrategy module.
m = __import__(module)
except Exception:
CustomPrint.errorPrint("Error in " +
str(sys._getframe().f_code.co_name))
traceback.print_exc()
sys.exit(1)
# For all the elements of parts except the first one (ie for all the following elements: [classes, graphRepresentation, strategyNodes, ColorStrategy, ColorNodeStrategy4])
for comp in parts[1:]:
# Construct the module during the loop. Ie, at first loop, construct module classes, then module classes.graphRepresentation, then ..., and finaly <class 'beholder.classes.graphRepresentation.strategyNodes.ColorStrategy.ColorNodeStrategy4'>.
m = getattr(m, comp)
# Return object. Example return object of type <class 'beholder.classes.graphRepresentation.strategyNodes.ColorStrategy.ColorNodeStrategy4'>.
return m()
def checkIfFileExist(pathOfFile):
"""Check if file exist in the given path. If not, print error and exit.
Args:
pathOfFile (str): The path of the file we want to test the existance.
"""
# Check if the file exist at the given pathOfFile. If not, print error and exit.
try:
inputFile = open(str(pathOfFile), 'r')
return True
except EnvironmentError as e: # OSError or IOError...
# Input file doesn't exist. Print a warning and exit.
CustomPrint.warningPrint("Warning: the input file " +
str(pathOfFile) + " doesn't exist.")
CustomPrint.warningPrint(os.strerror(e.errno))
#sys.exit(1)
return False
def askConfirmation(question, default="yes"):
"""Ask a yes/no question via raw_input() and return their answer.
Args:
question (str): a string that is the question presented to the user.
default (str, optional): the presumed answer if the user just hits <Enter>.
It must be "yes" (the default), "no" or None (meaning an answer is requiered of the user).
Returns:
bool: True if user says "yes" or False for "no".
Raises:
ValueError: If default is different from None, yes or no.
"""
# Valid answer from the user (yes, no , y, and n).
valid = {"yes": True, "y": True, "no": False, "n": False}
if (default is None):
prompt = " [y/n] "
elif (default == "yes"):
prompt = " [Y/n] "
elif (default == "no"):
prompt = " [y/N] "
else:
raise ValueError("Invalid default answer: '%s'" % default)
# Ask the answer while a valid answer is not given.
while True:
# Print the question in yellow.
CustomPrint.warningPrint(question + prompt)
# Ask the question and retrieve the answer of user in choice variable.
choice = input().lower()
# If the user don't answer the question by typing Enter, and default is not None, the answer is the default choice.
if (default is not None and choice == ''):
return valid[default]
# Else if the user type his answer, return the valid choice.
elif (choice in valid):
return valid[choice]
# Else, print a warning that ask to enter a valid answer and print the question again.
else:
sys.stdout.write(
"Please respond with 'yes' or 'no' (or 'y' or 'n') \n")
def removeFileIfExist(filePath):
# Python 2.x: define [FileNotFoundError] exception if it doesn't exist
try:
FileNotFoundError
except NameError:
FileNotFoundError = IOError
# Handle errors while calling os.remove()
try:
# As file at filePath is deleted now, so we should check if file exists or not before deleting them
if os.path.exists(filePath):
# Remove a file
os.remove(filePath)
else:
CustomPrint.warningPrint("Can not delete the file " +
str(filePath) +
" as it doesn't exists")
except:
CustomPrint.errorPrint("Error while deleting file " +
str(filePath))
traceback.print_exc()
def __displaysetupiaenvImageLogoFullScreen():
"""(Private static method) Display image of setupiaenv logo in Full screen with tkinter.
"""
# Try to open the image of setupiaenv logo.
try:
# Define the local directory where are store images of the project.
imagesDir = 'images'
# Tell the name of the image to display.
imageName = 'setupiaenvLogoFinal.png'
# Form the pah where is located the image to display.
pathOfImage = os.path.join(imagesDir, imageName)
# Use Pillow to open the image with the default image Viewer.
img = Image.open(pathOfImage)
# Show the image (open with Pillow) in Full screen with tkinter.
CliManagementSetupIAEnv.__fit_center(img)
# If the image cannot been found at the define location, print a warning and exit.
except IOError as e:
# Input file doesn't exist. Print a warning and exit.
CustomPrint.errorPrint("Error: the input file " +
str(pathOfImage) + " doesn't exist.")
CustomPrint.errorPrint(str(e))
sys.exit(1)
def __meta_data_script(progName):
"""(Private static method) Define meta-data of the script (name, version, description, help, ...).
Args:
progName (str): the name of the script that call gen_cli_args().
Returns:
str: progName. Program name of the script that call gen_cli_args().
str: progNameHard. Program name of the application (setupiaenv).
str: progVersion. Version number of the script.
str: versionCodeName. Code name of the current version of the script.
str: versionText. Well format text that give progVersion, and versionCodeName information.
str: progDescription. The Description of the program.
str: progExample_text. The examples given to help the user to run the script.
str: progMiscExample_text. The example given in the setupiaenv misc menu.
"""
# Hard code progName.
progNameHard = 'setupiaenv'
# The version of the script.
progVersion = '1.0.0'
# The version code name.
versionCodeName = 'First version'
# Text display when user ask the version of the program.
versionText = """
{} - {}
{} - {}
{} - {}
""".format(CustomPrint.highlight('Program Name', 'red'),
CustomPrint.highlight(progName, 'yellow'),
CustomPrint.highlight('Version', 'red'),
CustomPrint.highlight(progVersion, 'yellow'),
CustomPrint.highlight('Codename', 'red'),
CustomPrint.highlight(versionCodeName, 'yellow'))
# Give the description of the script.
progDescription = """
_______. _______ .___________. __ __ .______ __ ___ _______ .__ __. ____ ____
/ || ____|| || | | | | _ \ | | / \ | ____|| \ | | \ \ / /
| (----`| |__ `---| |----`| | | | | |_) | | | / ^ \ | |__ | \| | \ \/ /
\ \ | __| | | | | | | | ___/ | | / /_\ \ | __| | . ` | \ /
.----) | | |____ | | | `--' | | | | | / _____ \ | |____ | |\ | \ /
|_______/ |_______| |__| \______/ | _| |__| /__/ \__\ |_______||__| \__| \__/
Setup IA and machine-learning environnement
{} - {}
{} - {}
""".format(CustomPrint.highlight('Version', 'red'),
CustomPrint.highlight(progVersion, 'yellow'),
CustomPrint.highlight('Codename', 'red'),
CustomPrint.highlight(versionCodeName, 'yellow'))
# Give examples of how to use the script (general menu).
progExample_text = """Examples:
{}:
python {} -h
setupiaenv -h
{}:
python {} -v
setupiaenv -v
{}:
python {} -h
setupiaenv -h
{}:
python {} misc -h
setupiaenv misc -h
{}:
setupiaenv -b -a -envName iaenv -gpu
setupiaenv -b -a -envName iaenv
setupiaenv -b -a -envName iaenv -gpu -sys
(execute: "setupiaenv -h" to see more options)
setupiaenv misc -l
(execute: "setupiaenv misc -h" to see more options)
{}:
"python {}" can be replace by: "setupiaenv"
""".format(
CustomPrint.highlight('Get help', 'white_underline'), progName,
CustomPrint.highlight('Get version', 'white_underline'), progName,
CustomPrint.highlight('Print sub-command graph help',
'white_underline'), progName,
CustomPrint.highlight('Print sub-command misc help',
'white_underline'), progName,
CustomPrint.highlight('Typical command', 'white_underline'),
CustomPrint.highlight('Note', 'white_underline'), progName)
# Give examples of how to use the script (misc menu).
progMiscExample_text = """Examples:
{}:
python {} misc -h
setupiaenv misc -h
{}:
python {} misc -v
setupiaenv misc -v
{}:
python {} misc -l
setupiaenv misc -l
{}:
python {} misc -bc
setupiaenv misc -bc
{}:
python {} misc -sc
setupiaenv misc -sc
{}:
python {} misc -a
setupiaenv misc -a
{}:
python {} misc -Il
setupiaenv misc -Il
{}:
python {} misc -FIl
setupiaenv misc -FIl
{}:
"python {}" can be replace by: "setupiaenv"
""".format(
CustomPrint.highlight('Get help', 'white_underline'), progName,
CustomPrint.highlight('Get version', 'white_underline'), progName,
CustomPrint.highlight('Print setupiaenv logo',
'white_underline'), progName,
CustomPrint.highlight('Make me a big coffee',
'white_underline'), progName,
CustomPrint.highlight('Make me a small coffee', 'white_underline'),
progName,
CustomPrint.highlight(
'Gives the Answer to the Ultimate Question of Life, the Universe, and Everything',
'white_underline'), progName,
CustomPrint.highlight(
'Display image of setupiaenv logo with the default Image Viewer',
'white_underline'), progName,
CustomPrint.highlight(
'Display image of setupiaenv logo in Full Screen',
'white_underline'), progName,
CustomPrint.highlight('Note', 'white_underline'), progName)
# Return all the meta-data of the script.
return (progName, progNameHard, progVersion, versionCodeName,
versionText, progDescription, progExample_text,
progMiscExample_text)
import ntpath
""" Import packages useful for the ArgumentParser (with password)."""
# argparse is useful to use the argumentParser.
import argparse
""" Import all the custom classes."""
# Custom class to print errors, warnings, ...
from setupiaenv.classes.helpers.CustomPrint import CustomPrint
# Custom class that provide various sys static functions.
from setupiaenv.classes.helpers.SysTools import SysTools
# Import Tkinter.
try:
# for Python3
import tkinter ## notice lowercase 't' in tkinter here
except ImportError:
CustomPrint.warningPrint(
"Warning: Fail to import tkinter. Import Tkinter instead.")
# for Python2
import Tkinter ## notice capitalized T in Tkinter
""" Implementation of custom functions."""
class CliManagementSetupIAEnv():
"""Class for the management of cli for SetupIAEnv.
"""
@staticmethod
def __meta_data_script(progName):
"""(Private static method) Define meta-data of the script (name, version, description, help, ...).
Args:
progName (str): the name of the script that call gen_cli_args().
def main():
"""Main entry point of the splunkQuery application.
"""
# ***********************************************************************************************************************************************************************************************************
# Meta-data of the script.
# Get the name of the script.
progName = os.path.basename(__file__)
THIS_PATH = os.path.realpath(__file__)
THIS_DIR = SysTools.getPathLessPath_leaf(THIS_PATH)
# print(THIS_PATH)
# print(THIS_DIR)
# print(progName)
machine_type = os.uname().machine
# print(machine_type)
sysname = os.uname().sysname
# print(sysname)
# Constant variables.
# When start a task, add [*] icon in blue.
information_icon = CustomPrint.highlight('[*]', 'blue')
# When finished a task, add [+] icon in green.
success_icon = CustomPrint.highlight('[+]', 'green')
# ***********************************************************************************************************************************************************************************************************
# Management of the parameters to give and given to the script.
# Parse cli arguments.
args = CliManagementSetupIAEnv.gen_cli_args(str(progName))
# Check if misc [option] is used. If it is the case, perform the operation and exit.
CliManagementSetupIAEnv.checkIfMiscArguments(args)
# ***********************************************************************************************************************************************************************************************************
# If not force, allows to print warnings.
if (not args.force):
# Check if the script is executed on a 64-bits system.
if (machine_type != "x86_64"):
CustomPrint.warningPrint("WARNING:\n")
CustomPrint.warningPrint(
" Your operating system appears not to be 64-bit, but you are trying to\n"
)
CustomPrint.warningPrint(
" install a 64-bit version of SetupIAEnv.\n")
answer1 = SysTools.askConfirmation(
" Are sure you want to continue the installation? \n",
default="no")
if (not answer1):
CustomPrint.warningPrint("Aborting installation\n")
sys.exit(0)
# Check if the script is executed on a 64-bits system.
if (sysname != "Linux"):
CustomPrint.warningPrint("WARNING:\n")
CustomPrint.warningPrint(
" Your operating system does not appear to be Linux, \n")
CustomPrint.warningPrint(
" but you are trying to install a Linux version of SetupIAEnv.\n"
)
answer2 = SysTools.askConfirmation(
" Are sure you want to continue the installation? \n",
default="no")
if (not answer2):
CustomPrint.warningPrint("Aborting installation\n")
sys.exit(0)
print("\n")
print("Welcome to SetupIAEnv 1.0.0\n")
# Give execution rights to the installation script.
(exitcode0, stdout0, stderr0,
rc0) = SysTools.execute_command("chmod +x data/SetupIAEnv.sh")
# If in batch mode, pass the batch argument to the script.
if (args.batch):
# If we want to install Anaconda, pass the anaconda argument to the script.
if (args.anaconda):
# Run with bash -i because of this issue: https://stackoverflow.com/questions/55507519/python-activate-conda-env-through-shell-script
(exitcode1, stdout1, stderr1, rc1) = SysTools.execute_command(
"bash -i data/SetupIAEnv.sh batch anaconda " +
str(args.condaEnvName) + " " + str(args.gpuTensorflow) + " " +
str(args.installSystem))
# If we doesn't want to install Anaconda, pass the noAnaconda argument to the script.
else:
# Run with bash -i because of this issue: https://stackoverflow.com/questions/55507519/python-activate-conda-env-through-shell-script
(exitcode1, stdout1, stderr1, rc1) = SysTools.execute_command(
"bash -i data/SetupIAEnv.sh batch NoAnaconda" +
str(args.condaEnvName) + " " + str(args.gpuTensorflow) + " " +
str(args.installSystem))
# If not in batch mode, pass the noBatch argument to the script.
else:
# If we want to install Anaconda, pass the anaconda argument to the script.
if (args.anaconda):
# Run with bash -i because of this issue: https://stackoverflow.com/questions/55507519/python-activate-conda-env-through-shell-script
(exitcode1, stdout1, stderr1, rc1) = SysTools.execute_command(
"bash -i data/SetupIAEnv.sh noBatch anaconda" +
str(args.condaEnvName) + " " + str(args.gpuTensorflow) + " " +
str(args.installSystem))
# If we doesn't want to install Anaconda, pass the noAnaconda argument to the script.
else:
# Run with bash -i because of this issue: https://stackoverflow.com/questions/55507519/python-activate-conda-env-through-shell-script
(exitcode1, stdout1, stderr1, rc1) = SysTools.execute_command(
"bash -i data/SetupIAEnv.sh noBatch NoAnaconda" +
str(args.condaEnvName) + " " + str(args.gpuTensorflow) + " " +
str(args.installSystem))
def dataToDataframe(args,
information_icon=CustomPrint.highlight('[*]', 'blue'),
success_icon=CustomPrint.highlight('[+]', 'green')):
"""Read a file in universal format and convert it to a pandas dataframe.
Args:
args (argparse object): The argparse object that contains all the arguments given by the user in the Cli.
DebugLevel (int): The Debug Level.
Returns:
Pandas dataframe: The dataframe representation of the data.
"""
# ******If the input file is a .json file, we convert the data of that json in a dataframe with jsonToDataframe() function.******
if (hasattr(args, 'input') and hasattr(args, 'debug')):
# If the input file is a json file...
if (".json" in str(SysTools.path_leaf(args.input))):
if (args.debug):
print(
"\n***********************JSON TO DATAFRAME***********************"
)
if (args.debug):
print("\n" + information_icon +
" Convert json to dataframe...")
# Convert json (contained in the input file) to dataframe.
df = ConvertToDataframe.jsonToDataframe(str(args.input))
# Check all graph arguments (i.e.: -s, -dest, -ea, -isdir, -w and -l).
#CliManagementBeholder.checkGraphArguments(args, df)
if (args.debug):
print(success_icon + " Convert json to dataframe: done!\n")
if (args.debug):
print("Print the dataframe (from normalized json):")
print(df)
return df
# ******If the input file is a .csv file, we convert the data of that json in a dataframe with csvToDataframe() function.******
# If the input file is a csv file...
elif (".csv" in str(SysTools.path_leaf(args.input))):
if (args.debug):
print(
"\n***********************CSV TO DATAFRAME***********************"
)
if (args.debug):
print("\n" + information_icon +
" Convert csv to dataframe...")
# Convert csv to dataframe
df = ConvertToDataframe.csvToDataframe(str(args.input))
# Check all graph arguments (i.e.: -s, -dest, -ea, -isdir, -w and -l).
#CliManagementBeholder.checkGraphArguments(args, df)
if (args.debug):
print(success_icon + " Convert csv to dataframe: done!\n")
if (args.debug):
print("Print the dataframe:")
print(df)
return df
# If the input file is none of the previous type, throw an error and exit.
else:
CustomPrint.errorPrint(
"Error: the support format for the input string are: 'atom', 'csv', 'json', 'json_rows', 'raw' and 'xml'. Therefore the file extension of the input file should be one of these."
)
sys.exit(1)
else:
CustomPrint.errorPrint("Error: the parameter args in " +
str(sys._getframe().f_code.co_name) +
" must have input and debug as attributes.")
#traceback.print_exc()
sys.exit(1)
def cleanDataframe(dataframe,
args,
information_icon=CustomPrint.highlight('[*]', 'blue'),
success_icon=CustomPrint.highlight('[+]', 'green')):
"""Remove rows with null value in the source and destination columns.
Args:
dataframe (Pandas dataframe): the pandas dataframe to clean.
args (argparse object): argparse object.
information_icon (str, optional): string to mark the begining of a task.
success_icon (str, optional): string to mark the success of a task.
Returns:
Pandas dataframe: The cleaned dataframe.
"""
if (hasattr(args, 'source') and hasattr(args, 'debug')
and hasattr(args, 'arguments')
and hasattr(args, 'destination')):
# Check validity of -s argument.
# If args.source is not a column name, throw an error.
if (str(args.source) not in dataframe.columns):
CustomPrint.errorPrint(
"Error: source argument must be a column name.\nShould be one of the following: "
+ str(dataframe.columns))
sys.exit(1)
# If the debug level is greater or equal to 1 or if -a option is provided, print debug/arguments information.
if (args.debug or args.arguments):
print("Value of the destination argument: " +
str(args.destination))
# Check validity of -dest argument.
# If args.destination is not a column name, throw an error.
if (str(args.destination) not in dataframe.columns):
CustomPrint.errorPrint(
"Error: destination argument must be a column name.\nShould be one of the following: "
+ str(dataframe.columns))
sys.exit(1)
# If column source and destination are the same, throw an error.
if (str(args.source) == str(args.destination)):
CustomPrint.errorPrint(
"Error: source and destination argument must be different."
)
sys.exit(1)
# Copy the dataframe because we don't want to modify the original dataframe.
df = dataframe.copy()
if (args.debug):
print(
"\n" + information_icon +
" Cleaning dataframe: Remove rows with null value in the source and destination columns..."
)
# Remove rows with null value in the source column.
df = df[pd.notnull(df[str(args.source)])]
# Remove rows with null value in the destination column.
df = df[pd.notnull(df[str(args.destination)])]
if (args.debug):
print(
success_icon +
" Cleaning dataframe: Remove rows with null value in the source and destination columns: done!\n"
)
# Return the cleaned dataframe.
return df
else:
CustomPrint.errorPrint(
"Error: the parameter args in " +
str(sys._getframe().f_code.co_name) +
" must have source, debug, arguments and destination as attributes."
)
#traceback.print_exc()
sys.exit(1)