def test_automl_save_load(): X, y = dummy.load(seed=0) automl = AutoML(search_iterations=3, registry=[DummyAlgorithm]) automl.fit(X, y) pipe = automl.best_pipeline_ fp = BytesIO() automl.save_pipeline(fp) fp.seek(0) automl2 = AutoML(registry=[DummyAlgorithm]) automl2.load_pipeline(fp) pipe2 = automl2.best_pipeline_ assert repr(pipe) == repr(pipe2)
def automl_fit( input: Path, output: Path = Path("automl.bin"), target: str = None, ignore_cols: List[int] = typer.Option([]), evaluation_timeout: int = 5 * Min, memory_limit: int = 4 * Gb, search_timeout: int = 60 * 60, pop_size: int = 20, iterations: int = 100, random_state: int = None, format: str = None, ): """ 🏃 Train an AutoML instance on a dataset. """ try: dataset = _load_dataset(format, input, ignore_cols) except ValueError as e: logger.error(f"⚠️ Error: {str(e)}") return if target is None: target = dataset.columns[-1] columns = [c for c in dataset.columns if c != target] X = dataset[columns].values y = dataset[target].values automl = AutoML( output=VectorCategorical(), search_kwargs=dict( evaluation_timeout=evaluation_timeout, memory_limit=memory_limit, search_timeout=search_timeout, pop_size=pop_size, ), random_state=random_state, search_iterations=iterations, ) console.print(f"🏃 Training on {len(dataset)} items.") automl.fit(X, y, logger=RichLogger()) with output.open("wb") as fp: automl.save(fp) console.print(f"💾 Saving model to [green]{output.absolute()}[/].")
def run_automl_test( dataset, input, output, search_timeout, evaluation_timeout, expected_fitness ): automl = AutoML( input=input, output=output, evaluation_timeout=evaluation_timeout, search_timeout=search_timeout, cross_validation_steps=1, ) X, y = dataset automl.fit(X, y) assert automl.best_score_ >= expected_fitness
def generated_classifier_from_dataset(data: Collection, number_of_models: int = 5): models = [] lines, classes = load_training_entities(data) unique_clases = reduce(lambda x, y: x | y, [set(c) for c in classes]) for _ in range(number_of_models): classifier = AutoML( input=ag_List(ag_List(Word())), output=ag_List(ag_List(Postag())), ) classifier.fit([[w.text for w in l] for l in lines], classes) models.append(classifier) return SentencesAnnotator(models=models, collection_base=data, unique_classes=unique_clases)
def run_automl(X, y, name, input=None, output=CategoricalVector()): telegram = TelegramLogger( token=os.environ["TOKEN"], channel="@autogoal_board", name=name, ) console = ConsoleLogger() progress = ProgressLogger() automl = AutoML( search_iterations=1000, metalearning_log=True, search_kwargs=dict(search_timeout=2 * 60 * 60, pop_size=50), errors="ignore", input=input, output=output, cross_validation_steps=1, ) automl.fit(X, y, logger=[telegram, console, progress])
def test_automl_save_load(): X, y = dummy.generate(seed=0) automl = AutoML( input=(MatrixContinuousDense, Supervised[VectorCategorical]), output=VectorCategorical, search_iterations=3, registry=[DummyAlgorithm], ) automl.fit(X, y) pipe = automl.best_pipeline_ fp = BytesIO() automl.save(fp) fp.seek(0) automl2 = AutoML.load(fp) pipe2 = automl2.best_pipeline_ assert repr(pipe) == repr(pipe2)
# import high-level API from autogoal.ml import AutoML from autogoal.kb import MatrixContinuousDense, CategoricalVector # load data from autogoal.datasets import cars X, y = cars.load() # instantiate AutoML class automl = AutoML( input=MatrixContinuousDense(), output=CategoricalVector(), # ... other parameters and constraints ) # fit the model automl.fit(X, y) # save the best model with open("model.bin", "wb") as fp: automl.save(fp)
X, y = make_blobs( n_samples=n_samples, n_features=n_features, centers=centers, cluster_std=cluster_std, random_state=random_state, ) # # Instantiate AutoML, define input/output types and the score metric automl = AutoML( input=(MatrixContinuousDense, Supervised[VectorCategorical]), output=VectorCategorical, score_metric=silhouette_score, search_iterations=args.iterations, pop_size=args.popsize, selection=args.selection, evaluation_timeout=args.timeout, memory_limit=args.memory * 1024**3, early_stop=args.early_stop, search_timeout=args.global_timeout, target_fn=args.target, ) loggers = [ JsonLogger( f"unsupervised-log-({n_features}, {centers}, {cluster_std}, {random_state}).json" ) ] automl.fit(X, logger=loggers) # generated dataset seed
# AutoGOAL is first and foremost a framework for Automatic Machine Learning. # As such, it comes pre-packaged with hundreds of low-level machine learning # algorithms that can be automatically assembled into pipelines for different problems. # The core of this functionality lies in the [`AutoML`](/api/autogoal.ml#automl) class. # To illustrate the simplicity of its use, we will first load a dataset. from autogoal.datasets import cars X, y = cars.load() # Next, we import and instantiate the [`AutoML`](/api/autogoal.ml#automl) class. from autogoal.ml import AutoML automl = AutoML(errors='ignore') # Finally, we just call its `fit` method. AutoGOAL will automatically infer the input and # output type. automl.fit(X, y) # Sensible defaults are defined for each of the many parameters of `AutoML`. # Make sure to read the documentation for more information on the parameters. # ## Automatic pipeline discovery # AutoGOAL' automatic pipeline discovery relies on suitable type annotations # for the input and output of each algorithm. # For this functionality to work, all possible algorithms to use should # be defined following the "algorithm" protocol.
else: X, y = data X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3) # Finally we can instantiate out `AutoML` with all the custom # parameters we received from the command line. classifier = AutoML( output=CategoricalVector(), search_algorithm=PESearch, search_iterations=args.iterations, search_kwargs=dict( pop_size=args.popsize, selection=args.selection, evaluation_timeout=args.timeout, memory_limit=args.memory * 1024**3, early_stop=args.early_stop, search_timeout=args.global_timeout, target_fn=args.target, ), ) # Here we configure all the logging strategies we will use. # `MemoryLogger` stores each generation's info in a list that we can # later dump into a JSON log file. # `ProgressLogger` and `ConsoleLogger` are for pretty printing the results on the console. logger = MemoryLogger() loggers = [ProgressLogger(), ConsoleLogger(), logger]
from autogoal.contrib import find_classes # ## Experimentation # Instantiate the classifier. # Note that the input and output types here are defined to match the problem statement, # i.e., entity recognition. classifier = AutoML( search_algorithm=PESearch, input=(Tensor4, Supervised[VectorCategorical]), output=VectorCategorical, cross_validation_steps=1, # Since we only want to try neural networks, we restrict # the contrib registry to algorithms matching with `Keras`. registry=find_classes("Keras"), errors="raise", # Since image classifiers are heavy to train, let's give them a longer timeout... evaluation_timeout=5 * Min, search_timeout=1 * Hour, ) # Basic logging configuration. loggers = [RichLogger()] # Finally, loading the CIFAR dataset, running the `AutoML` instance, # and printing the results. from autogoal.datasets import cifar10
from autogoal.contrib.keras import KerasClassifier from autogoal.datasets import cars from autogoal.kb import CategoricalVector, MatrixContinuousDense from autogoal.ml import AutoML from autogoal.search import ConsoleLogger, ProgressLogger classifier = AutoML( input=MatrixContinuousDense(), registry=[KerasClassifier], search_kwargs=dict(memory_limit=0, evaluation_timeout=0), ) X, y = cars.load() classifier.fit(X, y, logger=[ConsoleLogger(), ProgressLogger()])
def test_run_unsupervised(): X = np.array([ [ 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, ], [ 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, ], [ 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, ], [ 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, ], [ 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, ], [ 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, ], [ 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, ], [ 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, ], [ 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, ], [ 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, ], ]) automl = AutoML( input=MatrixContinuousDense, output=VectorCategorical, score_metric=calinski_harabasz_score, search_timeout=120, ) automl.fit(X, logger=loggers)
# Como tenemos que representarlo según los tipos definidos en AutoGOAL # Vamos a importar los tipos que nos hacen falta para HAHA # En este caso podemos verlo como un listado (que se representa con el tipo Seq de Sequence) # de oraciones (que se representa con el tipo Sentence) , ya que los mensajes son muy cortos. # De las que conocemos (al menos para una parte para modelar como supervisado con el tipo Supervised) # la categoría de humor (esta categoría podemos representarla con el tipo VectorCategorical). # Definir el tipo de entrada como las oraciones + las clases supervisadas de las mismas, le deja claro a # AutoGOAL que queremos resolver el problema de forma supervisada con un entrenamiento. from autogoal.kb import Seq, Sentence, VectorCategorical, Supervised # ¿Cómo utilizamos esto en la clase AutoML? automl = AutoML( input=(Seq[Sentence], Supervised[VectorCategorical]), # **tipos de entrada** output=VectorCategorical, # **tipo de salida** # tenemos el parámetro score_metric para definir la función objetivo, # que si no le fijamos un valor utiliza por defecto la función `autogoal.ml.metrics.accuracy`. ) # Ya hasta aquí hemos definido el problema que queremos resolver # ahora solo nos resta ejecutar nuestro algoritmo, llamando al método `fit`. # Para monitorear el estado del proceso de AutoML, podemos pasar un logger al método `fit`. from autogoal.search import RichLogger # Entrenando... automl.fit(X_train, y_train, logger=RichLogger()) # Conociemdo que tan bueno es nuestro algoritmo score = automl.score(X_test, y_test) print(f"Score: {score:0.3f}")
# AutoGOAL Example: basic usage of the AutoML class from autogoal.datasets import cars from autogoal.kb import MatrixContinuousDense, Supervised, VectorCategorical from autogoal.ml import AutoML # Load dataset X, y = cars.load() # Instantiate AutoML and define input/output types automl = AutoML( input=(MatrixContinuousDense, Supervised[VectorCategorical]), output=VectorCategorical, search_iterations=1, ) # Run the pipeline search process automl.fit(X[0:10], y[0:10]) # Report the best pipeline print(automl.best_pipeline_) print(automl.best_score_) # Export the result of the search process onto a brand new image called "AutoGOAL-Cars" automl.export_portable()
# ## Experimentation # Instantiate the classifier. # Note that the input and output types here are defined to match the problem statement, # i.e., entity recognition. from autogoal.contrib import find_classes classifier = AutoML( search_algorithm=PESearch, input=(Seq[Seq[Word]], Supervised[Seq[Seq[Label]]]), output=Seq[Seq[Label]], registry=find_classes(exclude="Keras|Bert"), search_iterations=args.iterations, score_metric=meddocan.F1_beta, cross_validation_steps=1, pop_size=args.popsize, search_timeout=args.global_timeout, evaluation_timeout=args.timeout, memory_limit=args.memory * 1024 ** 3, ) # Basic logging configuration. loggers = [RichLogger()] if args.token: from autogoal.contrib.telegram import TelegramLogger telegram = TelegramLogger(token=args.token, name=f"MEDDOCAN", channel=args.channel,)
from autogoal.contrib import find_classes # ## Experimentation # Instantiate the classifier. # Note that the input and output types here are defined to match the problem statement, # i.e., entity recognition. classifier = AutoML( search_algorithm=PESearch, input=(Seq[Seq[Word]], Supervised[Seq[Seq[Label]]]), output=Seq[Seq[Label]], score_metric=meddocan.F1_beta, cross_validation_steps=1, # Since we only want to try neural networks, we restrict # the contrib registry to algorithms matching with `Keras`. registry=find_classes("Keras|Bert"), # We need to give some extra time because neural networks are slow evaluation_timeout=300, search_timeout=1800, ) # Basic logging configuration. loggers = [RichLogger()] # Finally, loading the MEDDOCAN dataset, running the `AutoML` instance, # and printing the results. X_train, y_train, X_test, y_test = meddocan.load()
else: X, y = data X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3) # Finally we can instantiate out `AutoML` with all the custom # parameters we received from the command line. classifier = AutoML( input=(MatrixContinuousDense, Supervised[VectorCategorical]), output=VectorCategorical, search_algorithm=PESearch, search_iterations=args.iterations, pop_size=args.popsize, selection=args.selection, evaluation_timeout=args.timeout, memory_limit=args.memory * 1024**3, early_stop=args.early_stop, search_timeout=args.global_timeout, target_fn=args.target, ) logger = MemoryLogger() loggers = [RichLogger()] # `TelegramLogger` outputs debug information to a custom Telegram channel, if configured. if args.token: from autogoal.contrib.telegram import TelegramLogger
# AutoGOAL Example: basic usage of the AutoML class from autogoal.datasets import cars from autogoal.kb import MatrixContinuousDense, Supervised, VectorCategorical from autogoal.ml import AutoML # Load dataset X, y = cars.load() # Instantiate AutoML and define input/output types automl = AutoML( input=(MatrixContinuousDense, Supervised[VectorCategorical]), output=VectorCategorical, ) # Run the pipeline search process automl.fit(X, y) # Report the best pipeline print(automl.best_pipeline_) print(automl.best_score_)
import numpy as np from autogoal.ml import AutoML from autogoal.contrib.keras import KerasImageClassifier, KerasImagePreprocessor from autogoal.datasets import cifar10 from autogoal.kb import CategoricalVector, Tensor4 from autogoal.search import ConsoleLogger, ProgressLogger automl = AutoML( input=Tensor4(), output=CategoricalVector(), registry=[KerasImageClassifier], # registry=[KerasImageClassifier, KerasImagePreprocessor], cross_validation_steps=1, search_kwargs=dict( pop_size=20, search_timeout=24 * 60 * 60, evaluation_timeout=0, memory_limit=0, save=False, ), search_iterations=1000, validation_split=1 / 6) Xtrain, ytrain, Xtest, ytest = cifar10.load() X = np.vstack((Xtrain, Xtest)) y = np.hstack((ytrain, ytest)) automl.fit(X, y, logger=[ConsoleLogger(), ProgressLogger()])
automl = AutoML( input=(Seq[Sentence], Supervised[VectorCategorical]), # **tipos de entrada** output=VectorCategorical, # **tipo de salida** # el score_metric define la función objetivo a optimizar y puede ser definida por nosotros en un método propio score_metric=balanced_accuracy_score, # el parámetro registry nos permite seleccionar un conjunto específico de algoritmo a utilizar en nuestra implementación. # Si no se define o se pone None se utilizan todos los algorismos disponibles en AutoGOAL. registry=None, # search_algorithm permite cambiar el algoritmo de optimization que utiliza AutoGOAL, en estos moemntos también está # implementada una búsqueda aleatoria o puedes implementar una nueva clase. search_algorithm=PESearch, # search_iterations se utiliza para definir la cantidad de iteraciones que queremos que haga nuestro algoritmo de búsqueda # osea cantidad de generaciones en la búsqued aevolutiva o en el random search_iterations=args.iterations, # search_kwargs este parámetro se utiliza para pasar opciones adicionales al algoritmo de búsqueda search_kwargs=dict( # pop_size es el tamaño de la población pop_size=args.popsize, # search_timeout es el tiempo máximo total que queremos dedicarle a la búsqueda en segundos search_timeout=args.global_timeout, # evaluation_timeout es el tiempo máximo para un pipeline, si la ejecución del pipeline se pasa de este texto # se detenine y se le asigna fitness cero. evaluation_timeout=args.timeout, # cantidad máxima de RAM por pipeline. Este número debe ser inferior a la RAM del dispositivo donde se ejecute la experimentación # para evitar que el despositivo de bloquee. memory_limit=args.memory * 1024 ** 3, ), # cross_validation_steps cantidad de veces que se evalúa cada pipeline cross_validation_steps=3, # validation_split por ciento del tamaño del training set que se utiliza para cross validation. validation_split=0.3, # cross_validation es la métrica que se utiliza para mezclar los score de los cross_validation_steps. También está "mean" cross_validation="median", # random_state es un número para fijar la semilla random de la búsqueda. Esto nos puede ayudar a que aparezcan pipelines similares a los de # otra ejecución. random_state=None, # errors determina que se hce cuando un pipeline lanza una excepción. "warn" lanza un wargnig, "ïgnore" los ignora y # "raise" que lanza la excepción y detiene la ejecución. errors="warn", )
from autogoal.contrib.keras import KerasSequenceClassifier from autogoal.contrib.torch import BertTokenizeEmbedding from autogoal.datasets import haha from autogoal.kb import CategoricalVector, List, Sentence, Tuple from autogoal.ml import AutoML from autogoal.search import ConsoleLogger, ProgressLogger classifier = AutoML( input=List(Sentence()), output=CategoricalVector(), registry=[KerasSequenceClassifier, BertTokenizeEmbedding], # search_kwargs=dict(memory_limit=4 * 1024 ** 3, evaluation_timeout=60), search_kwargs=dict(memory_limit=0, evaluation_timeout=0), ) Xtrain, Xtest, ytrain, ytest = haha.load(max_examples=10) # embedding = BertEmbedding() # tokens = embedding.run(Xtrain) # classifier = KerasSequenceClassifier().sample() # classifier.run((tokens, ytrain)) classifier.fit(Xtrain, ytrain, logger=[ConsoleLogger(), ProgressLogger()])
# AutoGOAL Example: basic usage of the AutoML class from autogoal.datasets import cars from autogoal.kb import MatrixContinuousDense, Supervised, VectorCategorical from autogoal.search import RichLogger from autogoal.ml import AutoML, calinski_harabasz_score # Load dataset X, y = cars.load() # Instantiate AutoML, define input/output types and the score metric automl = AutoML( input=(MatrixContinuousDense, Supervised[VectorCategorical]), output=VectorCategorical, score_metric=calinski_harabasz_score, ) # Run the pipeline search process automl.fit(X[0:10]) # Report the best pipeline print(automl.best_pipeline_) print(automl.best_score_)
print(args) # ## Experimentation # Instantiate the classifier. # Note that the input and output types here are defined to match the problem statement, # i.e., entity recognition. classifier = AutoML( search_algorithm=PESearch, input=List(List(Word())), output=List(List(Postag())), search_iterations=args.iterations, score_metric=meddocan.F1_beta, cross_validation_steps=1, search_kwargs=dict( pop_size=args.popsize, search_timeout=args.global_timeout, evaluation_timeout=args.timeout, memory_limit=args.memory * 1024 ** 3, ), ) # This custom logger is used for debugging purposes, to be able later to recover # the best pipelines and all the errors encountered in the experimentation process. class CustomLogger(Logger): def error(self, e: Exception, solution): if e and solution: with open("meddocan_errors.log", "a") as fp: fp.write(f"solution={repr(solution)}\nerror={e}\n\n")
for cls in find_classes(): print("Using: %s" % cls.__name__) # ## Experimentation # Instantiate the classifier. # Note that the input and output types here are defined to match the problem statement, # i.e., text classification. classifier = AutoML( search_algorithm=PESearch, input=(Seq[Sentence], Supervised[VectorCategorical]), output=VectorCategorical, search_iterations=args.iterations, score_metric=f1_score, errors="warn", pop_size=args.popsize, search_timeout=args.global_timeout, evaluation_timeout=args.timeout, memory_limit=args.memory * 1024**3, ) loggers = [RichLogger()] if args.token: from autogoal.contrib.telegram import TelegramLogger telegram = TelegramLogger( token=args.token, name=f"HAHA", channel=args.channel,
for cls in find_classes(): print("Using: %s" % cls.__name__) # ## Experimentation # Instantiate the classifier. # Note that the input and output types here are defined to match the problem statement, # i.e., text classification. classifier = AutoML( search_algorithm=PESearch, input=List(Sentence()), output=CategoricalVector(), search_iterations=args.iterations, score_metric=f1_score, search_kwargs=dict( pop_size=args.popsize, search_timeout=args.global_timeout, evaluation_timeout=args.timeout, memory_limit=args.memory * 1024**3, ), ) # This custom logger is used for debugging purposes, to be able later to recover # the best pipelines and all the errors encountered in the experimentation process. class CustomLogger(Logger): def error(self, e: Exception, solution): if e and solution: with open("haha_errors.log", "a") as fp:
# Una vez que tenemos listo nuestro algoritmo solo nos queda indicarle a la clase AutoML que lo utilice en la búsqueda # Estos son algunos import que nos hacen falta más adelante from autogoal.ml import AutoML from autogoal.contrib import find_classes # Probemos con HAHA from autogoal.datasets import haha # Cargando los datos X_train, y_train, X_test, y_test = haha.load() # Creando la instancia de AutoML con nuestra clase automl = AutoML( input=(Seq[Sentence], Supervised[VectorCategorical]), # **tipos de entrada** output=VectorCategorical, # **tipo de salida** # Agregando nuestra clase y todo el resto de algortimos de AutoGOAL registry=[NewAlgorithm] + find_classes(), ) # Ahora sencillamente tenemos que ejecutar AutoML y ya nuestro algoritmo aparecerá en algunos pipelines. # Debemos tener en cuenta que esto no garantiza qeu aparezca en el mejor pipeline encontrado, sino que se conectará # con el resto de los algoritmo como si fuera nativo de AutoGOAL. automl.fit(X_train, y_train) score = automl.score(X_test, y_test) print(score)