def create_model_block(className, projectDirectory='.', dryrun=None, force=None):
"""Generates a Model Block class.
The file will be located in ``project_name/models/blocks/block_name.py``
Parameters
----------
className : string (CamelCase)
Name of the class to be created
"""
project = HypergolProject(projectDirectory=projectDirectory, dryrun=dryrun, force=force)
className = NameString(className)
content = project.render(
templateName='model_block.py.j2',
templateData={'className': className},
filePath=Path(projectDirectory, 'models', 'blocks', className.asFileName)
)
return project.cli_final_message(creationType='ModelBlock', name=className, content=(content, ))
def create_data_model(className,
*args,
projectDirectory='.',
dryrun=None,
force=None,
project=None):
"""Generates domain class from the parameters derived from :class:`.BaseData`
Fails if the target file already exists unless ``force=True`` or ``--force`` in CLI is set.
Parameters
----------
className : string (CamelCase)
Name of the class to be created
projectDirectory : string (default='.')
Location of the project directory, the code will be created in ``projectDirectory/data_models/class_name.py``.
dryrun : bool (default=None)
If set to ``True`` it returns the generated code as a string
force : bool (default=None)
If set to ``True`` it overwrites the target file
*args : List of strings
member variables string
representation of the member variable in "name:type", "name:List[type]" or "name:type:id" format
Returns
-------
content : string
The generated code if ``dryrun`` is specified
"""
if project is None:
project = HypergolProject(projectDirectory=projectDirectory,
dryrun=dryrun,
force=force)
dataModel = DataModel(className=NameString(className), project=project)
for value in args:
dataModel.process_inputs(value)
temporalDependencies = sorted(
list({m.type_
for m in dataModel.conversions if m.isTemporal}))
dataModelDependencies = [{
'snake': m.type_.asSnake,
'name': m.type_
} for m in dataModel.conversions if not m.isTemporal and not m.isObject]
content = (
DataModelRenderer().add(
'from typing import List ',
dataModel.isListDependent).add(
'from datetime import {0} ',
temporalDependencies).add(
' ',
dataModel.isListDependent or len(temporalDependencies) >
0).add('from hypergol import BaseData ').add(
' ',
len(dataModelDependencies) > 0).add(
'from data_models.{snake} import {name}',
dataModelDependencies).
add(' ').add(
' ').add(
'class {className}(BaseData): ',
className=dataModel.className
).add(' ').add(
' def __init__(self, {arguments}): ',
arguments=', '.join(
dataModel.arguments)).add(
' self.{0} = {0} ',
dataModel.names).add(
' ',
len(dataModel.ids) > 0).add(
' def get_id(self): ',
len(dataModel.ids) > 0).add(
' return ({idString}, ) ',
len(dataModel.ids) > 0,
idString=', '.join(dataModel.ids)).
add(' ',
len(dataModel.conversions) > 0).add(
' def to_data(self): ',
len(dataModel.conversions) > 0).add(
' data = self.__dict__.copy() ',
len(dataModel.conversions) > 0).
add(" data['{name}'] = BaseData.to_string(data['{name}']) ",
[{
'name': m.name
} for m in dataModel.conversions if m.isObject]).
add(" data['{name}'] = data['{name}'].{conv}() ",
[{
'name': m.name,
'conv': m.to_
} for m in dataModel.conversions
if not m.isList and not m.isObject]).
add(" data['{name}'] = [v.{conv}() for v in data['{name}']] ",
[{
'name': m.name,
'conv': m.to_
} for m in dataModel.conversions if m.isList]).
add(' return data ',
len(dataModel.conversions) > 0).
add(' ',
len(dataModel.conversions) > 0).
add(' @classmethod ',
len(dataModel.conversions) > 0).
add(' def from_data(cls, data): ',
len(dataModel.conversions) > 0).
add(
" data['{name}'] = BaseData.from_string(data['{name}']) ",
[{
'name': m.name
} for m in dataModel.conversions if m.isObject]
).add(
" data['{name}'] = {type_}.{conv}(data['{name}']) ",
[{
'name': m.name,
'type_': str(m.type_),
'conv': m.from_
}
for m in dataModel.conversions if not m.isList and not m.isObject]
).add(
" data['{name}'] = [{type_}.{conv}(v) for v in data['{name}']] ",
[{
'name': m.name,
'type_': str(m.type_),
'conv': m.from_
} for m in dataModel.conversions if m.isList]
).add(' return cls(**data) ',
len(dataModel.conversions) > 0)).get()
project.create_text_file(content=content,
filePath=Path(project.dataModelsPath,
dataModel.className.asFileName))
project.render(templateName='test_data_models.py.j2',
templateData={
'name': dataModel.className,
'initialisations': ', '.join(dataModel.initialisations)
},
filePath=Path(project.testsPath,
f'test_{dataModel.className.asFileName}'))
return project.cli_final_message(creationType='Class',
name=dataModel.className,
content=(content, ))
def create_model(modelName,
trainingClass,
evaluationClass,
inputClass,
outputClass,
*args,
projectDirectory='.',
dryrun=None,
force=None):
"""Generates stubs for the Tensorflow model, data processing class and training script and shell script to run it from the command line. Shell scripts will be located in the project main directory (which should be the current directory when running them) and model files will be located in ``project_name/models/model_name/*.py``.
After creation the user must implement the ``process_training_batch()`` , ``process_evaluation_batch()``, ``process_input_batch()`` and ``process_output_batch`` member functions that take ``trainingClass``, ``evaluationClass``, ``inputClass`` and ``outputClass`` respectively.
The model must implement the ``get_loss()``, ``produce_metrics()`` and ``get_outputs()`` functions (see documentation of :class:`.BaseTensorflowModel` and the ``Tutorial`` for more detailed instructions)
The training script is generated with example stubs that should be modified to align with the created model.
Parameters
----------
modelName : string
Name of the model
trainingClass : BaseData
Datamodel class (must exist) of the Dataset that contains the training data
evaluationClass : BaseData
Datamodel class (must exist) that will contain the evaluation data
inputClass : BaseData
Datamodel class (must exist) that will be used as the input when serving the model
outputClass : BaseData
Datamodel class (must exist) that will be returned as output when serving the model
*args : BaseTensorflowModelBlock
Names of blocks that will build up the model
"""
project = HypergolProject(projectDirectory=projectDirectory,
dryrun=dryrun,
force=force)
modelName = NameString(modelName)
trainingClass = NameString(trainingClass)
evaluationClass = NameString(evaluationClass)
inputClass = NameString(inputClass)
outputClass = NameString(outputClass)
blocks = [NameString(value) for value in args]
project.check_dependencies(
[trainingClass, evaluationClass, inputClass, outputClass] + blocks)
project.create_model_directory(modelName=modelName)
project.render_simple(templateName='__init__.py.j2',
filePath=Path(project.modelsPath, modelName.asSnake,
'__init__.py'))
content = project.render(templateName='model.py.j2',
templateData={
'name': modelName,
},
filePath=Path(projectDirectory, 'models',
modelName.asSnake,
modelName.asFileName))
batchProcessorContent = project.render(
templateName='batch_processor.py.j2',
templateData={
'name': modelName,
'evaluationClass': evaluationClass,
'outputClass': outputClass,
},
filePath=Path(projectDirectory, 'models', modelName.asSnake,
f'{modelName.asSnake}_batch_processor.py'))
trainModelContent = project.render(
templateName='train_model.py.j2',
templateData={
'modelName':
modelName,
'trainingClass':
trainingClass,
'evaluationClass':
evaluationClass,
'blockDependencies':
[name for name in blocks if project.is_model_block_class(name)],
},
filePath=Path(projectDirectory, 'models', modelName.asSnake,
f'train_{modelName.asFileName}'))
scriptContent = project.render_executable(
templateName='train_model.sh.j2',
templateData={'snakeName': modelName.asSnake},
filePath=Path(projectDirectory, f'train_{modelName.asSnake}.sh'))
serveContent = project.render(
templateName='serve_model.py.j2',
templateData={
'modelName': modelName,
'inputClass': inputClass,
'outputClass': outputClass
},
filePath=Path(projectDirectory, 'models', modelName.asSnake,
f'serve_{modelName.asFileName}'))
serveScriptContent = project.render_executable(
templateName='serve_model.sh.j2',
templateData={'snakeName': modelName.asSnake},
filePath=Path(projectDirectory, f'serve_{modelName.asSnake}.sh'))
return project.cli_final_message(
creationType='Model',
name=modelName,
content=(content, batchProcessorContent, trainModelContent,
scriptContent, serveContent, serveScriptContent))
def create_pipeline(pipeLineName,
*args,
projectDirectory='.',
dryrun=None,
force=None,
project=None):
"""Generates a pipeline script from the parameters
Fails if the target file already exists unless ``force=True`` or ``--force`` in CLI is set.
Generates pipe_line_name.py in pipelines, imports all the classes listed in ``*args`` and creates stubs for them to be filled. Also creates the executable ``pipe_line_name.sh`` in the project directory with examples how to pass parameters from the shell.
Parameters
----------
pipeLineName : string (CamelCase)
Name of the pipeline to be created
projectDirectory : string (default='.')
Location of the project directory, the code will be created in ``projectDirectory/data_models/class_name.py``.
dryrun : bool (default=None)
If set to ``True`` it returns the generated code as a string
force : bool (default=None)
If set to ``True`` it overwrites the target file
*args : List of strings (CamelCase)
Classes to be imported into the generated code from the data model, fails if class not found in either ``data_models`` or ``tasks``
Returns
-------
content : string
The generated code if ``dryrun`` is specified
scriptContent : string
The generated shell script to run the pipeline if ``dryrun`` is specified
"""
if project is None:
project = HypergolProject(projectDirectory=projectDirectory,
dryrun=dryrun,
force=force)
pipeLineName = NameString(pipeLineName)
dependencies = [NameString(value) for value in args]
project.check_dependencies(dependencies)
content = project.render(
templateName='pipeline.py.j2',
templateData={
'snakeName':
pipeLineName.asSnake,
'taskDependencies':
[name for name in dependencies if project.is_task_class(name)],
'dataModelDependencies': [
name for name in dependencies
if project.is_data_model_class(name)
]
},
filePath=Path(projectDirectory, 'pipelines', pipeLineName.asFileName))
scriptContent = project.render_executable(
templateName='pipeline.sh.j2',
templateData={'snakeName': pipeLineName.asSnake},
filePath=Path(projectDirectory, f'{pipeLineName.asSnake}.sh'))
return project.cli_final_message(creationType='PipeLine',
name=pipeLineName,
content=(content, scriptContent))