class TestCreateModelBlock(HypergolCreateTestCase):
def __init__(self, methodName):
super(TestCreateModelBlock, self).__init__(projectName='TestProject',
methodName=methodName)
self.allPaths = [
Path(self.projectDirectory, 'models', 'blocks',
'test_model_block.py'),
Path(self.projectDirectory, 'models', 'blocks'),
Path(self.projectDirectory, 'models'),
Path(self.projectDirectory)
]
self.project = None
def setUp(self):
super().setUp()
self.project = HypergolProject(
projectDirectory=self.projectDirectory,
repoManager=TestRepoManager(raiseIfDirty=False))
self.project.create_project_directory()
self.project.create_models_directory()
self.project.create_blocks_directory()
def test_create_model_block_creates_files(self):
create_model_block(className='TestModelBlock',
projectDirectory=self.projectDirectory)
for filePath in self.allPaths:
self.assertEqual(os.path.exists(filePath), True)
def test_create_model_block_creates_content(self):
content = create_model_block(className='TestModelBlock',
projectDirectory=self.projectDirectory,
dryrun=True)
self.assertEqual(content[0], TEST_MODEL_BLOCK)
class TestCreatePipeline(HypergolCreateTestCase):
def __init__(self, methodName):
super(TestCreatePipeline, self).__init__(projectName='TestProject', methodName=methodName)
self.allPaths = [
Path(self.projectDirectory, 'test_pipeline.sh'),
Path(self.projectDirectory, 'pipelines', 'test_pipeline.py'),
Path(self.projectDirectory, 'pipelines'),
Path(self.projectDirectory)
]
self.project = None
def setUp(self):
super().setUp()
self.project = HypergolProject(
projectDirectory=self.projectDirectory,
repoManager=TestRepoManager(raiseIfDirty=False)
)
self.project.create_project_directory()
self.project.create_pipelines_directory()
def test_create_pipeline_creates_files(self):
create_pipeline(pipeLineName='TestPipeline', projectDirectory=self.projectDirectory)
for filePath in self.allPaths:
self.assertEqual(os.path.exists(filePath), True)
@mock.patch('hypergol.cli.create_pipeline.HypergolProject.check_dependencies')
@mock.patch('hypergol.cli.create_pipeline.HypergolProject.is_data_model_class', side_effect=lambda x: x.asClass in ['DataModelTestClass'])
@mock.patch('hypergol.cli.create_pipeline.HypergolProject.is_task_class', side_effect=lambda x: x.asClass in ['OtherTask', 'ExampleSource'])
def test_create_pipeline_creates_content(self, mock_is_task_class, mock_is_data_model_class, mock_check_dependencies):
content, scriptContent = create_pipeline('TestPipeline', 'DataModelTestClass', 'ExampleSource', 'OtherTask', projectDirectory='test_project', dryrun=True)
self.assertEqual(content, TEST_CONTENT)
self.assertEqual(scriptContent, TEST_SHELL)
class TestCreateTask(HypergolCreateTestCase):
def __init__(self, methodName):
super(TestCreateTask, self).__init__(projectName='TestProject',
methodName=methodName)
self.allPaths = [
Path(self.projectDirectory, 'tasks', 'test_task.py'),
Path(self.projectDirectory, 'tasks'),
Path(self.projectDirectory)
]
self.project = None
self.maxDiff = 10000
def setUp(self):
super().setUp()
self.project = HypergolProject(
projectDirectory=self.projectDirectory,
repoManager=TestRepoManager(raiseIfDirty=False))
self.project.create_project_directory()
self.project.create_tasks_directory()
def test_create_task_creates_files(self):
create_task(className='TestTask',
projectDirectory=self.projectDirectory)
for filePath in self.allPaths:
self.assertEqual(os.path.exists(filePath), True)
def test_create_task_creates_content(self):
content = create_task(className='TestTask',
projectDirectory=self.projectDirectory,
dryrun=True)
self.assertEqual(content[0], TEST_TASK)
def test_create_task_creates_content_source(self):
content = create_task(className='TestSource',
source=True,
projectDirectory=self.projectDirectory,
dryrun=True)
self.assertEqual(content[0], TEST_SOURCE)
class TestCreateModel(HypergolCreateTestCase):
def __init__(self, methodName):
super(TestCreateModel, self).__init__(projectName='TestProject',
methodName=methodName)
self.allPaths = [
Path(self.projectDirectory, 'models', 'test_model',
'test_model.py'),
Path(self.projectDirectory, 'models', 'test_model',
'test_model_batch_processor.py'),
Path(self.projectDirectory, 'models', 'test_model',
'train_test_model.py'),
Path(self.projectDirectory, 'models', 'test_model',
'serve_test_model.py'),
Path(self.projectDirectory, 'models', 'test_model'),
Path(self.projectDirectory, 'models'),
Path(self.projectDirectory, 'train_test_model.sh'),
Path(self.projectDirectory, 'serve_test_model.sh'),
Path(self.projectDirectory)
]
self.project = None
self.maxDiff = 30000
def setUp(self):
super().setUp()
self.project = HypergolProject(
projectDirectory=self.projectDirectory,
repoManager=TestRepoManager(raiseIfDirty=False))
self.project.create_project_directory()
self.project.create_models_directory()
@mock.patch(
'hypergol.cli.create_pipeline.HypergolProject.check_dependencies')
def test_create_model_creates_files(self, mock_check_dependencies):
create_model(modelName='TestModel',
trainingClass='TestTrainingClass',
evaluationClass='TestEvaluationClass',
inputClass='TestInput',
outputClass='TestOutput',
projectDirectory=self.projectDirectory)
for filePath in self.allPaths:
self.assertEqual(os.path.exists(filePath), True)
@mock.patch(
'hypergol.cli.create_pipeline.HypergolProject.check_dependencies')
@mock.patch(
'hypergol.cli.create_pipeline.HypergolProject.is_model_block_class',
side_effect=lambda x: x.asClass in ['TestBlock1', 'TestBlock2'])
def test_create_model_creates_content(self, mock_is_model_block_class,
mock_check_dependencies):
content, batchProcessorContent, trainModelContent, scriptContent, serveContent, serveScriptContent = create_model(
'TestModel',
'TestTrainingClass',
'TestEvaluationClass',
'TestInput',
'TestOutput',
'TestBlock1',
'TestBlock2',
projectDirectory=self.projectDirectory,
dryrun=True)
self.assertEqual(content, TEST_CONTENT)
self.assertEqual(batchProcessorContent, TEST_BATCH_PROCESSOR)
self.assertEqual(trainModelContent, TEST_TRAIN_MODEL)
self.assertEqual(scriptContent, TEST_SCRIPT)
self.assertEqual(serveContent, TEST_SERVE)
self.assertEqual(serveScriptContent, TEST_SERVE_SCRIPT)
def create_project(projectName, dryrun=None, force=None):
"""Generates the project directories and files
Fails if the target directory already exists unless ``force=True`` or ``--force`` in CLI is set.
Directories:
- ``data_models`` with ``__init__.py``
- ``pipelines`` with ``__init__.py``
- ``tasks`` with ``__init__.py``
- ``models`` with ``__init__.py``
- ``models\blocks`` with ``__init__.py``
- ``tests``
Executables:
- ``make_venv.sh`` to create a virtual environment
- ``run_tests.sh`` to run tests
- ``run_pylint.sh`` to run linting
Misc:
- ``requirements.txt``
- ``.gitignore``
- ``README.md``
- ``LICENSE`` <- Don't forget to add current year and your name or change it to the one you want
- ``pylintrc``
Parameters
----------
projectName : string (CamelCase)
Name of the project to be created
dryrun : bool (default=None)
If set to ``True`` it returns the generated code as a string
force : bool (default=None)
If set to ``True`` it overwrites the target file
"""
projectName = NameString(projectName)
project = HypergolProject(projectDirectory=projectName.asSnake,
dryrun=dryrun,
force=force)
project.create_project_directory()
project.create_data_models_directory()
project.render_simple(templateName='__init__.py.j2',
filePath=Path(project.dataModelsPath, '__init__.py'))
project.create_tasks_directory()
project.render_simple(templateName='__init__.py.j2',
filePath=Path(project.tasksPath, '__init__.py'))
project.create_pipelines_directory()
project.render_simple(templateName='__init__.py.j2',
filePath=Path(project.pipelinesPath, '__init__.py'))
project.create_models_directory()
project.render_simple(templateName='__init__.py.j2',
filePath=Path(project.modelsPath, '__init__.py'))
project.create_blocks_directory()
project.render_simple(templateName='__init__.py.j2',
filePath=Path(project.blocksPath, '__init__.py'))
project.create_tests_directory()
makeVenvScript = project.render_executable(templateName='make_venv.sh.j2',
templateData={},
filePath=Path(
project.projectDirectory,
'make_venv.sh'))
runTestScript = project.render_executable(templateName='run_tests.sh.j2',
templateData={},
filePath=Path(
project.projectDirectory,
'run_tests.sh'))
runPylintScript = project.render_executable(
templateName='run_pylint.sh.j2',
templateData={},
filePath=Path(project.projectDirectory, 'run_pylint.sh'))
requirementsContent = project.render_simple(
templateName='requirements.txt.j2',
filePath=Path(project.projectDirectory, 'requirements.txt'))
gitignoreContent = project.render_simple(templateName='.gitignore.j2',
filePath=Path(
project.projectDirectory,
'.gitignore'))
readmeContent = project.render_simple(templateName='README.md.j2',
filePath=Path(
project.projectDirectory,
'README.md'))
licenseContent = project.render_simple(templateName='LICENSE.j2',
filePath=Path(
project.projectDirectory,
'LICENSE'))
pylintrcContent = project.render_simple(templateName='pylintrc.j2',
filePath=Path(
project.projectDirectory,
'pylintrc'))
allContent = (makeVenvScript, runTestScript, runPylintScript,
requirementsContent, gitignoreContent, readmeContent,
licenseContent, pylintrcContent)
return project.cli_final_message(creationType='Project',
name=projectName,
content=allContent)