async def process(self, name_filter=None) -> int:
await self.tentacles_setup_manager.create_missing_tentacles_arch()
self.reset_worker()
self.progress = 1
all_tentacles = util.load_tentacle_with_metadata(self.reference_tentacles_root)
self.available_tentacles = util.load_tentacle_with_metadata(self.tentacle_path)
self.register_error_on_missing_tentacles(all_tentacles, name_filter)
to_install_tentacles = [tentacle
for tentacle in all_tentacles
if self._should_tentacle_be_processed(tentacle, name_filter)]
self.total_steps = len(to_install_tentacles)
self.register_to_process_tentacles_modules(to_install_tentacles)
await asyncio.gather(*[self._install_tentacle(tentacle) for tentacle in to_install_tentacles])
# install profiles if any
self._import_profiles_if_any()
# now that profiles are imported, update tentacles setup config
# and include missing tentacles in profile tentacles config
await self.tentacles_setup_manager.refresh_user_tentacles_setup_config_file(
self.tentacles_setup_config_to_update,
self.tentacles_path_or_url,
force_update_registered_tentacles=True,
newly_installed_tentacles=to_install_tentacles)
self.tentacles_setup_manager.cleanup_temp_dirs()
self.log_summary()
return len(self.errors)
async def refresh_user_tentacles_setup_config_file(
self,
tentacles_setup_config_to_update=None,
update_location=None,
force_update_registered_tentacles=False,
newly_installed_tentacles=None,
uninstalled_tentacles=None):
available_tentacle = util.load_tentacle_with_metadata(
self.tentacle_setup_root_path)
if not tentacles_setup_config_to_update:
reference_tentacle_setup_config = configuration.TentaclesSetupConfiguration(
bot_installation_path=self.bot_installation_path)
# Do not read activation config to force default values generation and avoid side effects on
# profiles activations
reference_tentacle_setup_config.read_config(
self.tentacle_setup_root_path, False)
else:
reference_tentacle_setup_config = tentacles_setup_config_to_update
await reference_tentacle_setup_config.fill_tentacle_config(
available_tentacle,
self.default_tentacle_config,
update_location=update_location,
force_update_registered_tentacles=force_update_registered_tentacles,
newly_installed_tentacles=newly_installed_tentacles,
uninstalled_tentacles=uninstalled_tentacles)
reference_tentacle_setup_config.save_config()
reference_tentacle_setup_config.refresh_profile_tentacles_config(
available_tentacle,
newly_installed_tentacles=newly_installed_tentacles,
uninstalled_tentacles=uninstalled_tentacles)
async def test_tentacle_bundle_exporter_with_specified_output_dir(
install_tentacles):
specified_output_dir = "out/dir/test"
# Export each tentacle in a bundle in a specified output dir
for tentacle in util.load_tentacle_with_metadata(constants.TENTACLES_PATH):
tentacle_package = models.TentaclePackage()
await exporters.TentacleExporter(
artifact=tentacle,
should_zip=True,
output_dir=specified_output_dir,
tentacles_folder=constants.TENTACLES_PATH,
use_package_as_file_name=True).export()
tentacle_package.add_artifact(tentacle)
await exporters.TentacleBundleExporter(
artifact=tentacle_package,
output_dir=specified_output_dir,
tentacles_folder=constants.TENTACLES_PATH,
use_package_as_file_name=True).export()
# Check if each tentacle bundle has been generated in the specified directory
output_files = os.listdir(specified_output_dir)
assert len(output_files) == 20
assert "daily_trading_mode.zip" in output_files
assert "[email protected]" in output_files
shutil.rmtree(specified_output_dir)
async def process(self, name_filter=None) -> int:
self.reset_worker()
if self.confirm_action(
"Remove all installed tentacles ?" if name_filter is None else
"Remove {', '.join(name_filter)} tentacles ?"):
to_uninstall_tentacles = None
if name_filter is None:
self.tentacles_setup_manager.delete_tentacles_arch(
force=True,
with_user_config=False,
bot_installation_path=self.bot_installation_path)
else:
self.progress = 1
self.available_tentacles = util.load_tentacle_with_metadata(
self.tentacle_path)
self.register_error_on_missing_tentacles(
self.available_tentacles, name_filter)
to_uninstall_tentacles = [
tentacle for tentacle in self.available_tentacles
if tentacle.name in name_filter
]
await asyncio.gather(*[
self._uninstall_tentacle(tentacle)
for tentacle in to_uninstall_tentacles
])
await self.tentacles_setup_manager.create_missing_tentacles_arch()
await self.tentacles_setup_manager.refresh_user_tentacles_setup_config_file(
self.tentacles_setup_config_to_update,
self.tentacles_path_or_url,
True,
uninstalled_tentacles=to_uninstall_tentacles)
self.log_summary()
return len(self.errors)
async def refresh_user_tentacles_setup_config_file(
self,
tentacles_setup_config_to_update=None,
update_location=None,
force_update_registered_tentacles=False,
newly_installed_tentacles=None,
uninstalled_tentacles=None):
available_tentacle = util.load_tentacle_with_metadata(
self.tentacle_setup_root_path)
if not tentacles_setup_config_to_update:
tentacle_setup_config = configuration.TentaclesSetupConfiguration(
bot_installation_path=self.bot_installation_path)
tentacle_setup_config.read_config(self.tentacle_setup_root_path)
else:
tentacle_setup_config = tentacles_setup_config_to_update
await tentacle_setup_config.fill_tentacle_config(
available_tentacle,
self.default_tentacle_config,
update_location=update_location,
force_update_registered_tentacles=force_update_registered_tentacles,
newly_installed_tentacles=newly_installed_tentacles,
uninstalled_tentacles=uninstalled_tentacles)
tentacle_setup_config.save_config()
tentacle_setup_config.refresh_profile_tentacles_config(
available_tentacle,
newly_installed_tentacles=newly_installed_tentacles,
uninstalled_tentacles=uninstalled_tentacles)
async def test_fill_tentacle_config():
async with aiohttp.ClientSession() as session:
await api.install_all_tentacles(_tentacles_local_path(), aiohttp_session=session)
setup_config = configuration.TentaclesSetupConfiguration()
available_tentacle = util.load_tentacle_with_metadata(constants.TENTACLES_PATH)
with mock.patch.object(setup_config, "_get_installation_context_bot_version", mock.Mock()) as bot_version_mock:
bot_version_mock.return_value = "1.0.5"
await setup_config.fill_tentacle_config(available_tentacle, constants.TENTACLE_CONFIG_FILE_NAME)
assert setup_config.installation_context == {
constants.TENTACLE_INSTALLATION_CONTEXT_OCTOBOT_VERSION: "1.0.5"
}
setup_config = configuration.TentaclesSetupConfiguration()
await setup_config.fill_tentacle_config(available_tentacle, constants.TENTACLE_CONFIG_FILE_NAME)
assert setup_config.installation_context == {
constants.TENTACLE_INSTALLATION_CONTEXT_OCTOBOT_VERSION:
constants.TENTACLE_INSTALLATION_CONTEXT_OCTOBOT_VERSION_UNKNOWN
}
assert not api.are_tentacles_up_to_date(setup_config,
constants.TENTACLE_INSTALLATION_CONTEXT_OCTOBOT_VERSION_UNKNOWN)
assert not api.are_tentacles_up_to_date(setup_config, '1.0.0')
setup_config.installation_context[constants.TENTACLE_INSTALLATION_CONTEXT_OCTOBOT_VERSION] = '2.0.0'
assert not api.are_tentacles_up_to_date(setup_config, '2.1.0')
assert api.are_tentacles_up_to_date(setup_config, '2.0.0')
assert api.are_tentacles_up_to_date(setup_config, '2.0.0b1')
_cleanup()
def reload_tentacle_by_tentacle_class(tentacles_path=constants.TENTACLES_PATH):
global _tentacle_by_tentacle_class
loaded_tentacles = util.load_tentacle_with_metadata(tentacles_path)
_tentacle_by_tentacle_class = {
klass: tentacle
for tentacle in loaded_tentacles
for klass in tentacle.tentacle_class_names
}
async def create_all_tentacles_bundle(
output_dir: str = constants.DEFAULT_EXPORT_DIR,
tentacles_folder: str = constants.TENTACLES_PATH,
exported_tentacles_package: str = None,
in_zip: bool = True,
with_dev_mode: bool = False,
cythonize: bool = False,
should_remove_artifacts_after_use: bool = False,
use_package_as_file_name: bool = False,
upload_url: str = None,
should_zip_bundle: bool = False) -> int:
logger = logging.get_logger("TentacleManagerApi")
error_count: int = 0
tentacle_bundle_exported_list = []
tentacles: list = util.load_tentacle_with_metadata(tentacles_folder)
tentacles_white_list = util.filter_tentacles_by_dev_mode_and_package(
tentacles=tentacles,
with_dev_mode=with_dev_mode,
package_filter=exported_tentacles_package)
for tentacle in tentacles_white_list:
try:
tentacle_package = models.TentaclePackage()
tentacle_exporter = exporters.TentacleExporter(
artifact=tentacle,
output_dir=output_dir,
tentacles_folder=tentacles_folder,
should_zip=in_zip,
with_dev_mode=with_dev_mode,
should_cythonize=cythonize,
use_package_as_file_name=use_package_as_file_name)
await tentacle_exporter.export()
tentacle_package.add_artifact(tentacle)
tentacle_bundle_exporter = exporters.TentacleBundleExporter(
artifact=tentacle_package,
tentacles_folder=tentacles_folder,
output_dir=output_dir,
should_zip=should_zip_bundle,
should_remove_artifacts_after_use=
should_remove_artifacts_after_use,
use_package_as_file_name=use_package_as_file_name)
await tentacle_bundle_exporter.export()
tentacle_bundle_exported_list.append(tentacle_bundle_exporter)
except Exception as e:
logger.error(
f"Error when exporting tentacle {tentacle.name} : {str(e)}")
error_count += 1
if upload_url is not None:
await asyncio.gather(*[
_upload_exported_tentacle_bundle(upload_url,
exported_tentacle_bundle)
for exported_tentacle_bundle in tentacle_bundle_exported_list
])
return error_count
async def _fetch_tentacles_for_requirement(self, repo):
await util.fetch_and_extract_tentacles(
repo,
self.tentacles_path_or_url or constants.DEFAULT_TENTACLES_URL,
self.aiohttp_session,
merge_dirs=True)
requirements_tentacles_path = path.join(
repo, constants.TENTACLES_ARCHIVE_ROOT)
self.fetched_for_requirements_tentacles = util.load_tentacle_with_metadata(
requirements_tentacles_path)
self.fetched_for_requirements_tentacles_versions = \
self._get_version_by_tentacle(self.fetched_for_requirements_tentacles)
async def process(self, name_filter=None) -> int:
# force reset of all init files
await self.tentacles_setup_manager.remove_tentacle_arch_init_files()
# create missing files and folders
await self.tentacles_setup_manager.create_missing_tentacles_arch()
self.reset_worker()
self.progress = 1
self.available_tentacles = util.load_tentacle_with_metadata(
self.tentacle_path)
self.total_steps = len(self.available_tentacles)
await asyncio.gather(*[
self._repair_tentacle(tentacle)
for tentacle in self.available_tentacles
])
await self.tentacles_setup_manager.refresh_user_tentacles_setup_config_file(
force_update_registered_tentacles=True)
self.log_summary()
return len(self.errors)
async def test_tentacle_bundle_exporter_for_an_unique_bundle_containing_all_tentacles(
install_tentacles):
# Export all tentacles and generate a bundle containing all
tentacle_package = models.TentaclePackage()
for tentacle in util.load_tentacle_with_metadata(constants.TENTACLES_PATH):
await exporters.TentacleExporter(
artifact=tentacle,
should_zip=True,
tentacles_folder=constants.TENTACLES_PATH,
use_package_as_file_name=True).export()
tentacle_package.add_artifact(tentacle)
await exporters.TentacleBundleExporter(
artifact=tentacle_package,
tentacles_folder=constants.TENTACLES_PATH,
should_remove_artifacts_after_use=True,
use_package_as_file_name=True).export()
# Check if the final bundle contains all exported tentacles and a metadata file
# check files count
output_files = os.listdir(constants.DEFAULT_EXPORT_DIR)
assert len(output_files) == 1
exported_bundle_path = os.path.join(constants.DEFAULT_EXPORT_DIR,
output_files[0])
output_files = os.listdir(exported_bundle_path)
assert len(output_files) == 11
assert "daily_trading_mode.zip" in output_files
assert "[email protected]_package" not in output_files
assert "[email protected]_package" not in output_files
assert "mixed_strategies_evaluator.zip" in output_files
assert "mixed_strategies_evaluator" not in output_files
assert constants.ARTIFACT_METADATA_FILE in output_files
# test multiple tentacle bundle metadata
with open(
os.path.join(exported_bundle_path,
constants.ARTIFACT_METADATA_FILE)) as metadata_file:
metadata_content = yaml.safe_load(metadata_file.read())
assert metadata_content[
constants.ARTIFACT_METADATA_ARTIFACT_TYPE] == "tentacle_package"
assert len(
metadata_content[constants.ARTIFACT_METADATA_TENTACLES]) == 10
assert "[email protected]" in metadata_content[
constants.ARTIFACT_METADATA_TENTACLES]
async def prepare_export(self):
if not self.with_dev_mode or self.exported_tentacles_package is not None:
self.tentacles = util.load_tentacle_with_metadata(self.tentacles_folder)
# remove dev-mode or non exported package tentacles if necessary
self.tentacles_white_list = util.filter_tentacles_by_dev_mode_and_package(
tentacles=self.tentacles,
with_dev_mode=self.with_dev_mode,
package_filter=self.exported_tentacles_package
)
# filter tentacles
self.tentacles_filter = util.TentacleFilter(self.tentacles, self.tentacles_white_list)
# set white list tentacles as TentaclePackage.artifacts
self.artifact.artifacts = self.tentacles_white_list
if self.should_zip:
self.copy_directory_content_to_temporary_dir(self.tentacles_folder,
ignore=self.tentacles_filter.should_ignore)
else:
self.copy_directory_content_to_working_dir(self.tentacles_folder,
ignore=self.tentacles_filter.should_ignore)
async def test_tentacle_bundle_exporter_for_each_tentacle(install_tentacles):
# Export each tentacle in a bundle
for tentacle in util.load_tentacle_with_metadata(constants.TENTACLES_PATH):
tentacle_package = models.TentaclePackage()
await exporters.TentacleExporter(
artifact=tentacle,
should_zip=True,
tentacles_folder=constants.TENTACLES_PATH,
use_package_as_file_name=True).export()
tentacle_package.add_artifact(tentacle)
await exporters.TentacleBundleExporter(
artifact=tentacle_package,
tentacles_folder=constants.TENTACLES_PATH,
use_package_as_file_name=True).export()
# Check if each tentacle bundle has been generated
# check files count
output_files = os.listdir(constants.DEFAULT_EXPORT_DIR)
assert len(output_files) == 20
assert "daily_trading_mode.zip" in output_files
assert "[email protected]" in output_files
assert "[email protected]" in output_files
assert "mixed_strategies_evaluator.zip" in output_files
assert "mixed_strategies_evaluator" not in output_files
async def process(self, name_filter=None) -> int:
self.available_tentacles = util.load_tentacle_with_metadata(
self.tentacle_path)
return await super().process(name_filter)