def install_all(config, path, no_dependencies, save_json, skip):
"""
Installing all dependencies for the ufo camera project into PATH
PATH has to be the path to an existing folder. The command installs all the required repositories into this folder.
Apart from the repositories, the command also installs the required system packages for the operation of the
UFO camera. For this it relies on the package installation method and operation system defined within the
ufotest config file.
"""
# The path sting which is passed into this function could also be a relative path such as ".." which stands for
# "the parent folder relative to the current one from which I am executing this". All functions expect an
# absolute path and "realpath" converts these relative expressions into absolute paths
path = os.path.realpath(path)
operating_system = CONFIG['install']['os']
ctitle('INSTALLING NECESSARY REQUIREMENTS')
cparams({
'operating system':
operating_system,
'package install command':
CONFIG['install'][operating_system]['package_install'],
'camera dimensions:':
f'{CONFIG.get_sensor_width()} x {CONFIG.get_sensor_height()}'
})
results = {}
# We will only actually execute the installation procedures if the skip flag is not set
if not skip:
if no_dependencies:
cprint('Skipping system packages...')
else:
install_dependencies(verbose=config.verbose())
for dependency_name, install_function in DEPENDENCY_INSTALL_FUNCTIONS.items(
):
csubtitle(f'Installing "{dependency_name}"')
result = install_function(path)
results[dependency_name] = result
else:
results['mock'] = mock_install_repository(path)
# Creating the JSON file if the flag was set.
if save_json:
try:
json_path = os.path.join(os.getcwd(), 'install.json')
with open(json_path, mode='w+') as file:
json.dump(results, file, indent=4, sort_keys=True)
except Exception as e:
cerror(f'Could not save "install.json" because: {str(e)}')
sys.exit(1)
sys.exit(0)
def run(self, test_only: bool = False) -> None:
"""Actually performs the build process.
This build process involves the following individual steps:
- The remote git repository is cloned to the local system and checked out to the desired branch/commit.
- The actual bit file is extracted from this repo.
- This bit file is being used to flash the new configuration to the fpga board.
- Finally the desired test suite is being run with this new hardware version.
:param test_only: A boolean flag, which can be used to skip the clone and flash steps of the build process.
If this flag is set, only the test step of the build process will be executed. This is mainly intended
for testing and mock executions. Defaults to False.
"""
try:
if not test_only:
# ~ CLONING THE REMOTE REPO
self.clone()
# ~ COPY THE BITFILE INTO THE REPORT FOLDER
self.copy_bitfile()
# ~ COPY REPOSITORY INTO REPORT FOLDER
# copy_repository copies the whole folder structure of the source repo which has just been cloned into
# the build folder. In the long run this could cause issues if the source repo is very big... But it is
# necessary to keep track of the script build versions
self.copy_repository()
# ~ FLASH THE NEW FIRMWARE TO THE HARDWARE
self.flash()
else:
# TODO: make this better
self.context.bitfile_path = ''
except BuildError:
exception_type, exception_value, exception_traceback = sys.exc_info(
)
cerror(f'{exception_type.__name__}: {exception_value}')
# Reloading the script manager seems important at this point because we obviously want the new
# scripts from this very build to be used later on during the test suite
self.config.sm.load_scripts()
# Something else that is important at this point is to run the setup routine for the camera again
# since it now has a new configuration flashed
self.test_runner.camera.set_up()
self.test()
self.context.complete()
def script_details(config, name):
"""
Shows the full details of the scripts identified by it's string NAME.
This command prints the full details for the given script, including its source code.
"""
# ~ Checking for eventual problems with the script
# First we need to check if a script with the given name even exists
if name in config.sm.scripts:
script = config.sm.scripts[name]
elif name in config.sm.fallback_scripts:
script = config.sm.fallback_scripts[name]
else:
cerror(
f'A script identified by "{name}" is nor registered with ufotest!')
sys.exit(1)
# Then we need to check if the file even exists / the content can be read, since we also want to display the
# content of it
try:
with open(script.path, mode='r+') as file:
content = file.read()
except:
cerror(
f'The file {script.path} does not exists and/or is not readable!')
sys.exit(1)
# ~ Displaying the results to the user in case there are no problems
ctitle(name)
cparams({
'type': script.data['class'],
'path': script.path,
'author': script.author
})
cprint('DESCRIPTION:\n' + script.description)
print()
cprint('CONTENT:\n' + content)
def invoke_script(config, name, args, fallback):
"""
Invokes the script identified by it's string NAME.
On default this command will attempt to invoke the most recent version of the script, which is subject to version
control and came with the most recent CI build. To use the stable fallback version which comes with the
ufotest software itself use the --fallback flag
"""
ctitle('Invoking script')
cparams({
'script name': name,
'additional args': args,
'use fallback?': fallback,
})
try:
result = config.sm.invoke(name, args, use_fallback=fallback)
if result['exit_code'] == 0:
cresult(f'script "{name}" exits with code 0')
cprint('STDOUT:\n' + result['stdout'])
elif result['exit_code'] != 0:
cerror(f'script "{name}" exits with code 1')
cerror('STDERR:\n' + result['stderr'])
cprint('STDOUT:\n' + result['stdout'])
sys.exit(1)
except KeyError:
cerror(f'A script with name "{name}" is not registered with ufotest!')
sys.exit(1)
sys.exit(0)
def execute(self) -> AbstractTestResult:
start_datetime = datetime.datetime.now()
try:
test_result = self.run()
except Exception as e:
exception_type, exception_value, exception_traceback = sys.exc_info(
)
test_result = MessageTestResult(
1, 'Test Execution failed with error "{}"'.format(str(e)))
# 09.09.2021: This is important for debugging!
# At least if verbose is enabled we also print the error with the traceback to the console. Because
# more often than not an error within a test turned out to be a bug in the test code rather than the
# the hardware actually not working
if self.config.verbose():
cerror(f'{exception_type.__name__}: {exception_value}')
traceback.print_stack(file=sys.stdout)
end_datetime = datetime.datetime.now()
test_result.start_datetime = start_datetime
test_result.end_start_time = end_datetime
return test_result
def build(self) -> None:
"""Actually performs the build process.
This build process involves the following individual steps:
- The remote git repository is cloned to the local system and checked out to the desired branch/commit.
- The actual bit file is extracted from this repo.
- This bit file is being used to flash the new configuration to the fpga board.
- Finally the desired test suite is being run with this new hardware version.
:deprecated:
:return: A BuildReport object which contains all the informations and metrics about the build process which are
important for the user.
"""
try:
click.secho('| | BUILDING FROM SOURCE | |', bold=True)
# -- CLONING THE REPO
self.clone()
# -- FLASHING THE BIT FILE
self.copy_bitfile()
self.flash()
# -- RUNNING THE TEST SUITE
self.test()
# -- SETTING THE END TIME
self.context.end_datetime = datetime.datetime.now()
except BuildError:
exception_type, exception_value, exception_traceback = sys.exc_info(
)
cerror(f'{exception_type.__name__}: {exception_value}')
except ZeroDivisionError as e:
pass
def build(config, suite, skip) -> None:
"""
Start a new CI build process using the test suite SUITE.
A build process first clones the target repository, which was specified within the "ci" section of the config file.
Specifically, it checks out the branch, which is also defined in the config, and uses the most recent commit to
that branch. After the repository has been cloned, the bit file within is used to flash a new configuration onto
the FPGA hardware. Finally, the given test suite is executed and the results are being saved to the archive.
"""
# ~ PRINT CONFIGURATION
ctitle('BUILDING FROM REMOTE REPOSITORY')
cparams({
'repository url': CONFIG.get_ci_repository_url(),
'repository branch': CONFIG.get_ci_branch(),
'bitfile relative path': CONFIG.get_ci_bitfile_path(),
'test suite': suite
})
# ~ RUNNING THE BUILD PROCESS
try:
with build_context_from_config(CONFIG) as build_context:
# The "build_context_from_config" function builds the context object entirely on the basis of the
# configuration file, which means, that it also uses the default test suite defined there. Since the build
# function is meant to be able to pass the test suite as a parameter, we will have to overwrite this piece
# of config manually.
build_context.test_suite = suite
build_runner = BuildRunner(build_context)
build_runner.run(test_only=skip)
build_report = BuildReport(build_context)
build_report.save(build_context.folder_path)
sys.exit(0)
except BuildError as e:
cerror(
f'An error has occurred during the build process...\n {str(e)}')
sys.exit(1)
def install(config, dependency, path, save_json, skip):
"""
Installs a given DEPENDENCY into the folder provided as PATH.
This command is used to install individual dependencies for running the ufo camera system on the local machine.
"""
# The path sting which is passed into this function could also be a relative path such as ".." which stands for
# "the parent folder relative to the current one from which I am executing this". All functions expect an
# absolute path and "realpath" converts these relative expressions into absolute paths
path = os.path.realpath(path)
# The "skip" flag is mainly for testing the CLI. It is supposed to prevent the actual functionality of
# the command to be executed so the test case does not take so long. So in this case we do not actually
# run the installation function, but still have to create a mock results dict
if skip:
result = mock_install_repository(path)
else:
# Each install function takes a single argument which is the folder path of the folder into which the
# dependency (repo) is to be installed. They return a dict which describes the outcome of the installation
# procedure with the following three fields: "success", "path", "git"
install_function = DEPENDENCY_INSTALL_FUNCTIONS[dependency]
result = install_function(path)
# If the the "--save-json" flag was provided with the command, we'll also save the result of the installation
# process as a JSON file to the current working directory.
if save_json:
try:
json_path = os.path.join(os.getcwd(), 'install.json')
with open(json_path, mode='w+') as file:
json.dump(result, file, indent=4, sort_keys=True)
except Exception as e:
cerror(f'Could not save "install.json" because: {str(e)}')
sys.exit(1)
sys.exit(0)
def init(force, update):
"""
Initializes the installation folder for this application.
This folder will be located at "$HOME/.ufotest". The init furthermore includes the creation of the necessary
sub folder structures within the installation folder as well as the creation of the default config file and all the
static assets for the web interface.
The --force flag can be used to *overwrite* an existing installation. It will delete the existing installation,
if one exists, and then create an entirely new installation.
The --update flag can be used to perform an update rather than a complete installation. This can be invoked with an
installation present and will not overwrite custom configuration files. The only thing which will be replaced
during such an update will be the static files. The new static files from the current system package version of
ufotest will be copied to the active installation folder.
"""
installation_path = get_path()
ctitle('INITIALIZING UFOTEST INSTALLATION')
cparams({'installation path': installation_path})
if update:
if check_path(installation_path, is_dir=True):
update_install(installation_path)
cresult('Updated ufotest installation')
sys.exit(0)
else:
cerror(
'Cannot perform update without already existing installation!')
sys.exit(1)
if check_path(installation_path, is_dir=True):
if force:
shutil.rmtree(get_path())
cprint('Deleted old installation folder')
else:
cerror('An installation folder already exists at the given path!')
cerror(
'Please use the --force flag if you wish to forcefully replace the existing installation'
)
sys.exit(1)
else:
cprint(' Installation folder does not yet exist')
# This method actually executes the necessary steps.
init_install()
cresult(
'UfoTest successfully initialized, use the --help option for further commands'
)
sys.exit(0)
def run(self):
try:
while self.running:
time.sleep(1)
if not BuildQueue.is_empty() and not BuildLock.is_locked():
build_request = BuildQueue.pop()
try:
# ~ RUN THE BUILD PROCESS
build_report = self.run_build(
build_request) # raises: BuildError
# ~ SEND REPORT MAILS
# After the build process has terminated we want to inform the relevant people of the outcome.
# This is mainly Two people: The maintainer of the repository is getting an email and pusher
self.send_report_mails(build_request, build_report)
except BuildError as error:
cerror(
'The build process was terminated due to a build error!'
)
cerror(str(error))
except smtplib.SMTPAuthenticationError as error:
cerror(
'The email credentials provided in the config file were not accepted by the server!'
)
cerror(str(error))
except OSError as error:
cerror(
'Report mails could not be sent because there is no network connection'
)
cerror(str(error))
except KeyboardInterrupt:
cprint('\n...Stopping BuildWorker')
def cli(ctx, version, verbose, conf, mock):
"""
UfoTest command line interface
The main way to use UfoTest's functionality is by invoking the appropriate command. Each command has it's own set
of arguments and options. To list these options and to show an explanation of the commands purpose, use the
--help option which is available for every command:
ufotest <subcommand> --help
"""
# If the version option of the base command is invoked, this means that we simply want to print the version of the
# project and then terminate execution
if version:
# "get_version" reads the version of the software from the corresponding file in the source folder.
version = get_version()
click.secho(version, bold=True)
sys.exit(0)
# We acquire an instance of the config object here in the base command and then simply pass it as a context to each
# respective sub command. Sure, the Config class is a singleton anyways and we could also invoke it in every
# sub command, but like this we can do something cool. We simply add a --verbose option to this base command and
# then save the value in the config, since we pass it along this also affects all sub commands and we dont need
# to implement the --verbose option for every individual sub command.
config = Config()
config['context']['verbose'] = verbose
ctx.obj = config
# This fixes an important bug: Previously when any command was executed, the program attempted to call the prepare
# method which would then in turn load the plugins. But that caused a problem when initially attempting to install
# ufotest. For an initial installation there is not config file yet, but "prepare" and the init of the script and
# plugin manager need certain config values! Thus ufotest could never be installed. Now we will treat the
# installation command "init" as a special case which does not need the use the plugin system.
if ctx.invoked_subcommand != 'init':
# "prepare" sets up the correct jinja environment and initializes the plugin manager and the
# script manager
config.prepare()
# Usually I wouldnt want to have to add the custom filters here but, sadly I have to due to pythons import
# system. I would have liked to do it in "prepare" but in that module i cannot import anything from util (where
# the actual method comes from) since util imports from that module...
config.template_environment.filters[
'format_byte_size'] = format_byte_size
# This hook can be used to execute generic functionality before any command specific code is executed
config.pm.do_action('pre_command',
config=config,
namespace=globals(),
context=ctx)
for overwrite_string in conf:
try:
config.apply_overwrite(overwrite_string)
except Exception:
if config.verbose():
cerror((
f'Could not apply config overwrite for value {overwrite_string}. Please check if the string is '
f'formatted correctly. The correct formatting would be "key.subkey=value". Also check if the '
f'variable even exists in the config file.'))
if mock:
# The "--mock" option for the main ufotest command implies that the mock camera class is to be used for the
# execution of all sub commands. The first thing we need to do for this is to set the context flag "mock" to
# True. This flag can be used by sub commands to check if the mock option has been invoked.
# We could also check for the class of the camera, but that would be unreliable because technically a plugin
# could change the mock camera class!
config['context']['mock'] = True
# This is what I meant: The exact camera class used as the mock is also subject to change by filter hook. On
# default it will be MockCamera, but a plugin could decide to extend this class by subclassing or replace it
# with something completely different
mock_camera_class = config.pm.apply_filter('mock_camera_class',
MockCamera)
# Then of course we modify the main "camera_class" filter to actually inject this mock camera class to be used
# in all following scenarios.
config.pm.register_filter('camera_class',
lambda value: mock_camera_class, 1)
def flash(config, file: str, type: str) -> None:
"""
Uses the given FILE to flash a new firmware configuration to the internal memory of the FPGA board, where FILE is
the string absolute path of the file which contains a valid specification of the fpga configuration.
The --type option can be used to select which kind of configuration file to be flashed. currently only BIT files
are supported.
"""
# -- ECHO CONFIGURATION
ctitle('FLASHING BITFILE TO FPGA')
click.secho('--| bitfile path: {}\n'.format(file))
# ~ SKIP FOR MOCK
# If the "--mock" option is active then we completely skip this command since presumably there is not actual
# camera connected to do the flashing.
# TODO: In the future we could add an action hook here to be able to inject come actual code for the case of
# mock camera + flash command.
if config['context']['mock']:
cresult('Skip flash when due to --mock option being enabled')
sys.exit(0)
# ~ CHECKING IF THE GIVEN FILE EVEN EXISTS
file_path = os.path.abspath(file)
file_exists = check_path(file_path, is_dir=False)
if not file_exists:
cerror('The given path for the bitfile does not exist')
sys.exit(1)
# ~ CHECKING IF THE FILE EVEN IS A BIT FILE
file_extension = file_path.split('.')[-1].lower()
is_bit_file = file_extension == 'bit'
if not is_bit_file:
cerror('The given path does not refer to file of the type ".bit"')
sys.exit(1)
# ~ CHECKING IF VIVADO IS INSTALLED
vivado_installed = check_vivado()
if not vivado_installed:
cerror(
'Vivado is not installed, please install Vivado to be able to flash the fpga!'
)
sys.exit(1)
if CONFIG.verbose():
cprint('Vivado installation found')
# ~ STARTING VIVADO SETTINGS
vivado_command = CONFIG['install']['vivado_settings']
run_command(vivado_command, cwd=os.getcwd())
if CONFIG.verbose():
cprint('Vivado setup completed')
# ~ FLASHING THE BIT FILE
flash_command = "{setup} ; {command} -nolog -nojournal -mode batch -source {tcl} -tclargs {file}".format(
setup=CONFIG['install']['vivado_settings'],
command=CONFIG['install']['vivado_command'],
tcl=CONFIG.sm.scripts['fpga_bitprog'].path,
file=file_path)
exit_code, _ = run_command(flash_command)
if not exit_code:
cresult('Flashed FPGA with: {}'.format(file_path))
sys.exit(0)
else:
cerror('There was an error during the flashing of the FPGA')
sys.exit(1)
def frame(config, output, display):
"""
Capture a single frame from the camera.
If this command is invoked without any additional options, the frame will be captured from the camera and then
saved to the location "/tmp/frame.png".
The output location for the image file can be overwritten by using the --output option to specify another path.
The --display flag can be used to additionally display the image to the user after the frame has been captured.
This feature requires a graphical interface to be available to the system. The frame will be opened in a seperate
matplotlib figure window.
"""
config.pm.do_action('pre_command_frame',
config=config,
namespace=globals(),
output=output,
display=display)
ctitle('CAPTURING FRAME')
cparams({
'output path':
output,
'display frame':
display,
'sensor_dimensions':
f'{CONFIG.get_sensor_width()} x {CONFIG.get_sensor_height()}'
})
# Setup all the important environment variables and stuff
setup_environment()
exit_code, _ = run_command('rm /tmp/frame*')
if not exit_code:
cprint('Removed previous frame buffer')
# ~ GET THE FRAME FROM THE CAMERA
# "get_frame" handles the whole communication process with the camera, it requests the frame, saves the raw data,
# decodes it into an image and then returns the string path of the final image file.
try:
camera_class = config.pm.apply_filter('camera_class', UfoCamera)
camera = camera_class(config)
frame = camera.get_frame()
except PciError as error:
cerror('PCI communication with the camera failed!')
cerror(f'PciError: {str(error)}')
sys.exit(1)
except FrameDecodingError as error:
cerror('Decoding of the frame failed!')
cerror(f'FrameDecodingError: {str(error)}')
sys.exit(1)
# ~ Saving the frame as a file
_, file_extension = output.split('.')
if file_extension == 'raw':
cerror('Currently the frame cannot be saved as .raw format!')
else:
from PIL import Image
image = Image.fromarray(frame)
image.save(output)
# ~ Displaying the image if the flag is set
if display:
# An interesting thing is that matplotlib is imported in-time here and not at the top of the file. This actually
# had to be changed due to a bug. When using ufotest in a headless environment such as a SSH terminal session
# it would crash immediately, because a headless session does not work with the graphical matplotlib. Since
# it really only is needed for this small section here, it makes more sense to just import it in-time.
import matplotlib.pyplot as plt
matplotlib.use('TkAgg')
plt.imshow(frame)
plt.show()
sys.exit(0)