Ejemplo n.º 1
0
def prepare_repository(project):
    debug.message("Checkout commit %s" % project["commit"], indent=1)

    repo = Git(repository.get_path(project), repository.get_origin_url(project))

    # Check out the correct commit and create a reference to it (deployed)
    repo.checkout_commit(project["commit"])
Ejemplo n.º 2
0
def watch(build_queue):
    while True:
        debug.set_prefix("repository_watcher")
        debug.message("Retrieving projects")
        try:
            projects = Api.get_projects()

            for project in projects:
                debug.message("Check repository status for project %s" % project["Name"])
                repository = Git(get_path(project), get_origin_url(project))
                commit_count = handle_incoming_commits(repository.check_for_new_commits_on_origin(), project,
                                                       repository, build_queue)
                if commit_count > 0:
                    repository.merge_origin()

                # Add <initial_nr_commits> commits if this is a new repository
                if project["Commits"] is None or len(project["Commits"]) == 0:
                    handle_incoming_commits(
                        repository.get_commits(registry.config["repositories"]["initial_nr_commits"]), project,
                        repository, build_queue)

        except ValueError, e:
            debug.exception("Error retrieving projects", e)
        except GitError, e:
            debug.exception("Error with Git repository", e)
Ejemplo n.º 3
0
    def _get_next_filename(self, file_type):

        """This private method keeps track of and returns the file name of the next capture or playback file that can be written to.

        Args: file_type(integer), eg: either CAPTURE_FILE or PLAYBACK_FILE.
        Returns: filename(string)
        Raises: EndOfReplay
        """
        debug.message(debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO, "%s: %s" % (SboDataSourceReplay.__name__, "_get_next_filename"))

        # Capture and Playback files are numbered sequentially starting at One.
        self.replay_file_number[file_type] = self.replay_file_number[file_type] + 1

        # Once the file number for a Capture file has reached the maximum allowed, set a flag.
        # With the flag is set, this method will return the last Capture filename to be used.
        if file_type == CAPTURE_FILE and self.replay_file_number[file_type] == FILE_NUMBER_MAX:
            self.capture_limit = True

        # Once the file number for a Playback file has gone over the maximum allowed, raise an exception.
        if file_type == PLAYBACK_FILE and self.replay_file_number[file_type] > FILE_NUMBER_MAX:
            raise SboDataSourceReplay.EndOfReplay("Can not playback more than %s files." % FILE_NUMBER_MAX)

        formatted_capture_file_number = str(self.replay_file_number[file_type]).zfill(FILE_NUMBER_DIGITS)

        # The capture filename template holds the filename of the capture data files with a dummy file number.
        # This statement replaces the 'nnnnn' in the dummy file with the real number, calculated above.
        filename = re.sub(r'n{5}', formatted_capture_file_number, CAPTURE_FILENAME_TEMPLATE)

        return filename
Ejemplo n.º 4
0
    def reinitialise_playback(self):
        """This public method is called to re-initialise certain attributes ready for the playback() method to be called again.

        Raises:
            CallOrderError: Raised if playback has not previously been initialised.

        This simple method has no arguments and no returns.
        """
        debug.message(
            debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO,
            "%s: %s" % (SboDataSourceReplay.__name__, "reinitialise_playback"))

        if not self.playback_previously_initialised_flag:
            raise SboDataSourceReplay.CallOrderError(
                "The reinitalise_playback() method was called before the initialise_playback() method."
            )

        # Re-initialise the playback attributes.
        self.last_file_played_back = [
            INITIAL_LAST_LIVE_FILE_PLAYED_BACK,
            INITIAL_LAST_NON_LIVE_FILE_PLAYED_BACK
        ]
        self.replay_file_number[PLAYBACK_FILE] = INITIAL_PLAYBACK_FILE_NUMBER
        self.load_next_file_for_playback = True
        self.file_contents = [INITIAL_DATA_IDENTIFIER, INITIAL_RAW_DATA]

        self.playback_initialised_flag = True
Ejemplo n.º 5
0
    def __init__(self):

        debug.message(debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO,
                      "%s: %s" % (SboDataSourceReplay.__name__, "__init__"))

        # The base directory contains the script that was used to invoke the Python interpreter.
        self.base_directory = os.sys.path[0]

        self.capture_limit = False
        self.first_capture = True

        self.replay_file_number = [
            INITIAL_CAPTURE_FILE_NUMBER, INITIAL_PLAYBACK_FILE_NUMBER
        ]
        self.replay_folder_path = None
        self.replay_mode = None
        self.minimum_request_period_live = None
        self.minimum_request_period_non_live = None

        self.playback_previously_initialised_flag = False
        self.playback_initialised_flag = False
        self.last_file_played_back = [
            INITIAL_LAST_LIVE_FILE_PLAYED_BACK,
            INITIAL_LAST_NON_LIVE_FILE_PLAYED_BACK
        ]
        self.load_next_file_for_playback = True
        self.file_contents = [INITIAL_DATA_IDENTIFIER, INITIAL_RAW_DATA]
        self.last_datestamp = None
Ejemplo n.º 6
0
    def load(self):
        debug.message("Reading config")
        try:
            with open(self.config_file, "r") as file:
                self.data = yaml.load(file.read())

        except Exception, e:
            debug.exception("Exception while reading config", e)
Ejemplo n.º 7
0
def shutdown(signal, frame):
    debug.message("Caught signal %s, shutting down." % str(signal))

    if registry.pid_file is not None:
        try:
            os.unlink(registry.pid_file)
        except OSError:
            pass

    sys.exit(0)
Ejemplo n.º 8
0
    def reset_playback(self, replay_folder_path, replay_mode, minimum_request_period):

        """This method allows playback to be reset during testing.
           Note: For testing purposes only, the parameters passed to the initalise_playback() method will keep changing.
           This method calls initalise_playback() followed by reinitalise_playback() to ensure these changes are applied.
        """

        debug.message(debug_flags.TEST_SBO_DATA_SOURCE_REPLAY_EXTRA, debug.TESTUNIT, "initialising playback...")

        self.sbo_data_source_replay.initalise_playback(replay_folder_path, replay_mode, minimum_request_period)
        self.sbo_data_source_replay.reinitialise_playback()
Ejemplo n.º 9
0
def start(callback, foreground=False):
    if foreground:
        debug.message("Running in foreground mode")
        run(callback, foreground)

    pid = os.fork()
    if pid:
        debug.message("Running in daemon mode, pid: %s" % str(pid))
        write_pid(pid)
        sys.exit(0)
    else:
        run(callback)
Ejemplo n.º 10
0
    def get_last_datestamp(self):

        """This public method returns the date stamp of the last raw data replay file to be loaded for playback.

        Returns:
            last_datestamp: A string representing a date stamp of the the last valid raw data file to be opened.

        This simple method has no arguments and raises no errors.
        """
        debug.message(debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO, "%s: %s (%s)" % (SboDataSourceReplay.__name__, "get_last_datestamp", self.last_datestamp))

        return self.last_datestamp
Ejemplo n.º 11
0
    def playback_previously_initalised(self):

        """This public method returns a flag which indicates weather playback has been successfully initialised at least once.

        Returns:
            playback_previously_initialised_flag: A boolean representation of the former initialisation of playback.

        This simple method has no arguments and raises no errors.
        """
        debug.message(debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO, "%s: %s (%s)" % (SboDataSourceReplay.__name__, "playback_previously_initalised", self.playback_previously_initialised_flag))

        return self.playback_previously_initialised_flag
Ejemplo n.º 12
0
    def playback_initialised(self):

        """This public method returns the playback initialised flag which is set only when playback is initialised.

        Returns:
            playback_initialised_flag: A boolean representation of the initialised state of playback.

        This simple method has no arguments and raises no errors.
        """
        debug.message(debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO, "%s: %s (%s)" % (SboDataSourceReplay.__name__, "playback_initialised", self.playback_initialised_flag))

        return self.playback_initialised_flag
Ejemplo n.º 13
0
    def test_01_object_creation(self):

        """Test the class initialiser method."""

        debug.message(debug_flags.TEST_SBO_DATA_SOURCE_REPLAY, debug.TESTUNIT, "STARTING %s:" % "test_01_object_creation")

        # A: Test that the instantiation of the class was successful.
        actual_object = self.sbo_data_source_replay
        expected_class = SboDataSourceReplay
        self.assertIsInstance(actual_object, expected_class, "[A] The tested object is not an instance of the SboDataSourceReplay class.")

        # B: Test that the base directory exists.
        actual_result = os.path.exists(self.sbo_data_source_replay.base_directory)
        self.assertTrue(actual_result, "[B] The base directory does not exist.")
Ejemplo n.º 14
0
    def get_last_datestamp(self):
        """This public method returns the date stamp of the last raw data replay file to be loaded for playback.

        Returns:
            last_datestamp: A string representing a date stamp of the the last valid raw data file to be opened.

        This simple method has no arguments and raises no errors.
        """
        debug.message(
            debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO,
            "%s: %s (%s)" % (SboDataSourceReplay.__name__,
                             "get_last_datestamp", self.last_datestamp))

        return self.last_datestamp
Ejemplo n.º 15
0
    def read_file(file_path):

        """This method reads the captured data from a replay file."""

        debug.message(debug_flags.TEST_SBO_DATA_SOURCE_REPLAY_EXTRA, debug.TESTUNIT, "reading file...")

        file_handle = open(file_path, "r")

        data_identifier = file_handle.readline().rstrip('\n')
        datestamp = file_handle.readline().rstrip('\n')
        raw_data = file_handle.readline()

        file_handle.close()

        return (data_identifier, datestamp, raw_data)
Ejemplo n.º 16
0
    def setUpClass(cls): # pylint: disable-msg=C0103

        """This method is executed once at the start of this Unit Test."""

        debug.message(debug_flags.TEST_SBO_DATA_SOURCE_REPLAY, debug.TESTUNIT, "STARTING %s..." % TestSboDataSourceReplay.__name__)

        # Create an instance of SboDataSourceReplay class.
        cls.sbo_data_source_replay = SboDataSourceReplay()

        # Use the constants defined in the sbo_data_source_replay.py module to determine the current capture directory.
        base_directory = os.sys.path[0]
        capture_folder = sbo_data_source_replay.CAPTURE_FOLDER
        capture_current_subfolder = sbo_data_source_replay.CAPTURE_CURRENT_SUBFOLDER

        cls.capture_current_path = os.path.join(base_directory, capture_folder, capture_current_subfolder)
Ejemplo n.º 17
0
    def playback_previously_initalised(self):
        """This public method returns a flag which indicates weather playback has been successfully initialised at least once.

        Returns:
            playback_previously_initialised_flag: A boolean representation of the former initialisation of playback.

        This simple method has no arguments and raises no errors.
        """
        debug.message(
            debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO,
            "%s: %s (%s)" %
            (SboDataSourceReplay.__name__, "playback_previously_initalised",
             self.playback_previously_initialised_flag))

        return self.playback_previously_initialised_flag
Ejemplo n.º 18
0
    def playback_initialised(self):
        """This public method returns the playback initialised flag which is set only when playback is initialised.

        Returns:
            playback_initialised_flag: A boolean representation of the initialised state of playback.

        This simple method has no arguments and raises no errors.
        """
        debug.message(
            debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO,
            "%s: %s (%s)" %
            (SboDataSourceReplay.__name__, "playback_initialised",
             self.playback_initialised_flag))

        return self.playback_initialised_flag
Ejemplo n.º 19
0
    def publish(self):
        projects = self.get_projects()

        for project in projects:
            debug.message("Publishing project %s" % project["name"], indent=3)
            build_directory = self.create_build_subdirectory(project)
            project_directory = os.path.join(project["directory"], registry.config["build"]["outdir"], "Release")

            project_type = self.get_project_type(project)
            if project_type == CSharpProjectTypes.Console or project_type == CSharpProjectTypes.Test:
                self.publish_project(project_directory, build_directory)
            elif project_type == CSharpProjectTypes.WebApp:
                self.publish_project(os.path.join(project_directory, "_PublishedWebsites", project["name"]),
                                     build_directory)
            else:
                raise Exception("Invalid project type or project type could not be inferred.")
Ejemplo n.º 20
0
    def test_03_initalise_playback(self):

        """Test the initialisation of playback."""

        debug.message(debug_flags.TEST_SBO_DATA_SOURCE_REPLAY, debug.TESTUNIT, "STARTING %s:" % "test_03_initalise_playback")

        # A: Test that playback is not available before initialisation.
        actual_result = self.sbo_data_source_replay.playback_initialised()
        expected_result = False
        self.assertEqual(actual_result, expected_result, "[A] The actual result doesn't match the expected result.")

        # B: Test that the playback() method raises an error when called before initialisation.
        self.assertRaises(SboDataSourceReplay.CallOrderError, self.sbo_data_source_replay.playback)

        # C: Test that the reinitialise_playback() method raises an error when called before initialisation.
        self.assertRaises(SboDataSourceReplay.CallOrderError, self.sbo_data_source_replay.reinitialise_playback)

        # Define valid parameters.
        replay_folder_path = 'test_data/playback_files/set_003'
        replay_mode = NORMAL_REPLAY_MODE
        minimum_request_period = [datetime.timedelta(seconds = MINIMUM_REQUEST_PERIOD_LIVE), datetime.timedelta(seconds = MINIMUM_REQUEST_PERIOD_NON_LIVE)]

        # D: Test that the method can cope with invalid inputs by raising an exception.
        for number, invalid_replay_folder_path in enumerate(self.test_types, start=1):

            # Test type 6 is a valid values of replay folder path.
            if number == 6:
                continue

            debug.message(debug_flags.TEST_SBO_DATA_SOURCE_REPLAY_EXTRA, debug.TESTUNIT, "Testing replay folder path with invalid type %s of %s." % (number, len(self.test_types)))
            self.assertRaises(SboDataSourceReplay.PlaybackInitialisationError, self.sbo_data_source_replay.initalise_playback, invalid_replay_folder_path, replay_mode, minimum_request_period)

        # Note: If an unexpected type is used for the replay mode parameter, it will be replaced with a default replay mode.
        # Skipping replay mode type test.

        # E: Test that the method can cope with invalid inputs by raising an exception.
        for number, invalid_minimum_request_period in enumerate(self.test_types, start=1):

            debug.message(debug_flags.TEST_SBO_DATA_SOURCE_REPLAY_EXTRA, debug.TESTUNIT, "Testing minimum request period with invalid type %s of %s." % (number, len(self.test_types)))
            self.assertRaises(SboDataSourceReplay.PlaybackInitialisationError, self.sbo_data_source_replay.initalise_playback, replay_folder_path, replay_mode, invalid_minimum_request_period)

        # F: Test that the method can cope with invalid inputs by raising an exception.
        for number, invalid_minimum_request_period_value in enumerate(self.test_types, start=1):

            # Initialise playback with invalid minimum request period values.
            invalid_minimum_request_period = [invalid_minimum_request_period_value, invalid_minimum_request_period_value]

            debug.message(debug_flags.TEST_SBO_DATA_SOURCE_REPLAY_EXTRA, debug.TESTUNIT, "Testing minimum request period value with invalid type %s of %s." % (number, len(self.test_types)))
            self.assertRaises(SboDataSourceReplay.PlaybackInitialisationError, self.sbo_data_source_replay.initalise_playback, replay_folder_path, replay_mode, invalid_minimum_request_period)

        # Attempt to successfully initialise the playback() method.
        self.sbo_data_source_replay.initalise_playback(replay_folder_path, replay_mode, minimum_request_period)

        # G: Test that the initialisation of playback was successful.
        actual_result = self.sbo_data_source_replay.playback_initialised()
        expected_result = True
        self.assertEqual(actual_result, expected_result, "[G] The actual result doesn't match the expected result.")
Ejemplo n.º 21
0
def handle_incoming_commits(commits, project, repository, build_queue):
    for commit in commits:
        debug.message("Add commit %s to database" % commit.oid, indent=2)

        commit_date = datetime.fromtimestamp(int(commit.commit_time)).strftime('%Y-%m-%d %H:%M:%S')

        if len(commit.parents) > 0:
            changes = repository.get_changes(commit, commit.parents[0])
        else:
            changes = repository.get_changes(commit)

        Api.add_commit(project["ID"], commit.oid, commit_date, commit.message, commit.author.name, changes)

        project["commit"] = commit.oid
        build_queue.put(project)

    return len(commits)
Ejemplo n.º 22
0
    def merge_origin(self):
        """
        Merge commits from origin.

        :rtype : bool
        :return: True if merge is fast-forward, false otherwise.
        """
        remote_commit = self.repository.revparse_single("origin/%s" % self.origin_branch)

        debug.message("Merge origin/%s" % self.origin_branch, indent=1)
        merge_result = self.repository.merge(remote_commit.oid)

        if merge_result.is_fastforward:
            reference = self.repository.lookup_reference("refs/heads/master")
            reference.target = merge_result.fastforward_oid.hex

            return True

        return False
Ejemplo n.º 23
0
    def reinitialise_playback(self):

        """This public method is called to re-initialise certain attributes ready for the playback() method to be called again.

        Raises:
            CallOrderError: Raised if playback has not previously been initialised.

        This simple method has no arguments and no returns.
        """
        debug.message(debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO, "%s: %s" % (SboDataSourceReplay.__name__, "reinitialise_playback"))

        if not self.playback_previously_initialised_flag:
            raise SboDataSourceReplay.CallOrderError("The reinitalise_playback() method was called before the initialise_playback() method.")

        # Re-initialise the playback attributes.
        self.last_file_played_back = [INITIAL_LAST_LIVE_FILE_PLAYED_BACK, INITIAL_LAST_NON_LIVE_FILE_PLAYED_BACK]
        self.replay_file_number[PLAYBACK_FILE] = INITIAL_PLAYBACK_FILE_NUMBER
        self.load_next_file_for_playback = True
        self.file_contents = [INITIAL_DATA_IDENTIFIER, INITIAL_RAW_DATA]

        self.playback_initialised_flag = True
Ejemplo n.º 24
0
    def check_for_new_commits_on_origin(self):
        """
        Check for new commits on remote repository.

        :rtype : list
        :return: a list of commits
        """
        remote = self.check_origin()
        remote.fetch()

        remote_commit = self.repository.revparse_single("origin/%s" % self.origin_branch)
        local_commit = self.repository.revparse_single("master")

        debug.message("Local is at %s" % str(local_commit.oid), indent=1)
        debug.message("Remote is at %s" % str(remote_commit.oid), indent=1)

        commits = []
        for commit in self.repository.walk(remote_commit.oid, pygit2.GIT_SORT_TIME):
            if local_commit.oid == commit.oid:
                break

            commits.append(commit)

        commits.reverse()

        commit_count = len(commits)
        if commit_count == 1:
            text = "commit"
        else:
            text = "commits"

        debug.message("Master is %d %s behind origin/%s" % (commit_count, text, self.origin_branch), indent=1)

        return commits
Ejemplo n.º 25
0
    def __init__(self):

        debug.message(debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO, "%s: %s" % (SboDataSourceReplay.__name__, "__init__"))

        # The base directory contains the script that was used to invoke the Python interpreter.
        self.base_directory = os.sys.path[0]

        self.capture_limit = False
        self.first_capture = True

        self.replay_file_number = [INITIAL_CAPTURE_FILE_NUMBER, INITIAL_PLAYBACK_FILE_NUMBER]
        self.replay_folder_path = None
        self.replay_mode = None
        self.minimum_request_period_live = None
        self.minimum_request_period_non_live = None

        self.playback_previously_initialised_flag = False
        self.playback_initialised_flag = False
        self.last_file_played_back = [INITIAL_LAST_LIVE_FILE_PLAYED_BACK, INITIAL_LAST_NON_LIVE_FILE_PLAYED_BACK]
        self.load_next_file_for_playback = True
        self.file_contents = [INITIAL_DATA_IDENTIFIER, INITIAL_RAW_DATA]
        self.last_datestamp = None
Ejemplo n.º 26
0
    def _get_next_filename(self, file_type):
        """This private method keeps track of and returns the file name of the next capture or playback file that can be written to.

        Args: file_type(integer), eg: either CAPTURE_FILE or PLAYBACK_FILE.
        Returns: filename(string)
        Raises: EndOfReplay
        """
        debug.message(
            debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO,
            "%s: %s" % (SboDataSourceReplay.__name__, "_get_next_filename"))

        # Capture and Playback files are numbered sequentially starting at One.
        self.replay_file_number[
            file_type] = self.replay_file_number[file_type] + 1

        # Once the file number for a Capture file has reached the maximum allowed, set a flag.
        # With the flag is set, this method will return the last Capture filename to be used.
        if file_type == CAPTURE_FILE and self.replay_file_number[
                file_type] == FILE_NUMBER_MAX:
            self.capture_limit = True

        # Once the file number for a Playback file has gone over the maximum allowed, raise an exception.
        if file_type == PLAYBACK_FILE and self.replay_file_number[
                file_type] > FILE_NUMBER_MAX:
            raise SboDataSourceReplay.EndOfReplay(
                "Can not playback more than %s files." % FILE_NUMBER_MAX)

        formatted_capture_file_number = str(
            self.replay_file_number[file_type]).zfill(FILE_NUMBER_DIGITS)

        # The capture filename template holds the filename of the capture data files with a dummy file number.
        # This statement replaces the 'nnnnn' in the dummy file with the real number, calculated above.
        filename = re.sub(r'n{5}', formatted_capture_file_number,
                          CAPTURE_FILENAME_TEMPLATE)

        return filename
Ejemplo n.º 27
0
    def run(self):
        debug.message("Starting build of project %s" % self.project["Name"], indent=1)

        build_result = 0
        for solution in self.get_solutions():
            build_result += self.run_build(solution)

            if build_result != 0:
                Api.update_status(self.commit["ID"], CommitStatus.BuildError)
                raise Exception("Build of solution %s failed" % solution["name"])

            self.publish()
            debug.message("Build of solution %s succeeded!" % solution["name"], indent=2)

        Api.update_status(self.commit["ID"], CommitStatus.Build)
        debug.message("Build of project %s completed" % self.project["Name"], indent=1)
Ejemplo n.º 28
0
    def _create_new_capture_folder(self):
        """This private method ensures that the necessary folder structure is in place for capturing raw data files.

        This simple method has no arguments or returns.
        It raises no errors.
        """
        debug.message(
            debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO, "%s: %s" %
            (SboDataSourceReplay.__name__, "_create_new_capture_folder"))

        # Ensure a folder exists to capture raw data to.
        capture_folder_path = os.path.join(self.base_directory, CAPTURE_FOLDER)

        if not os.path.exists(capture_folder_path):
            os.mkdir(capture_folder_path)

        debug.message(
            debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO,
            "Full path of capture root folder: '%s'" % capture_folder_path)

        # Ensure a sub-folder exists to capture raw data to.
        capture_current_path = os.path.join(capture_folder_path,
                                            CAPTURE_CURRENT_SUBFOLDER)

        if os.path.exists(capture_current_path):

            # If any raw data has previously been captured, the folder needs to be date stamped and archived.
            datestamp = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
            capture_archive_subfolder = CAPTURE_ARCHIVE_SUBFOLDER_PREFIX + datestamp

            capture_archive_path = os.path.join(capture_folder_path,
                                                capture_archive_subfolder)

            debug.message(debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO,
                          "capture_archive_path: %s" % capture_archive_path)

            # Archive the previous folder.
            os.rename(capture_current_path, capture_archive_path)

        # Either the capture folder doesn't exist yet or it has just been archived.
        os.mkdir(capture_current_path)

        self.first_capture = False
Ejemplo n.º 29
0
    def _create_new_capture_folder(self):

        """This private method ensures that the necessary folder structure is in place for capturing raw data files.

        This simple method has no arguments or returns.
        It raises no errors.
        """
        debug.message(debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO, "%s: %s" % (SboDataSourceReplay.__name__, "_create_new_capture_folder"))

        # Ensure a folder exists to capture raw data to.
        capture_folder_path = os.path.join(self.base_directory, CAPTURE_FOLDER)

        if not os.path.exists(capture_folder_path):
            os.mkdir(capture_folder_path)

        debug.message(debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO, "Full path of capture root folder: '%s'" % capture_folder_path)

        # Ensure a sub-folder exists to capture raw data to.
        capture_current_path = os.path.join(capture_folder_path, CAPTURE_CURRENT_SUBFOLDER)

        if os.path.exists(capture_current_path):

            # If any raw data has previously been captured, the folder needs to be date stamped and archived.
            datestamp = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
            capture_archive_subfolder = CAPTURE_ARCHIVE_SUBFOLDER_PREFIX + datestamp

            capture_archive_path = os.path.join(capture_folder_path, capture_archive_subfolder)

            debug.message(debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO, "capture_archive_path: %s" % capture_archive_path)

            # Archive the previous folder.
            os.rename(capture_current_path, capture_archive_path)

        # Either the capture folder doesn't exist yet or it has just been archived.
        os.mkdir(capture_current_path)

        self.first_capture = False
Ejemplo n.º 30
0
    def capture(self, raw_data, frame_type):
        """This public method writes the given raw data to a file.

        The next available capture file number is determined and the file is created.
        The raw data passed to this method is identified and a data identifier code is written to the file.
        After that, the current date and time is written and then the raw data its self.

        Args:
            raw_data: A string of raw data to be captured.
            frame_type: An integer which specifies if the raw data represents live or non-live betting data.

        Returns:
            None

        Raises:
            DataCaptureError: Raised on an error creating a new capture folder,
                an inability to identify the incoming raw data,
                the detection of invalid raw data
                or the failure to write a file to the server.
        """
        debug.message(debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO,
                      "%s: %s" % (SboDataSourceReplay.__name__, "capture"))

        # The number of files captures has reached the limit.
        if self.capture_limit:
            debug.message(
                debug_flags.SBO_DATA_SOURCE_REPLAY_WARNINGS, debug.WARNING,
                "%s: Can not capture more than %s files." %
                (SboDataSourceReplay.__name__, FILE_NUMBER_MAX))
            return

        # The first capture flag is set on instantiation of this class.
        # After the new capture folder is created the flag is cleared.
        if self.first_capture:
            try:
                self._create_new_capture_folder()

            except OSError as exception_instance:
                raise SboDataSourceReplay.DataCaptureError(
                    "capture() experienced %s: %s" %
                    (type(exception_instance).__name__, exception_instance))

        # The data identifier is useful when analysing the data files.
        if raw_data == '':
            data_identifier = NO_DATA

        elif frame_type == LIVE_DATA_FRAME:
            data_identifier = LIVE_DATA

        elif frame_type == NON_LIVE_DATA_FRAME:
            data_identifier = NON_LIVE_DATA

        else:
            raise SboDataSourceReplay.DataCaptureError(
                "Unable to identify raw data.")

        # The date stamp is useful when analysing the data files.
        datestamp = str(datetime.datetime.now())

        try:
            file_contents = data_identifier + '\n' + datestamp + '\n' + raw_data

        except TypeError as exception_instance:
            raise SboDataSourceReplay.DataCaptureError(
                "capture() experienced %s: %s" %
                (type(exception_instance).__name__, exception_instance))

        capture_filename = self._get_next_filename(CAPTURE_FILE)
        capture_file_path = os.path.join(self.base_directory, CAPTURE_FOLDER,
                                         CAPTURE_CURRENT_SUBFOLDER,
                                         capture_filename)

        try:
            file_handle = open(capture_file_path, mode='w', encoding='utf-8')

            debug.message(
                debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO,
                "Writing raw data to: '%s/%s'" %
                (CAPTURE_CURRENT_SUBFOLDER, capture_filename))

            try:
                # Any failure here will be caught by the outer except.
                file_handle.write(file_contents)

            finally:
                # The file will get closed even if the above try causes an exception.
                file_handle.close()

        except IOError as exception_instance:
            raise SboDataSourceReplay.DataCaptureError(
                "capture() experienced %s: %s" %
                (type(exception_instance).__name__, exception_instance))
Ejemplo n.º 31
0
    def capture(self, raw_data, frame_type):

        """This public method writes the given raw data to a file.

        The next available capture file number is determined and the file is created.
        The raw data passed to this method is identified and a data identifier code is written to the file.
        After that, the current date and time is written and then the raw data its self.

        Args:
            raw_data: A string of raw data to be captured.
            frame_type: An integer which specifies if the raw data represents live or non-live betting data.

        Returns:
            None

        Raises:
            DataCaptureError: Raised on an error creating a new capture folder,
                an inability to identify the incoming raw data,
                the detection of invalid raw data
                or the failure to write a file to the server.
        """
        debug.message(debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO, "%s: %s" % (SboDataSourceReplay.__name__, "capture"))

        # The number of files captures has reached the limit.
        if self.capture_limit:
            debug.message(debug_flags.SBO_DATA_SOURCE_REPLAY_WARNINGS, debug.WARNING, "%s: Can not capture more than %s files." % (SboDataSourceReplay.__name__, FILE_NUMBER_MAX))
            return

        # The first capture flag is set on instantiation of this class.
        # After the new capture folder is created the flag is cleared.
        if self.first_capture:
            try:
                self._create_new_capture_folder()

            except OSError as exception_instance:
                raise SboDataSourceReplay.DataCaptureError("capture() experienced %s: %s" % (type(exception_instance).__name__, exception_instance))

        # The data identifier is useful when analysing the data files.
        if raw_data == '':
            data_identifier = NO_DATA

        elif frame_type == LIVE_DATA_FRAME:
            data_identifier = LIVE_DATA

        elif frame_type == NON_LIVE_DATA_FRAME:
            data_identifier = NON_LIVE_DATA

        else:
            raise SboDataSourceReplay.DataCaptureError("Unable to identify raw data.")

        # The date stamp is useful when analysing the data files.
        datestamp = str(datetime.datetime.now())

        try:
            file_contents = data_identifier + '\n' + datestamp + '\n' + raw_data

        except TypeError as exception_instance:
            raise SboDataSourceReplay.DataCaptureError("capture() experienced %s: %s" % (type(exception_instance).__name__, exception_instance))

        capture_filename = self._get_next_filename(CAPTURE_FILE)
        capture_file_path = os.path.join(self.base_directory, CAPTURE_FOLDER, CAPTURE_CURRENT_SUBFOLDER, capture_filename)

        try:
            file_handle = open(capture_file_path, mode='w', encoding='utf-8')

            debug.message(debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO, "Writing raw data to: '%s/%s'" % (CAPTURE_CURRENT_SUBFOLDER, capture_filename))

            try:
                # Any failure here will be caught by the outer except.
                file_handle.write(file_contents)

            finally:
                # The file will get closed even if the above try causes an exception.
                file_handle.close()

        except IOError as exception_instance:
            raise SboDataSourceReplay.DataCaptureError("capture() experienced %s: %s" % (type(exception_instance).__name__, exception_instance))
Ejemplo n.º 32
0
    def initalise_playback(self, replay_folder_path, replay_mode, minimum_request_period):

        """This public method ensures that the object is initialised ready for the playback() method to be called.

        This method initialises the replay folder path, establishes the replay mode and stores the minimum request periods
        ready to the playback function to be called.

        Args:
            replay_folder_path: A string representing the path of the folder containing files to be replayed.
            replay_mode: An integer representing either Normal or Fast replay mode.
            minimum_request_period: A list of minimum request periods used during normal playback to limit the frequency in which the data is played back.

        Returns:
            None

        Raises:
            PlaybackInitialisationError: Raised if invalid arguments are passed to this method,
                preventing it from initialising the required parameters ready for playback.
        """
        debug.message(debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO, "%s: %s" % (SboDataSourceReplay.__name__, "initalise_playback"))

        try:
            # If the replay folder path does not exist then playback can not be initialised.
            replay_folder_path = os.path.join(self.base_directory, replay_folder_path)

        except AttributeError as exception_instance:
            raise SboDataSourceReplay.PlaybackInitialisationError("initalise_playback() experienced %s: %s" % (type(exception_instance).__name__, exception_instance))

        if not os.path.exists(replay_folder_path):
            self.playback_initialised_flag = False
            raise SboDataSourceReplay.PlaybackInitialisationError("Can not initialise playback, invalid replay folder given. '%s'" % replay_folder_path)

        # This folder specifies the path of the files to be played back.
        self.replay_folder_path = replay_folder_path
        debug.message(debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO, "Specified replay folder: '%s'" % self.replay_folder_path)

        # Fast replay mode allows the minimum request periods to be ignored during playback.
        if replay_mode == NORMAL_REPLAY_MODE:
            self.replay_mode = NORMAL_REPLAY_MODE

        elif replay_mode == FAST_REPLAY_MODE:
            self.replay_mode = FAST_REPLAY_MODE

        else:
            self.replay_mode = DEFAULT_REPLAY_MODE

        debug.message(debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO, "%s specified." % REPLAY_MODE_DESCRIPTION[self.replay_mode])

        try:
            # The minimum request periods are used during normal playback to limit the frequency in which the data can be played back.
            self.minimum_request_period_live = minimum_request_period[LIVE_DATA_FRAME]
            self.minimum_request_period_non_live = minimum_request_period[NON_LIVE_DATA_FRAME]

        except (TypeError, IndexError, KeyError) as exception_instance:
            raise SboDataSourceReplay.PlaybackInitialisationError("initalise_playback() experienced %s: %s" % (type(exception_instance).__name__, exception_instance))

        try:
            # Ensure that the minimum request periods are valid timedelta objects.
            # Attempt to call the total_seconds() attribute as this will only succeed on a timedelta object.
            self.minimum_request_period_live.total_seconds()
            self.minimum_request_period_non_live.total_seconds()

        except AttributeError as exception_instance:
            raise SboDataSourceReplay.PlaybackInitialisationError("initalise_playback() experienced %s: %s" % (type(exception_instance).__name__, exception_instance))

        self.playback_initialised_flag = True
        self.playback_previously_initialised_flag = True
Ejemplo n.º 33
0
    def playback(self):

        """This public method returns the contents of the next available replay file.

        The method looks in the replay folder setup during playback initialisation and attempts to read data from
        each file in turn. On reading the file data the data identifier that was encoded into the file during capture is
        read and the data is handled accordingly.
        The replay mode and minimum request periods specified during initialisation are observed.

        Args: None

        Returns:
            raw_data: A string of raw data retrieved from the replay file.

        Raises:
            CallOrderError: Raised if playback has not previously been initialised.
            EndOfReplay: Raised when all the replay files in the specified folder have been played back.
            FileReadError: Raised if an error is encountered while attempting to read the file contents of a replay file.
            SimulatedConnectionError: Raised when the data identifier of a replay file indicated no data. Used only during testing.
            PlaybackError: Raised when a reply file with an unknown data identifier is discovered.
        """
        debug.message(debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO, "%s: %s" % (SboDataSourceReplay.__name__, "playback"))

        if not self.playback_initialised_flag:
            raise SboDataSourceReplay.CallOrderError("The playback() method was called before the initialise_playback() method.")

        if self.load_next_file_for_playback:

            playback_filename = self._get_next_filename(PLAYBACK_FILE)
            replay_file_path = os.path.join(self.replay_folder_path, playback_filename)

            if not os.path.exists(replay_file_path):

                self.playback_initialised_flag = False
                raise SboDataSourceReplay.EndOfReplay("Can not find file '%s' in folder '%s'" % (playback_filename, self.replay_folder_path))

            try:
                file_handle = open(replay_file_path, mode='r', encoding='utf-8')

                debug.message(debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO, "Opening file for playback: '%s'" % replay_file_path)

                try:
                    # Any failure here will be caught by the outer except.

                    data_identifier = file_handle.readline().rstrip('\n')
                    datestamp = file_handle.readline().rstrip('\n')
                    captured_raw_data = file_handle.readline()

                    # If the file was captured with no raw data it should already have the No Data identifier.
                    # But if it has been modified it may not.
                    if captured_raw_data == '':
                        data_identifier = NO_DATA

                    # Raw data will only be returned form this function if certain conditions are met.
                    # If these conditions are not met this time, they may be met on future calls to this function.
                    # Store the data identifier and captured raw data until the next file is loaded.
                    self.file_contents = [data_identifier, captured_raw_data]

                finally:
                    # The file will get closed even if the above try causes an exception.
                    file_handle.close()

            except IOError as exception_instance:
                raise SboDataSourceReplay.FileReadError("playback() experienced %s: %s" % (type(exception_instance).__name__, exception_instance))

            debug.message(debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO, "Data Identifier: %s" % data_identifier)
            debug.message(debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO, "Date stamp: %s" % datestamp)

            # Keep track of the date stamp of the last valid raw data file to be opened.
            if data_identifier == LIVE_DATA or data_identifier == NON_LIVE_DATA:
                self.last_datestamp = datestamp

            self.load_next_file_for_playback = False

        # If non of the following conditions are met then raw data will be returned as None.
        raw_data = None

        # In Fast Replay Mode, data can be read at any time.
        # In Normal Replay Mode, data can not be read more frequent than the minimum request period.

        if self.file_contents[DATA_IDENTIFIER] == LIVE_DATA:

            if (self.replay_mode == FAST_REPLAY_MODE or
                self.replay_mode == NORMAL_REPLAY_MODE and
                (self.last_file_played_back[LIVE_DATA_FRAME] is None or
                 (datetime.datetime.now() - self.last_file_played_back[LIVE_DATA_FRAME]) > self.minimum_request_period_live)):

                raw_data = self.file_contents[RAW_DATA]

                # The date stamp makes it possible for the class to know when live raw data was last returned.
                self.last_file_played_back[LIVE_DATA_FRAME] = datetime.datetime.now()
                self.load_next_file_for_playback = True

            else:
                debug.message(debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO, "Tried to playback live replay data file before minimum request period expired.")

        elif self.file_contents[DATA_IDENTIFIER] == NON_LIVE_DATA:

            if (self.replay_mode == FAST_REPLAY_MODE or
                self.replay_mode == NORMAL_REPLAY_MODE and
                (self.last_file_played_back[NON_LIVE_DATA_FRAME] is None or
                 (datetime.datetime.now() - self.last_file_played_back[NON_LIVE_DATA_FRAME]) > self.minimum_request_period_non_live)):

                raw_data = self.file_contents[RAW_DATA]

                # The date stamp makes it possible for the class to know when non-live raw data was last returned.
                self.last_file_played_back[NON_LIVE_DATA_FRAME] = datetime.datetime.now()
                self.load_next_file_for_playback = True

            else:
                debug.message(debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO, "Tried to playback non-live replay data file before minimum request period expired.")

        elif self.file_contents[DATA_IDENTIFIER] == NO_DATA:

            # This identifier simulates a SBO server connection error.
            # This method must be re-initialised before it can be used again.
            self.playback_initialised_flag = False

            raise SboDataSourceReplay.SimulatedConnectionError("Simulated SBO server connection error.")

        else:

            # A file with an unknown Data Identifier will not be read.
            # This method must be re-initialised before it can be used again.
            self.playback_initialised_flag = False

            raise SboDataSourceReplay.PlaybackError("Unknown Data Identifier: '%s'" % self.file_contents[DATA_IDENTIFIER])

        return raw_data
Ejemplo n.º 34
0
    def test_05_playback_fast(self):

        """Test the playback of captured data files in fast replay mode."""

        debug.message(debug_flags.TEST_SBO_DATA_SOURCE_REPLAY, debug.TESTUNIT, "STARTING %s:" % "test_05_playback_fast")

        # In Fast Replay Mode, the minimum request periods should be ignored.
        # This means that playback should always return data without having to wait.

        # Define the parameters for fast replay mode.
        replay_folder_path = 'test_data/playback_files/set_003'
        replay_mode = FAST_REPLAY_MODE
        minimum_request_period = [datetime.timedelta(seconds = MINIMUM_REQUEST_PERIOD_LIVE), datetime.timedelta(seconds = MINIMUM_REQUEST_PERIOD_NON_LIVE)]

        # Attempt to successfully initialise the playback() method.
        self.reset_playback(replay_folder_path, replay_mode, minimum_request_period)

        # Attempt to playback data file 1.
        raw_data = self.sbo_data_source_replay.playback()

        # A: Test that the length of the returned raw data is as expected for data file 1.
        actual_result = len(raw_data)
        expected_result = 1003
        self.assertEqual(actual_result, expected_result, "[A] The actual result doesn't match the expected result.")

        # Attempt to playback data file 2.
        raw_data = self.sbo_data_source_replay.playback()

        # B: Test that the length of the returned raw data is as expected for data file 2.
        actual_result = len(raw_data)
        expected_result = 59
        self.assertEqual(actual_result, expected_result, "[B] The actual result doesn't match the expected result.")

        # Attempt to playback data file 3.
        raw_data = self.sbo_data_source_replay.playback()

        # C: Test that the length of the returned raw data is as expected for data file 3.
        actual_result = len(raw_data)
        expected_result = 50
        self.assertEqual(actual_result, expected_result, "[C] The actual result doesn't match the expected result.")

        # Attempt to playback data file 4.
        raw_data = self.sbo_data_source_replay.playback()

        # D: Test that the length of the returned raw data is as expected for data file 4.
        actual_result = len(raw_data)
        expected_result = 67
        self.assertEqual(actual_result, expected_result, "[D] The actual result doesn't match the expected result.")

        # Create a temporary instance of the class.
        # Force the Playback file number to a high value to check how the class handles running out of numbers.
        temp_sbo_data_source_replay = SboDataSourceReplay()
        temp_sbo_data_source_replay.replay_file_number[PLAYBACK_FILE] = 99996

        # Define the parameters for fast replay mode.
        replay_folder_path = 'test_data/playback_files/set_004'
        replay_mode = FAST_REPLAY_MODE
        minimum_request_period = [datetime.timedelta(seconds = MINIMUM_REQUEST_PERIOD_LIVE), datetime.timedelta(seconds = MINIMUM_REQUEST_PERIOD_NON_LIVE)]

        # Attempt to successfully initialise the playback() method.
        temp_sbo_data_source_replay.initalise_playback(replay_folder_path, replay_mode, minimum_request_period)

        # Attempt to playback data file 99997.
        raw_data = temp_sbo_data_source_replay.playback()

        # E: Test that the length of the returned raw data is as expected for data file 99997.
        actual_result = len(raw_data)
        expected_result = 1115
        self.assertEqual(actual_result, expected_result, "[E] The actual result doesn't match the expected result.")

        # Attempt to playback data file 99998.
        raw_data = temp_sbo_data_source_replay.playback()

        # F: Test that the length of the returned raw data is as expected for data file 99998.
        actual_result = len(raw_data)
        expected_result = 59
        self.assertEqual(actual_result, expected_result, "[F] The actual result doesn't match the expected result.")

        # Attempt to playback data file 99999.
        raw_data = temp_sbo_data_source_replay.playback()

        # G: Test that the length of the returned raw data is as expected for data file 99999.
        actual_result = len(raw_data)
        expected_result = 60
        self.assertEqual(actual_result, expected_result, "[G] The actual result doesn't match the expected result.")

        # H: Test that trying to read again after reaching the limit raises an EndOfReplay exception.
        self.assertRaises(SboDataSourceReplay.EndOfReplay, temp_sbo_data_source_replay.playback)
Ejemplo n.º 35
0
    def test_04_playback_normal(self):

        """Test the playback of captured data files in normal replay mode."""

        debug.message(debug_flags.TEST_SBO_DATA_SOURCE_REPLAY, debug.TESTUNIT, "STARTING %s:" % "test_04_playback_normal")

        replay_mode = NORMAL_REPLAY_MODE
        minimum_request_period = [datetime.timedelta(seconds = MINIMUM_REQUEST_PERIOD_LIVE), datetime.timedelta(seconds = MINIMUM_REQUEST_PERIOD_NON_LIVE)]

        # Attempt to successfully initialise the playback() method.
        # Folder 'set_001' is empty.
        empty_replay_folder_path = 'test_data/playback_files/set_001'
        self.sbo_data_source_replay.initalise_playback(empty_replay_folder_path, replay_mode, minimum_request_period)

        # A: Test that trying to play back from an empty directory raises an exception.
        self.assertRaises(SboDataSourceReplay.EndOfReplay, self.sbo_data_source_replay.playback)

        # Attempt to successfully initialise the playback() method.
        # The file in the 'set_002' folder has a corrupt data identifier.
        corrupt_replay_folder_path = 'test_data/playback_files/set_002'
        self.reset_playback(corrupt_replay_folder_path, replay_mode, minimum_request_period)

        # B: Test that the method raises an error when reading a corrupt file.
        self.assertRaises(SboDataSourceReplay.PlaybackError, self.sbo_data_source_replay.playback)

        # After playback raises an error, it requires the initalise_playback() method to be called again.

        # C: Test that the playback() method raises an error when called before initialisation.
        self.assertRaises(SboDataSourceReplay.CallOrderError, self.sbo_data_source_replay.playback)

        # Attempt to successfully initialise the playback() method.
        # The 'set_003' folder contains a series of valid playback files.
        replay_folder_path = 'test_data/playback_files/set_003'
        self.reset_playback(replay_folder_path, replay_mode, minimum_request_period)

        # Attempt to playback data file 1.
        raw_data = self.sbo_data_source_replay.playback()

        # D: Test that the length of the returned raw data is as expected for data file 1.
        actual_result = len(raw_data)
        expected_result = 1003
        self.assertEqual(actual_result, expected_result, "[D] The actual result doesn't match the expected result.")

        # Attempt to playback data file 2.
        # This is the first file containing non-live data so can be played back straight away.
        # Note that if it contained live data, it could not be played back until the minimum request period had elapsed.
        raw_data = self.sbo_data_source_replay.playback()

        # E: Test that the length of the returned raw data is as expected for data file 2.
        actual_result = len(raw_data)
        expected_result = 59
        self.assertEqual(actual_result, expected_result, "[E] The actual result doesn't match the expected result.")

        # Attempt to playback data file 3.
        # This is a file containing live data and not enough time has elapsed since the last file was read.
        raw_data = self.sbo_data_source_replay.playback()

        # F: Test that no raw data is returned.
        actual_result = raw_data
        expected_result = None
        self.assertEqual(actual_result, expected_result, "[F] The actual result doesn't match the expected result.")

        # Wait the minimum required time, plus a safety margin and try again.
        seconds = MINIMUM_REQUEST_PERIOD_LIVE + SAFETY_MARGIN
        debug.message(debug_flags.TEST_SBO_DATA_SOURCE_REPLAY, debug.TESTUNIT, "Waiting for %s seconds..." % seconds)
        time.sleep(seconds)

        # Attempt to playback data file 3.
        raw_data = self.sbo_data_source_replay.playback()

        # G: Test that the length of the returned raw data is as expected for data file 3.
        actual_result = len(raw_data)
        expected_result = 50
        self.assertEqual(actual_result, expected_result, "[G] The actual result doesn't match the expected result.")

        # Get the last date stamp which should be for data file 3.
        datestamp_3 = self.sbo_data_source_replay.get_last_datestamp()

        # H: Test that the length of the returned raw data is as expected for data file 3.
        actual_result = len(datestamp_3)
        expected_result = 26
        self.assertEqual(actual_result, expected_result, "[H] The actual result doesn't match the expected result.")

        # Attempt to playback data file 4.
        # This is a file containing non-live data and not enough time has elapsed since the last file was read.
        raw_data = self.sbo_data_source_replay.playback()

        # I: Test that no raw data is returned.
        actual_result = raw_data
        expected_result = None
        self.assertEqual(actual_result, expected_result, "[I] The actual result doesn't match the expected result.")

        # Wait the minimum required time, plus a safety margin and try again.
        seconds = MINIMUM_REQUEST_PERIOD_NON_LIVE + SAFETY_MARGIN
        debug.message(debug_flags.TEST_SBO_DATA_SOURCE_REPLAY, debug.TESTUNIT, "Waiting for %s seconds..." % seconds)
        time.sleep(seconds)

        # Attempt to playback data file 4.
        raw_data = self.sbo_data_source_replay.playback()

        # J: Test that the length of the returned raw data is as expected for data file 4.
        actual_result = len(raw_data)
        expected_result = 67
        self.assertEqual(actual_result, expected_result, "[J] The actual result doesn't match the expected result.")

        # Get the last date stamp which should be for data file 4.
        datestamp_4 = self.sbo_data_source_replay.get_last_datestamp()

        # K: Test that the length of the returned raw data is as expected for data file 4.
        actual_result = len(datestamp_4)
        expected_result = 26
        self.assertEqual(actual_result, expected_result, "[K] The actual result doesn't match the expected result.")

        # L: Test the last two date stamps are not identical.
        self.assertNotEqual(datestamp_3, datestamp_4, "[L] The last two date stamps are identical.")

        # Attempt to playback data file 5.
        # This is a file containing no data and is marked with connection to server lost.

        # M: Test that a simulated connection error is raised.
        self.assertRaises(SboDataSourceReplay.SimulatedConnectionError, self.sbo_data_source_replay.playback)

        # N: Test that the previous connection error has cause playback to become unavailable.
        actual_result = self.sbo_data_source_replay.playback_initialised()
        expected_result = False
        self.assertEqual(actual_result, expected_result, "[N] The actual result doesn't match the expected result.")
Ejemplo n.º 36
0
    def test_02_capture(self):

        """Test the capturing of raw data to file."""

        debug.message(debug_flags.TEST_SBO_DATA_SOURCE_REPLAY, debug.TESTUNIT, "STARTING %s:" % "test_02_capture")

        # Define valid parameters for the capture() method.
        valid_raw_data = "$Page.onUpdate([35210,0,1,[[[3,'TEST LEAGUE 1','','']],0],[[1158696,1158767],[],[1158767,1162975,1164090],0],[[],[],[]]]);"
        valid_frame_type = LIVE_DATA_FRAME

        # A: Test that the method can cope with invalid inputs by raising an exception.
        for number, invalid_raw_data in enumerate(self.test_types, start=1):

            # Test type 6 is a valid value of raw data.
            if number == 6:
                continue

            debug.message(debug_flags.TEST_SBO_DATA_SOURCE_REPLAY_EXTRA, debug.TESTUNIT, "Testing raw data with invalid type %s of %s." % (number, len(self.test_types)))
            self.assertRaises(SboDataSourceReplay.DataCaptureError, self.sbo_data_source_replay.capture, invalid_raw_data, valid_frame_type)

        # B: Test that the method can cope with invalid inputs by raising an exception.
        for number, invalid_frame_type in enumerate(self.test_types, start=1):

            # Test types 2 to 6 are valid values of frame types.
            if number in range(2, 6):
                continue

            debug.message(debug_flags.TEST_SBO_DATA_SOURCE_REPLAY_EXTRA, debug.TESTUNIT, "Testing frame type with invalid type %s of %s." % (number, len(self.test_types)))
            self.assertRaises(SboDataSourceReplay.DataCaptureError, self.sbo_data_source_replay.capture, valid_raw_data, invalid_frame_type)

        # Send the valid parameters to the capture() method.
        self.sbo_data_source_replay.capture(valid_raw_data, valid_frame_type)

        # This will be the first capture since the object was created, so it should attempt to check/create the capture directory structure.
        expected_capture_path = os.path.join(self.sbo_data_source_replay.base_directory, 'sbo_data_captured/current')

        # C: Test that the expected capture path exists.
        actual_result = os.path.exists(expected_capture_path)
        self.assertTrue(actual_result, "[C] The expected capture path does not exist.")

        # The first file should be numbered 1, padded with zeros to total 5 digits.
        expected_capture_file_path = os.path.join(expected_capture_path, 'today-double-data-00001.dat')

        # D: Test that the expected capture file exists.
        actual_result = os.path.exists(expected_capture_file_path)
        self.assertTrue(actual_result, "[D] The expected capture file does not exist.")

        # Read the contents of the captured file.
        data_identifier, datestamp, raw_data = self.read_file(expected_capture_file_path)

        # E: Test that the data identifier is as expected.
        actual_result = data_identifier
        expected_result = 'IN PLAY DATA'
        self.assertEqual(actual_result, expected_result, "[E] The actual result doesn't match the expected result.")

        # F: Test that the date stamp is as expected.
        actual_result = len(datestamp)
        expected_result = 26
        self.assertEqual(actual_result, expected_result, "[F] The actual result doesn't match the expected result.")

        # G: Test that the raw data is as expected.
        actual_result = raw_data
        expected_result = valid_raw_data
        self.assertEqual(actual_result, expected_result, "[G] The actual result doesn't match the expected result.")

        # Send non-live data to be captured.
        frame_type = NON_LIVE_DATA_FRAME
        self.sbo_data_source_replay.capture(valid_raw_data, frame_type)

        # H: Test that the expected capture file exists.
        expected_capture_file_path = os.path.join(expected_capture_path, 'today-double-data-00002.dat')
        actual_result = os.path.exists(expected_capture_file_path)
        self.assertTrue(actual_result, "[H] The expected capture file does not exist.")

        # Read the contents of the captured file.
        data_identifier, datestamp, raw_data = self.read_file(expected_capture_file_path)

        # I: Test that the data identifier is as expected.
        actual_result = data_identifier
        expected_result = 'NOT IN PLAY DATA'
        self.assertEqual(actual_result, expected_result, "[I] The actual result doesn't match the expected result.")

        # Send empty data to be captured.
        raw_data = ''
        self.sbo_data_source_replay.capture(raw_data, valid_frame_type)

        # J: Test that the expected capture file exists.
        expected_capture_file_path = os.path.join(expected_capture_path, 'today-double-data-00003.dat')
        actual_result = os.path.exists(expected_capture_file_path)
        self.assertTrue(actual_result, "[J] The expected capture file does not exist.")

        # Read the contents of the captured file.
        data_identifier, datestamp, raw_data = self.read_file(expected_capture_file_path)

        # K: Test that the data identifier is as expected.
        actual_result = data_identifier
        expected_result = 'CONNECTION TO SERVER LOST'
        self.assertEqual(actual_result, expected_result, "[K] The actual result doesn't match the expected result.")

        # Archived capture folders are date stamped to the nearest second.
        # Allow some time to pass before archiving again to avoid trying to overwrite previous results.
        seconds = 2
        debug.message(debug_flags.TEST_SBO_DATA_SOURCE_REPLAY, debug.TESTUNIT, "Waiting for %s seconds..." % seconds)
        time.sleep(seconds)

        # Create a temporary instance of the class.
        # Force the Capture file number to a high value to check how the class handles running out of numbers.
        temp_sbo_data_source_replay = SboDataSourceReplay()
        temp_sbo_data_source_replay.replay_file_number[CAPTURE_FILE] = 99997

        # Call the Capture method 4 times to ensure the class runs out of Capture file numbers.
        # On the first call, the start number will be incremented and filename 'today-double-data-99998.dat' should be used.
        # On the second call, 'today-double-data-99999.dat' should be used. This is the last available file name.
        # On any subsequent calls, data should not be captured.
        # A warning should be displayed on the console and the application should not crash.
        temp_sbo_data_source_replay.capture(valid_raw_data, valid_frame_type)
        temp_sbo_data_source_replay.capture(valid_raw_data, valid_frame_type)
        temp_sbo_data_source_replay.capture(valid_raw_data, valid_frame_type)
        temp_sbo_data_source_replay.capture(valid_raw_data, valid_frame_type)

        # Get a list of filenames in the current capture directory.
        current_filenames = os.listdir(self.capture_current_path)

        # L: Test that the current capture directory contains the expected files.
        actual_result = sorted(current_filenames)
        expected_result = ['today-double-data-99998.dat', 'today-double-data-99999.dat']
        self.assertListEqual(actual_result, expected_result, "[L] The actual result doesn't match the expected result.")
Ejemplo n.º 37
0
    def initalise_playback(self, replay_folder_path, replay_mode,
                           minimum_request_period):
        """This public method ensures that the object is initialised ready for the playback() method to be called.

        This method initialises the replay folder path, establishes the replay mode and stores the minimum request periods
        ready to the playback function to be called.

        Args:
            replay_folder_path: A string representing the path of the folder containing files to be replayed.
            replay_mode: An integer representing either Normal or Fast replay mode.
            minimum_request_period: A list of minimum request periods used during normal playback to limit the frequency in which the data is played back.

        Returns:
            None

        Raises:
            PlaybackInitialisationError: Raised if invalid arguments are passed to this method,
                preventing it from initialising the required parameters ready for playback.
        """
        debug.message(
            debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO,
            "%s: %s" % (SboDataSourceReplay.__name__, "initalise_playback"))

        try:
            # If the replay folder path does not exist then playback can not be initialised.
            replay_folder_path = os.path.join(self.base_directory,
                                              replay_folder_path)

        except AttributeError as exception_instance:
            raise SboDataSourceReplay.PlaybackInitialisationError(
                "initalise_playback() experienced %s: %s" %
                (type(exception_instance).__name__, exception_instance))

        if not os.path.exists(replay_folder_path):
            self.playback_initialised_flag = False
            raise SboDataSourceReplay.PlaybackInitialisationError(
                "Can not initialise playback, invalid replay folder given. '%s'"
                % replay_folder_path)

        # This folder specifies the path of the files to be played back.
        self.replay_folder_path = replay_folder_path
        debug.message(
            debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO,
            "Specified replay folder: '%s'" % self.replay_folder_path)

        # Fast replay mode allows the minimum request periods to be ignored during playback.
        if replay_mode == NORMAL_REPLAY_MODE:
            self.replay_mode = NORMAL_REPLAY_MODE

        elif replay_mode == FAST_REPLAY_MODE:
            self.replay_mode = FAST_REPLAY_MODE

        else:
            self.replay_mode = DEFAULT_REPLAY_MODE

        debug.message(
            debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO,
            "%s specified." % REPLAY_MODE_DESCRIPTION[self.replay_mode])

        try:
            # The minimum request periods are used during normal playback to limit the frequency in which the data can be played back.
            self.minimum_request_period_live = minimum_request_period[
                LIVE_DATA_FRAME]
            self.minimum_request_period_non_live = minimum_request_period[
                NON_LIVE_DATA_FRAME]

        except (TypeError, IndexError, KeyError) as exception_instance:
            raise SboDataSourceReplay.PlaybackInitialisationError(
                "initalise_playback() experienced %s: %s" %
                (type(exception_instance).__name__, exception_instance))

        try:
            # Ensure that the minimum request periods are valid timedelta objects.
            # Attempt to call the total_seconds() attribute as this will only succeed on a timedelta object.
            self.minimum_request_period_live.total_seconds()
            self.minimum_request_period_non_live.total_seconds()

        except AttributeError as exception_instance:
            raise SboDataSourceReplay.PlaybackInitialisationError(
                "initalise_playback() experienced %s: %s" %
                (type(exception_instance).__name__, exception_instance))

        self.playback_initialised_flag = True
        self.playback_previously_initialised_flag = True
Ejemplo n.º 38
0
    def playback(self):
        """This public method returns the contents of the next available replay file.

        The method looks in the replay folder setup during playback initialisation and attempts to read data from
        each file in turn. On reading the file data the data identifier that was encoded into the file during capture is
        read and the data is handled accordingly.
        The replay mode and minimum request periods specified during initialisation are observed.

        Args: None

        Returns:
            raw_data: A string of raw data retrieved from the replay file.

        Raises:
            CallOrderError: Raised if playback has not previously been initialised.
            EndOfReplay: Raised when all the replay files in the specified folder have been played back.
            FileReadError: Raised if an error is encountered while attempting to read the file contents of a replay file.
            SimulatedConnectionError: Raised when the data identifier of a replay file indicated no data. Used only during testing.
            PlaybackError: Raised when a reply file with an unknown data identifier is discovered.
        """
        debug.message(debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO,
                      "%s: %s" % (SboDataSourceReplay.__name__, "playback"))

        if not self.playback_initialised_flag:
            raise SboDataSourceReplay.CallOrderError(
                "The playback() method was called before the initialise_playback() method."
            )

        if self.load_next_file_for_playback:

            playback_filename = self._get_next_filename(PLAYBACK_FILE)
            replay_file_path = os.path.join(self.replay_folder_path,
                                            playback_filename)

            if not os.path.exists(replay_file_path):

                self.playback_initialised_flag = False
                raise SboDataSourceReplay.EndOfReplay(
                    "Can not find file '%s' in folder '%s'" %
                    (playback_filename, self.replay_folder_path))

            try:
                file_handle = open(replay_file_path,
                                   mode='r',
                                   encoding='utf-8')

                debug.message(
                    debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO,
                    "Opening file for playback: '%s'" % replay_file_path)

                try:
                    # Any failure here will be caught by the outer except.

                    data_identifier = file_handle.readline().rstrip('\n')
                    datestamp = file_handle.readline().rstrip('\n')
                    captured_raw_data = file_handle.readline()

                    # If the file was captured with no raw data it should already have the No Data identifier.
                    # But if it has been modified it may not.
                    if captured_raw_data == '':
                        data_identifier = NO_DATA

                    # Raw data will only be returned form this function if certain conditions are met.
                    # If these conditions are not met this time, they may be met on future calls to this function.
                    # Store the data identifier and captured raw data until the next file is loaded.
                    self.file_contents = [data_identifier, captured_raw_data]

                finally:
                    # The file will get closed even if the above try causes an exception.
                    file_handle.close()

            except IOError as exception_instance:
                raise SboDataSourceReplay.FileReadError(
                    "playback() experienced %s: %s" %
                    (type(exception_instance).__name__, exception_instance))

            debug.message(debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO,
                          "Data Identifier: %s" % data_identifier)
            debug.message(debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO,
                          "Date stamp: %s" % datestamp)

            # Keep track of the date stamp of the last valid raw data file to be opened.
            if data_identifier == LIVE_DATA or data_identifier == NON_LIVE_DATA:
                self.last_datestamp = datestamp

            self.load_next_file_for_playback = False

        # If non of the following conditions are met then raw data will be returned as None.
        raw_data = None

        # In Fast Replay Mode, data can be read at any time.
        # In Normal Replay Mode, data can not be read more frequent than the minimum request period.

        if self.file_contents[DATA_IDENTIFIER] == LIVE_DATA:

            if (self.replay_mode == FAST_REPLAY_MODE
                    or self.replay_mode == NORMAL_REPLAY_MODE and
                (self.last_file_played_back[LIVE_DATA_FRAME] is None or
                 (datetime.datetime.now() -
                  self.last_file_played_back[LIVE_DATA_FRAME]) >
                 self.minimum_request_period_live)):

                raw_data = self.file_contents[RAW_DATA]

                # The date stamp makes it possible for the class to know when live raw data was last returned.
                self.last_file_played_back[
                    LIVE_DATA_FRAME] = datetime.datetime.now()
                self.load_next_file_for_playback = True

            else:
                debug.message(
                    debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO,
                    "Tried to playback live replay data file before minimum request period expired."
                )

        elif self.file_contents[DATA_IDENTIFIER] == NON_LIVE_DATA:

            if (self.replay_mode == FAST_REPLAY_MODE
                    or self.replay_mode == NORMAL_REPLAY_MODE and
                (self.last_file_played_back[NON_LIVE_DATA_FRAME] is None or
                 (datetime.datetime.now() -
                  self.last_file_played_back[NON_LIVE_DATA_FRAME]) >
                 self.minimum_request_period_non_live)):

                raw_data = self.file_contents[RAW_DATA]

                # The date stamp makes it possible for the class to know when non-live raw data was last returned.
                self.last_file_played_back[
                    NON_LIVE_DATA_FRAME] = datetime.datetime.now()
                self.load_next_file_for_playback = True

            else:
                debug.message(
                    debug_flags.SBO_DATA_SOURCE_REPLAY_INFOS, debug.INFO,
                    "Tried to playback non-live replay data file before minimum request period expired."
                )

        elif self.file_contents[DATA_IDENTIFIER] == NO_DATA:

            # This identifier simulates a SBO server connection error.
            # This method must be re-initialised before it can be used again.
            self.playback_initialised_flag = False

            raise SboDataSourceReplay.SimulatedConnectionError(
                "Simulated SBO server connection error.")

        else:

            # A file with an unknown Data Identifier will not be read.
            # This method must be re-initialised before it can be used again.
            self.playback_initialised_flag = False

            raise SboDataSourceReplay.PlaybackError(
                "Unknown Data Identifier: '%s'" %
                self.file_contents[DATA_IDENTIFIER])

        return raw_data