コード例 #1
0
    def current_function(self):
        """
        Thread reads the current requests and generates the prefetch requests
        that determines the next segment for all the current fetched bitrates.

        We use a separate prefetch queue to ensure that the prefetch does not affect the performance
        of the current requests
        """
        config_cdash.LOG.info('Current Thread: Started thread. Stop value = {}'.format(self.stop.is_set()))
        while not self.stop.is_set():
            try:
                current_request, username, session_id = self.current_queue.get(timeout=None)
                config_cdash.LOG.info('Retrieved the file: {}'.format(current_request))
            except Queue.Empty:
                config_cdash.LOG.error('Current Thread: Thread GET returned Empty value')
                current_request = None
                continue
            # Determining the next bitrates and adding to the prefetch list
            if current_request:
                if config_cdash.PREFETCH_SCHEME == 'SMART':
                    throughput = get_throughput_info(username, session_id, config_cdash.LIMIT, config_cdash.SCHEME)
                    config_cdash.LOG.info('average of throughput: = {}'.format(throughput))
                    prefetch_request, prefetch_bitrate = get_prefetch(current_request, config_cdash.PREFETCH_SCHEME, throughput)
                else:
                    prefetch_request, prefetch_bitrate = get_prefetch(current_request, config_cdash.PREFETCH_SCHEME, None)
            if not segment_exists(prefetch_request):
                config_cdash.LOG.info('Segment not there {}'.format(prefetch_request))
                if check_content_server(prefetch_request):
                    config_cdash.LOG.info('Current Thread: Current segment: {}, Next segment: {}'.format(current_request,
                                                                                                  prefetch_request))
                    self.prefetch_queue.put(prefetch_request)
                    config_cdash.LOG.info('Pre-fetch queue count = {}'.format(self.prefetch_queue.qsize()))
                else:
                    config_cdash.LOG.info('Current Thread: Invalid Next segment: {}'.format(current_request, prefetch_request))
            else:
                config_cdash.LOG.info('Segment already there {}'.format(prefetch_request))
        else:
            config_cdash.LOG.warning('Current Thread: terminated')
コード例 #2
0
ファイル: CacheManager.py プロジェクト: Sheydakm/Cache-n-DASH
    def current_function(self):
        """
        Thread reads the current requests and generates the prefetch requests
        that determines the next segment for all the current fetched bitrates.

        We use a separate prefetch queue to ensure that the prefetch does not affect the performance
        of the current requests
        """
        config_cdash.LOG.info('Current Thread: Started thread. Stop value = {}'.format(self.stop.is_set()))
        while not self.stop.is_set():
            try:
                current_request, username, session_id = self.current_queue.get(timeout=None)
                config_cdash.LOG.info('Retrieved the file: {}'.format(current_request))
            except Queue.Empty:
                config_cdash.LOG.error('Current Thread: Thread GET returned Empty value')
                current_request = None
                continue
            # Determining the next bitrates and adding to the prefetch list
            if current_request:
                if config_cdash.PREFETCH_SCHEME == 'SMART':
                    a=0.8
                    d=0.2
                    config_cdash.LOG.info('smart')
                    throughput_client= self.get_throughput_client(username, session_id)
                    if throughput_client== None:
                        config_cdash.LOG.info('throughput client:=None')
                        throughput= self.get_throughput_info(username, session_id, config_cdash.LIMIT, config_cdash.SCHEME)
                        config_cdash.LOG.info('average of throughput: = {}'.format(throughput))
                        forecast_throughput=throughput
                        self.insert_forecast(username, session_id,0.0)
                        self.insert_trend(username, session_id,0.0)
                        config_cdash.LOG.info('forecast throughput equal throughput:= {}'.format(forecast_throughput))
                        prefetch_request, prefetch_bitrate = get_prefetch(current_request, config_cdash.PREFETCH_SCHEME,forecast_throughput)
                    else:
                        config_cdash.LOG.info('throughput client:= {}'.format(throughput_client))
                        At_1=float(throughput_client)
                        config_cdash.LOG.info('type of At_1:= {}'.format(type(At_1)))
                        FIT_t_1=self.get_previous_forecast(username, session_id)
                        config_cdash.LOG.info('previous forecast:= {}'.format(FIT_t_1))
                        config_cdash.LOG.info('type of FIT_t_1:= {}'.format(type(FIT_t_1)))
                        Tt_1=self.get_previous_trend(username, session_id)
                        config_cdash.LOG.info('previous trend:= {}'.format(Tt_1))
                        config_cdash.LOG.info('type of Tt_1:= {}'.format(type(Tt_1)))
                        Ft=FIT_t_1+a*(At_1-FIT_t_1)
                        Tt=Tt_1+d*(Ft-FIT_t_1)
                        FIT_t=Ft+Tt
                        self.insert_forecast(username, session_id,FIT_t)
                        self.insert_trend(username, session_id,Tt)
                        config_cdash.LOG.info('forecast throughput: {}'.format(FIT_t))
                        prefetch_request, prefetch_bitrate = get_prefetch(current_request, config_cdash.PREFETCH_SCHEME,FIT_t)
                else:
                    prefetch_request, prefetch_bitrate = get_prefetch(current_request, config_cdash.PREFETCH_SCHEME, None)
                    config_cdash.LOG.info('not smart')
            if not segment_exists(prefetch_request):
                config_cdash.LOG.info('Segment not there {}'.format(prefetch_request))
                if check_content_server(prefetch_request):
                    t=type(prefetch_request)
                    config_cdash.LOG.info('Type will be saved in queue: {}'.format(t))
                    config_cdash.LOG.info('Current Thread: Current segment: {}, Next segment: {}'.format(current_request,
                                                                                                  prefetch_request))
                    self.prefetch_queue.put(prefetch_request)
                    self.backup_prefetch_queue.put(prefetch_request)
                    config_cdash.LOG.info('Pre-fetch queue count = {}'.format(self.prefetch_queue.qsize()))
                else:
                    config_cdash.LOG.info('Current Thread: Invalid Next segment: {}'.format(current_request, prefetch_request))
            else:
                config_cdash.LOG.info('Segment already there {}'.format(prefetch_request))
        else:
            config_cdash.LOG.warning('Current Thread: terminated')
コード例 #3
0
ファイル: cache_server.py プロジェクト: Sheydakm/Cache-n-DASH
    def do_GET(self):
  
        """Function to handle the get message"""
        start_time=time.time()
        config_cdash.LOG.info('T2 = {}'.format(start_time))
        entry_id = username = session_id = request_id = "NULL"
        request_size = throughput = request_time=request_t =time_c= "NULL"
        segment_size=seg_time="NULL"
        try:
            username = self.headers['Username']
            config_cdash.LOG.info("username {}".format(username))
            session_id = self.headers['Session-ID']
            config_cdash.LOG.info("Session-ID {}".format(session_id))
            if self.headers['Time']:
                time_c=self.headers['Time']
            else:
                time_c=0
            config_cdash.LOG.info("time_c {}".format(time_c))
            global client_throughput
            client_throughput=self.headers['Throughput']
            if client_throughput==None:
                client_throughput=0.0
            if client_throughput=='NULL':
                client_throughput=0.0
            config_cdash.LOG.info("Client Throughput {}".format(client_throughput))
            segment_size = self.headers['segment_size']
            config_cdash.LOG.info("segment_size {}".format(segment_size))
            seg_time = self.headers['seg_time']
            config_cdash.LOG.info("seg_time {}".format(seg_time))
            #global client_throughput
            #client_throughput=self.headers['Throughput']
            #if client_throughput==None:
            #    client_throughput=0.0
            #if client_throughput=='NULL':
            #    client_throughput=0.0
            #config_cdash.LOG.info("Client Throughput {}".format(client_throughput))
        except KeyError:
            config_cdash.LOG.warning('Could not find the username or session-ID for request from host:{}'.format(
                self.client_address))
        #start_time=time.time()
        #config_cdash.LOG.info('T2 = {}'.format(start_time))
        #s_time=str(datetime.datetime.time(datetime.datetime.now()))
        global MPD_DICT
        request = self.path.strip("/").split('?')[0]
        config_cdash.LOG.info("Received request {}".format(request))
        # check if mpd file requested is in Cache Server (dictionary)
        if request in MPD_DICT:
            config_cdash.LOG.info('Found MPD in MPD_DICT')
            request_path = request.replace('/', os.path.sep)
            make_sure_path_exists(config_cdash.MPD_FOLDER)
            local_mpd_path = os.path.join(config_cdash.MPD_FOLDER, request_path)
            for header in self.headers:
                config_cdash.LOG.info(header)
            self.send_response(HTTP_OK)
            for header, header_value in MPD_DICT[request]['http_headers'].items():
                self.send_header(header, header_value)
            self.end_headers()

            with open(local_mpd_path, 'rb') as request_file:
                #start_time = timeit.default_timer()
                self.wfile.write(request_file.read())
                # Elapsed time in seconds
                T3=time.time()
                request_t = T3 - start_time
                config_cdash.LOG.info('T3 = {}'.format(T3))
                config_cdash.LOG.info('Served the MPD file from the cache server')
            request_size = stat(local_mpd_path).st_size

        elif request in config_cdash.MPD_SOURCE_LIST:
            config_cdash.LOG.info("MPD: not in cache. Retrieving from Content server".format(request))
            # if mpd is in content server save it in cache server and put it in MPD_DICT and json file
            mpd_headers = None
            if "Big" in request:
                mpd_url = config_cdash.CONTENT_SERVER +config_cdash.SERVER[0]+ request
            elif "Elephants" in request:
                mpd_url = config_cdash.CONTENT_SERVER +config_cdash.SERVER[1]+ request
            elif "OfForest" in request:
                mpd_url = config_cdash.CONTENT_SERVER +config_cdash.SERVER[2]+ request
            elif "Tears" in request:
                mpd_url = config_cdash.CONTENT_SERVER +config_cdash.SERVER[3]+ request
            #mpd_url = config_cdash.CONTENT_SERVER + request
            try:
                content_server_response = urllib2.urlopen(mpd_url)
                mpd_headers = content_server_response.headers
                config_cdash.LOG.info('Fetching MPD from {}'.format(mpd_url))
            except urllib2.HTTPError as http_error:
                config_cdash.LOG.error('Unable to fetch MPD file from the content server url {}. HTTPError: {}'.format(
                    mpd_url, http_error.code))
            request_path = request.replace('/', os.path.sep)
            local_mpd_path = os.path.join(config_cdash.MPD_FOLDER, request_path)
            make_sure_path_exists(os.path.dirname(local_mpd_path))
            with open(local_mpd_path, 'wb') as local_mpd_file:
                shutil.copyfileobj(content_server_response, local_mpd_file)
            config_cdash.LOG.info('Downloaded the MPD: {} to {}'.format(content_server_response, local_mpd_path))
            self.send_response(HTTP_OK)
            for header, header_value in mpd_headers.items():
                self.send_header(header, header_value)
            self.end_headers()

            with open(local_mpd_path, 'rb') as request_file:
                #start_time = timeit.default_timer()
                self.wfile.write(request_file.read())
                # Elapsed time in seconds
                T3=time.time()
                request_t = T3 - start_time
                config_cdash.LOG.info('T3 = {}'.format(T3))
            # file_size in bytes
            request_size = stat(local_mpd_path).st_size
            config_cdash.LOG.info('Served MPD file:{}'.format(local_mpd_path))
            # client_ip, client_port = self.client_address
            mpd_headers_dict = dict(mpd_headers)
            config_cdash.LOG.info('Parsing MPD file')
            parse_mpd(local_mpd_path, request, mpd_headers_dict, (username, session_id ))
        elif 'm4s' in request:
            # Check if it is a valid request
            config_cdash.LOG.info('Request for m4s {}'.format(request))
            if check_content_server(request):
                local_file_path, http_headers = cache_manager.fetch_file(request, username, session_id)
                T3=time.time()
                request_t = T3 - start_time
                config_cdash.LOG.info('T3 = {}'.format(T3))
                #cache_manager.current_queue.put((request, username, session_id))
                config_cdash.LOG.debug('M4S request: local {}, http_headers: {}'.format(local_file_path, http_headers))
                self.send_response(HTTP_OK)
                for header, header_value in http_headers.items():
                    self.send_header(header, header_value)
                self.end_headers()
                with open(local_file_path, 'rb') as request_file:
                    #start_time = timeit.default_timer()
                    self.wfile.write(request_file.read())
                    #T3=time.time()
                    #request_t = T3 - start_time
                    #config_cdash.LOG.info('T3 = {}'.format(T3))
                request_size = stat(local_file_path).st_size
                # If valid request sent
                entry_id = datetime.datetime.now()
                cursor = TH_CONN.cursor()
                #Client_transfer=abs(float(time_c)-start_time)
                #FMT = '%H:%M:%S.%f'
                #Client_transfer = datetime.datetime.strptime(s_time, FMT) - datetime.datetime.strptime(time_c, FMT)
                #secs = Client_transfer.total_seconds()
                if time_c=='NULL':
                    time_c=0
                    Client_transfer=0
                else:
                    Client_transfer=abs(float(time_c)-start_time)
                config_cdash.LOG.info('sent header time {}'.format(time_c))
                config_cdash.LOG.info('start time {}'.format(start_time))
                config_cdash.LOG.info('Client transfer time {}'.format(Client_transfer))
                config_cdash.LOG.info('request time {}'.format(request_t))
                if request_t=='NULL':
                    request_t=0.0
                if request_size=='NULL':
                    request_size=0.0
                request_size = float(request_size)*8
                config_cdash.LOG.info('transfer size:{} '.format(request_size))
                request_time=float(request_t+(Client_transfer*2))
                throughput = float(request_size)/(request_time)
                #config_cdash.LOG.info('Adding row to Throughput database : '
                #                      'INSERT INTO THROUGHPUTDATA VALUES ({}, {}, {}, {}, {}, {}, {}, {},{},{});'.format(entry_id, username, session_id, request_id, request_size, request_time, throughput                                        ,client_throughput,'None','None'))
                #cursor.execute('INSERT INTO THROUGHPUTDATA(ENTRYID, USERNAME, SESSIONID, REQUESTSIZE, REQUESTTIME, THROUGHPUT, C_THROUGHPUT) '
                #                      'VALUES (?,?, ?, ?, ?, ?, ?);', (entry_id, username, session_id, request_size, request_time,throughput,client_throughput))
                #TH_CONN.commit()
                cache_manager.list_data.append((entry_id, username, session_id, request_size, request_time,throughput,client_throughput,None,None))
                #config_cdash.LOG.info('list_data in the begining:{} '.format(cache_manager.list_data))
                cache_manager.current_queue.put((request, username, session_id))

            else:
                config_cdash.LOG.warning('Invalid video file request: {}'.format(request))
        else:
            self.send_response(HTTP_NOT_FOUND)
            config_cdash.LOG.warning('Could not find file {}'.format(request))
            return
コード例 #4
0
    def do_GET(self):
        """Function to handle the get message"""
        entry_id = username = session_id = request_id = "NULL"
        request_size = throughput = request_time = "NULL"
        try:
            username = self.headers['Username']
            session_id = self.headers['Session-ID']
        except KeyError:
            config_cdash.LOG.warning('Could not find the username or session-ID for request from host:{}'.format(
                self.client_address))

        global MPD_DICT
        request = self.path.strip("/").split('?')[0]
        # check if mpd file requested is in Cache Server (dictionary)
        if request in MPD_DICT:
            config_cdash.LOG.info('Found MPD in MPD_DICT')
            request_path = request.replace('/', os.path.sep)
            make_sure_path_exists(config_cdash.MPD_FOLDER)
            local_mpd_path = os.path.join(config_cdash.MPD_FOLDER, request_path)
            for header in self.headers:
                config_cdash.LOG.info(header)
            self.send_response(HTTP_OK)
            for header, header_value in MPD_DICT[request]['http_headers'].items():
                self.send_header(header, header_value)
            self.end_headers()

            with open(local_mpd_path, 'rb') as request_file:
                start_time = timeit.default_timer()
                self.wfile.write(request_file.read())
                # Elapsed time in seconds
                request_time = timeit.default_timer() - start_time
                config_cdash.LOG.info('Served the MPD file from the cache server')
            request_size = stat(local_mpd_path).st_size

        elif request in config_cdash.MPD_SOURCE_LIST:
            config_cdash.LOG.info("MPD: not in cache. Retrieving from Content server".format(request))
            # if mpd is in content server save it in cache server and put it in MPD_DICT and json file
            mpd_headers = None
            mpd_url = config_cdash.CONTENT_SERVER + request
            try:
                content_server_response = urllib2.urlopen(mpd_url)
                mpd_headers = content_server_response.headers
                config_cdash.LOG.info('Fetching MPD from {}'.format(mpd_url))
            except urllib2.HTTPError as http_error:
                config_cdash.LOG.error('Unable to fetch MPD file from the content server url {}. HTTPError: {}'.format(
                    mpd_url, http_error.code))
            request_path = request.replace('/', os.path.sep)
            local_mpd_path = os.path.join(config_cdash.MPD_FOLDER, request_path)
            make_sure_path_exists(os.path.dirname(local_mpd_path))
            with open(local_mpd_path, 'wb') as local_mpd_file:
                shutil.copyfileobj(content_server_response, local_mpd_file)
            config_cdash.LOG.info('Downloaded the MPD: {} to {}'.format(content_server_response, local_mpd_path))
            self.send_response(HTTP_OK)
            for header, header_value in mpd_headers.items():
                self.send_header(header, header_value)
            self.end_headers()

            with open(local_mpd_path, 'rb') as request_file:
                start_time = timeit.default_timer()
                self.wfile.write(request_file.read())
                # Elapsed time in seconds
                request_time = timeit.default_timer() - start_time
            # file_size in bytes
            request_size = stat(local_mpd_path).st_size
            config_cdash.LOG.info('Served MPD file:{}'.format(local_mpd_path))
            # client_ip, client_port = self.client_address
            mpd_headers_dict = dict(mpd_headers)
            config_cdash.LOG.info('Parsing MPD file')
            parse_mpd(local_mpd_path, request, mpd_headers_dict, (username, session_id ))
        elif 'm4s' in request:
            # Check if it is a valid request
            config_cdash.LOG.info('Request for m4s {}'.format(request))
            if check_content_server(request):
                local_file_path, http_headers = cache_manager.fetch_file(request, username, session_id)
                config_cdash.LOG.debug('M4S request: local {}, http_headers: {}'.format(local_file_path, http_headers))
                self.send_response(HTTP_OK)
                for header, header_value in http_headers.items():
                    self.send_header(header, header_value)
                self.end_headers()
                with open(local_file_path, 'rb') as request_file:
                    start_time = timeit.default_timer()
                    self.wfile.write(request_file.read())
                    request_time = timeit.default_timer() - start_time
                request_size = stat(local_file_path).st_size

            else:
                config_cdash.LOG.warning('Invalid video file request: {}'.format(request))
        else:
            self.send_response(HTTP_NOT_FOUND)
            config_cdash.LOG.warning('Could not find file {}'.format(request))
            return
        # If valid request sent
        entry_id = datetime.datetime.now()
        cursor = TH_CONN.cursor()
        throughput = float(request_size)/request_time
        config_cdash.LOG.info('Adding row to Throughput database : '
                              'INSERT INTO THROUGHPUTDATA VALUES ({}, {}, {}, {}, {}, {}, {});'.format(
            entry_id, username, session_id, request_id, request_size, request_time, throughput))
        cursor.execute('INSERT INTO THROUGHPUTDATA(ENTRYID, USERNAME, SESSIONID, REQUESTSIZE, REQUESTTIME, THROUGHPUT) '
                       'VALUES (?, ?, ?, ?, ?, ?);', (entry_id, username, session_id, request_size, request_time,
                                                      throughput))
        TH_CONN.commit()