class FTPConfig:
file_manager = None
host = ''
username = ''
password = ''
replace_path = ''
http_url = ''
config = ConfigParser()
profile_tag = ''
def __init__(self, config_file, script_name='', unique_run_id=''):
self.file_manager = FileManager(script_name, self.__class__.__name__,
unique_run_id)
self.config.read(config_file)
def load(self, profile_tag):
self.host = ""
self.username = ""
self.password = ""
self.replace_path = ""
self.http_url = ""
self.profile_tag = profile_tag
is_loaded = False
if profile_tag in self.config:
self.host = self.config.get(profile_tag, 'ftpHost', fallback='')
self.username = self.config.get(profile_tag,
'ftpUser',
fallback='')
self.password = self.config.get(profile_tag,
'ftpPass',
fallback='')
self.replace_path = self.config.get(profile_tag,
'ftpReplace',
fallback='')
self.http_url = self.config.get(profile_tag,
'httpUrl',
fallback='')
is_loaded = True
else:
self.file_manager.log("Unknown profile: " + profile_tag +
" ; Please check ftp.ini.")
self.file_manager.out("Unknown ftp profile: " + profile_tag,
"FTPConfig.error.txt")
return is_loaded
class PreparingSubtitlesDASH:
file_manager = None
ftp_config = None
ftp = None
language_code_map = dict({"en": "en",
"id": "id",
"ko": "ko",
"ms": "ms", #Malay
"th": "th",
"vi": "vi",
"ar": "ar",
"my": "my", #Burmese,
"zz": "zz", #(Other)
"zh": "zh",
"zho": "zh",
"zh-Hans": "zh-Hans",
"zh-Hant": "zh-Hant",
"mis": "zz"})
def __init__(self):
self.file_manager = FileManager(SCRIPT_NAME, self.__class__.__name__, UNIQUE_RUN_ID)
self.ftp_config = FTPConfig(ROOT+"/resources/DownloadStorageFiles/ftp.ini", SCRIPT_NAME, UNIQUE_RUN_ID)
def run(self, input_file, input_directory):
self.file_manager.log("FUNCTION run() args -> input_file:" + input_file +
", input_directory:" + input_directory)
with open(input_file, "r") as f:
self.file_manager.log("OPEN input_file:" + input_file)
for line in f:
media_id = str(line).replace('\n', '')
self.file_manager.log("READLN media_id:" + media_id)
if media_id:
local_input_directory = input_directory + media_id + "/"
manifest_extension = "mpd"
local_media_dash = local_input_directory + "manifest." + manifest_extension
# replace_list = local_media_directory + manifest_extension + "_replace.txt"
# rollback_list = local_media_directory + manifest_extension + "_rollback.txt"
remote_dash_storage_host = ""
remote_dash_basepath = ""
manifest_info_file = local_input_directory + manifest_extension + "_info.txt"
with open(manifest_info_file, "r") as dash_basepath_file:
self.file_manager.log("OPEN [" + media_id + "] manifest_info_file:" + manifest_info_file)
for dash_basepath_line in dash_basepath_file:
self.file_manager.log("READLN [" + media_id + "] dash_basepath_line:" + dash_basepath_line)
dash_basepath_line = str(dash_basepath_line).replace('\n', '')
remote_dash_storage_host = dash_basepath_line.split(",")[0]
remote_dash_basepath = dash_basepath_line.split(",")[1]
self.file_manager.log("STORE [" + media_id + "] " +
"remote_dash_storage_host:" + input_file +
", remote_dash_storage_host:" + input_directory)
count_match = 0
if os.path.exists(local_media_dash):
self.file_manager.log("CONTINUE [" + media_id + "] (exists) local_media_dash:" + local_media_dash)
manifest_dom = minidom.parse(local_media_dash)
for docAdaptationSet in manifest_dom.getElementsByTagName('AdaptationSet'):
if docAdaptationSet.attributes['contentType']:
if docAdaptationSet.attributes['contentType'].value == 'text':
count_match += 1
self.file_manager.log("AdaptationSet contentType is text")
subtitle_language = docAdaptationSet.attributes['lang'].value #lang="zh-Hans"
docRepresentation = self.getNodeByName(docAdaptationSet, "Representation")
docBaseURL = self.getNodeByName(docRepresentation, "BaseURL")
subtitle_uri = ""
if docBaseURL:
subtitle_uri = self.getNodeText(docBaseURL)
if subtitle_language and subtitle_uri:
self.file_manager.log("PROCESS [" + media_id + "]" +
" subtitle_language: " + subtitle_language +
", subtitle_uri: " + subtitle_uri)
local_replace_sourcepath = "replace/" + media_id + "/"
local_rollback_sourcepath = "rollback/" + media_id + "/"
self.file_manager.validate_dir(self.file_manager.out_path, local_replace_sourcepath) # ensure folder exists
self.file_manager.validate_dir(self.file_manager.out_path, local_rollback_sourcepath) # ensure folder exists
local_replace_sourcepath = self.file_manager.out_path + "/" + local_replace_sourcepath
local_rollback_sourcepath = self.file_manager.out_path + "/" + local_rollback_sourcepath
# Copy vtt from DownloadStorageFiles and rename
fixed_subtitle_input = local_input_directory + subtitle_language + ".vtt"
local_replace_sourcefile = local_replace_sourcepath + subtitle_uri
self.file_manager.log("COPY START [" + media_id + "] Replace preparation " +
" fixed_subtitle_input: " + fixed_subtitle_input +
", local_replace_sourcefile: " + local_replace_sourcefile)
copyfile(fixed_subtitle_input, local_replace_sourcefile)
self.file_manager.log("COPY END [" + media_id + "] Replace preparation " +
" fixed_subtitle_input: " + fixed_subtitle_input +
", local_replace_sourcefile: " + local_replace_sourcefile)
# Download remote vtt file
remote_location = remote_dash_basepath + subtitle_uri
local_rollback_sourcefile = local_rollback_sourcepath + subtitle_uri
self.file_manager.log("COPY START [" + media_id + "] Rollback preparation " +
" remote_location: " + remote_location +
", local_rollback_sourcefile: " + local_rollback_sourcefile)
if self.download(remote_location, local_rollback_sourcefile, remote_dash_storage_host):
self.file_manager.out(media_id + "," + local_replace_sourcefile + "," +
remote_dash_storage_host + "," +
remote_location,
manifest_extension + "_replace.txt")
self.file_manager.out(media_id + "," + local_rollback_sourcefile + "," +
remote_dash_storage_host + "," +
remote_location,
manifest_extension + "_rollback.txt")
else:
self.file_manager.log("FAILED [" + media_id + "]" +
" to download source vtt for backup")
self.file_manager.log("COPY END [" + media_id + "] Rollback preparation " +
" remote_location: " + remote_location +
", local_rollback_sourcefile: " + local_rollback_sourcefile)
else:
self.file_manager.log("SKIP [" + media_id + "] Subtitle has empty values" +
" subtitle_language: " + subtitle_language +
", subtitle_uri: " + subtitle_uri)
else:
self.file_manager.log("SKIP [" + media_id + "] (do not exist) local_media_dash:" + local_media_dash)
if count_match == 0:
self.file_manager.log("SKIP [" + media_id + "] Manifest doesn't have subtitles, mediaId: " + media_id)
return True
def getNodeByName(self, nodes, name):
if nodes:
for node in nodes.childNodes:
if node.nodeName == name:
return node
return None
def getNodeText(self, nodes):
if nodes:
for node in nodes.childNodes:
if node.wholeText:
return node.wholeText
return ""
def download(self, source_file, destination_file, profile_tag):
is_success = True
try:
if self.ftp_config.profile_tag != profile_tag:
if self.ftp is not None:
self.ftp.quit()
self.ftp_config.load(profile_tag) # load storage_host as profile_tag
self.ftp = ftplib.FTP(self.ftp_config.host)
self.ftp.login(self.ftp_config.username, self.ftp_config.password)
# local_filepath = self.file_manager.get_outfile(destination_file)
local_file = open(destination_file, 'wb')
self.file_manager.log("Download: " + source_file + " >> " +
self.ftp.retrbinary('RETR ' + source_file, local_file.write))
local_file.close()
except ftplib.error_perm:
err_msg = "ERROR FTP file does not exist, filepath:" + source_file
self.file_manager.log(err_msg)
self.file_manager.out(err_msg, FILE_ERROR_OUT)
self.file_manager.delete_outfile(destination_file)
is_success = False
except ftplib.all_errors:
err_msg = "ERROR FTP all errors, filepath:" + source_file
self.file_manager.log(err_msg)
self.file_manager.out(err_msg, FILE_ERROR_OUT)
self.file_manager.delete_outfile(destination_file)
is_success = False
return is_success
class DownloadStorageFiles1:
file_manager = None
ftp_config = None
ftp = None
def __init__(self):
self.file_manager = FileManager(SCRIPT_NAME, self.__class__.__name__,
UNIQUE_RUN_ID)
self.ftp_config = FTPConfig(
ROOT + "/resources/DownloadStorageFiles/ftp.ini", SCRIPT_NAME,
UNIQUE_RUN_ID)
def download(self, input_file):
self.file_manager.log("--Download START -- :" + input_file)
with open(input_file, "r") as f:
for line in f:
line_path = str(line).replace('\n', '')
data = str(line_path).split(',')
if len(data) >= 3:
media_id = data[0]
self.file_manager.validate_dir(self.file_manager.out_path,
media_id +
"/") # ensure folder exists
storage_filehost = data[1]
storage_filepath = data[2]
source_filename = storage_filepath.split('/')[-1]
source_basepath = storage_filepath.replace(
source_filename, '')
# file_name = source_filename.split('.')[0]
file_extension = source_filename.split('.')[-1]
# local_filename = source_filename
local_filename = "manifest." + file_extension
if len(data) == 4:
local_filename = data[3] + "." + file_extension
else:
self.file_manager.out(
storage_filehost + "," + source_basepath,
media_id + "/" + file_extension + "_info.txt",
True)
try:
if self.ftp_config.profile_tag != storage_filehost:
if self.ftp is not None:
self.ftp.quit()
self.ftp_config.load(
storage_filehost
) # load storage_host as profile_tag
self.ftp = ftplib.FTP(self.ftp_config.host)
self.ftp.login(self.ftp_config.username,
self.ftp_config.password)
local_filepath = self.file_manager.get_outfile(
media_id + "/" + local_filename)
local_file = open(local_filepath, 'wb')
self.file_manager.log(
"Download: " + storage_filepath +
" >> " + self.ftp.retrbinary(
'RETR ' + storage_filepath, local_file.write))
local_file.close()
except ftplib.error_perm:
err_msg = "FTP file does not exist,mediaId:" + media_id + ",filepath:" + storage_filepath
self.file_manager.log(err_msg)
self.file_manager.out(err_msg, FILE_ERROR_OUT)
self.file_manager.delete_outfile(media_id + "/" +
local_filename)
except ftplib.all_errors:
err_msg = "FTP all errors,mediaId:" + media_id
self.file_manager.log(err_msg)
self.file_manager.out(err_msg, FILE_ERROR_OUT)
self.file_manager.delete_outfile(media_id + "/" +
local_filename)
else:
self.file_manager.log("Line data not enough info:" +
line_path)
self.file_manager.out(
"Line data not enough info:" + line_path,
FILE_ERROR_OUT)
self.file_manager.log("--Download END -- :" + input_file)
class PreparingSubtitlesHLS3:
file_manager = None
ftp_config = None
ftp = None
language_code_map = dict({"en": "en",
"id": "id",
"ko": "ko",
"ms": "ms", #Malay
"th": "th",
"vi": "vi",
"ar": "ar",
"my": "my", #Burmese,
"zz": "zz", #(Other)
"zh": "zh",
"zho": "zh",
"zh-Hans": "zh-Hans",
"zh-Hant": "zh-Hant",
"mis": "zz"})
def __init__(self, sub_folder):
self.file_manager = FileManager(SCRIPT_NAME, self.__class__.__name__, sub_folder)
self.ftp_config = FTPConfig(ROOT+"/resources/DownloadStorageFiles/ftp.ini", SCRIPT_NAME, sub_folder)
def run(self, input_file, input_directory):
self.file_manager.log("FUNCTION run() args -> input_file:" + input_file +
", input_directory:" + input_directory)
with open(input_file, "r") as f:
self.file_manager.log("OPEN input_file:" + input_file)
for line in f:
media_id = str(line).replace('\n', '')
self.file_manager.log("READLN media_id:" + media_id)
if media_id:
local_input_directory = input_directory + media_id + "/"
manifest_extension = "m3u8"
local_media_hls = local_input_directory + "manifest." + manifest_extension
remote_hls_storage_host = ""
remote_hls_basepath = ""
manifest_info_file = local_input_directory + manifest_extension + "_info.txt"
with open(manifest_info_file, "r") as hls_basepath_file:
self.file_manager.log("OPEN [" + media_id + "] manifest_info_file:" + manifest_info_file)
for hls_basepath_line in hls_basepath_file:
self.file_manager.log("READLN [" + media_id + "] hls_basepath_line:" + hls_basepath_line)
hls_basepath_line = str(hls_basepath_line).replace('\n', '')
remote_hls_storage_host = hls_basepath_line.split(",")[0]
remote_hls_basepath = hls_basepath_line.split(",")[1]
self.file_manager.log("STORE [" + media_id + "] " +
"remote_hls_storage_host:" + input_file +
", remote_hls_storage_host:" + input_directory)
count_match = 0
if os.path.exists(local_media_hls):
self.file_manager.log("CONTINUE [" + media_id + "] (exists) local_media_hls:" + local_media_hls)
with open(local_media_hls) as manifest_hls_file:
for manifest_hls_line in manifest_hls_file:
manifest_hls_line = str(manifest_hls_line).replace('\n', '')
#EXT-X-MEDIA:TYPE=SUBTITLES
# 27chars
if manifest_hls_line.find("EXT-X-MEDIA:TYPE=SUBTITLES") >= 0:
count_match += 1
cue_time = datetime.datetime(100,1,1,0,0,0)
cue_time_minute = -1
subtitle_language = self.getValue(manifest_hls_line, "LANGUAGE")
subtitle_segment_uri = self.getValue(manifest_hls_line, "URI")
subtitle_segment_uri_layers = str(subtitle_segment_uri).split("/")
if subtitle_language and subtitle_segment_uri and len(subtitle_segment_uri_layers) == 2:
self.file_manager.log("PROCESS [" + media_id + "]" +
" subtitle_language: " + subtitle_language +
", subtitle_segment_uri: " + subtitle_segment_uri)
subtitle_segment_dir = subtitle_segment_uri_layers[0]
subtitle_segment_file = subtitle_segment_uri_layers[1]
remote_vtt_basepath = remote_hls_basepath + subtitle_segment_dir + "/"
local_replace_basepath = "replace/" + media_id + "/"
local_rollback_basepath = "rollback/" + media_id + "/"
local_replace_sourcepath = local_replace_basepath + subtitle_language + "/"
local_rollback_sourcepath = local_rollback_basepath + subtitle_language + "/"
self.file_manager.validate_dir(self.file_manager.out_path, local_replace_sourcepath) # ensure folder exists
self.file_manager.validate_dir(self.file_manager.out_path, local_rollback_sourcepath) # ensure folder exists
local_replace_base_fullpath = self.file_manager.out_path + "/" + local_replace_basepath
local_rollback_base_fullpath = self.file_manager.out_path + "/" + local_rollback_basepath
local_replace_source_fullpath = self.file_manager.out_path + "/" + local_replace_sourcepath
local_rollback_source_fullpath = self.file_manager.out_path + "/" + local_rollback_sourcepath
# download m3u8 subtitle segments to local (save as language code)
remote_location = remote_hls_basepath + subtitle_segment_uri
local_replace_subtitle_segment = local_replace_source_fullpath + subtitle_segment_file
if self.download(remote_location, local_replace_subtitle_segment, remote_hls_storage_host):
self.file_manager.log("DOWNLOAD [" + media_id + "] Subtitle Segment " +
" remote_hls_storage_host: " + remote_hls_storage_host +
", subtitle_language: " + subtitle_language +
", remote_location: " + remote_location +
", local_replace_subtitle_segment: " + local_replace_subtitle_segment)
fixed_subtitle_input = local_input_directory + subtitle_language + ".vtt"
# parse and stored fixed time interval into array list
list_of_time_intervals = []
with open(fixed_subtitle_input) as fixed_subtitle_input_file:
for fixed_subtitle_input_line in fixed_subtitle_input_file:
if fixed_subtitle_input_line.find(" --> ") >= 0:
list_of_time_intervals.append(fixed_subtitle_input_line)
time_interval_index = 0
# find remote webvtt to local
with open(local_replace_subtitle_segment) as local_replace_subtitle_segment_file:
for local_replace_subtitle_segment_line in local_replace_subtitle_segment_file:
local_replace_subtitle_segment_line = str(local_replace_subtitle_segment_line).replace('\n', '')
if local_replace_subtitle_segment_line.find(".webvtt") >= 0:
subtitle_uri = local_replace_subtitle_segment_line
# download remote webvtt to local for rollback
remote_vtt_location = remote_vtt_basepath + subtitle_uri
local_rollback_vtt_sourcefile = local_rollback_source_fullpath + subtitle_uri
self.file_manager.log("PROCESS [" + media_id + "] Rollback preparation - START " +
" remote_vtt_location: " + remote_vtt_location +
", local_rollback_vtt_sourcefile: " + local_rollback_vtt_sourcefile)
if self.download(remote_vtt_location, local_rollback_vtt_sourcefile, remote_hls_storage_host):
self.file_manager.out(media_id + "," + local_rollback_vtt_sourcefile + "," +
remote_hls_storage_host + "," +
remote_vtt_location,
manifest_extension + "_rollback.txt")
else:
self.file_manager.log("FAILED [" + media_id + "]" +
" to download source vtt for backup" +
", subtitle_language: " + subtitle_language +
", remote_hls_storage_host: " + remote_hls_storage_host +
", remote_vtt_location: " + remote_vtt_location)
self.file_manager.log("PROCESS [" + media_id + "] Rollback preparation - END ")
if os.path.exists(local_rollback_vtt_sourcefile):
# preparing replace files
local_replace_vtt_sourcefile = local_replace_source_fullpath + subtitle_uri
self.file_manager.log("PROCESS [" + media_id + "] Replace preparation - START" +
", local_replace_vtt_sourcefile: " + local_replace_vtt_sourcefile)
with open(local_rollback_vtt_sourcefile) as local_rollback_vtt_file:
# write to local_replace_vtt_sourcefile
cue_time_minute += 1
for local_rollback_vtt_line in local_rollback_vtt_file:
line_entry = local_rollback_vtt_line
if local_rollback_vtt_line.find(" --> ") >= 0:
line_entry = list_of_time_intervals[time_interval_index]
time_interval_index += 1
if local_rollback_vtt_line.find("LOCAL:00:00:00.000") >= 0:
idt = cue_time + datetime.timedelta(minutes=cue_time_minute)
header_local = "LOCAL:" + idt.strftime("%H:%M:%S.000")
line_entry = local_rollback_vtt_line.replace("LOCAL:00:00:00.000", header_local)
line_entry = line_entry.replace("\n","")
self.file_manager.out(line_entry, local_replace_sourcepath + subtitle_uri)
self.file_manager.out(media_id + "," + local_replace_vtt_sourcefile + "," +
remote_hls_storage_host + "," +
remote_vtt_location,
manifest_extension + "_replace.txt")
self.file_manager.log("PROCESS [" + media_id + "] Replace preparation - END")
else:
self.file_manager.log("FAILED [" + media_id + "]" +
" to download subtitle segment for further processing" +
" remote_hls_storage_host: " + remote_hls_storage_host +
", remote_location: " + remote_location)
else:
self.file_manager.log("SKIP [" + media_id + "] Subtitle has empty values" +
" subtitle_language: " + subtitle_language +
", subtitle_segment_uri: " + subtitle_segment_uri)
else:
self.file_manager.log("FAILED [" + media_id + "]" +
" cannot find local HLS file URI:" + local_media_hls)
if count_match == 0:
self.file_manager.log("SKIP [" + media_id + "] Manifest doesn't have subtitles, mediaId: " + media_id)
self.file_manager.log("--END -- :" + input_file)
return True
def getValue(self, line, key):
if line:
arr = str(line).split(",")
for item in arr:
if item.startswith(key):
keyval = item.split("=")
if len(keyval) == 2:
return keyval[1].replace("\"", "")
return ""
def download(self, source_file, destination_file, profile_tag):
is_success = True
try:
if self.ftp_config.profile_tag != profile_tag:
if self.ftp is not None:
self.ftp.quit()
self.ftp_config.load(profile_tag) # load storage_host as profile_tag
self.ftp = ftplib.FTP(self.ftp_config.host)
self.ftp.login(self.ftp_config.username, self.ftp_config.password)
# local_filepath = self.file_manager.get_outfile(destination_file)
local_file = open(destination_file, 'wb')
self.file_manager.log("Download: " + source_file + " >> " +
self.ftp.retrbinary('RETR ' + source_file, local_file.write))
local_file.close()
except ftplib.error_perm:
err_msg = "ERROR FTP file does not exist, filepath:" + source_file
self.file_manager.log(err_msg)
self.file_manager.out(err_msg, FILE_ERROR_OUT)
self.file_manager.delete_outfile(destination_file)
is_success = False
except ftplib.all_errors:
err_msg = "ERROR FTP all errors, filepath:" + source_file
self.file_manager.log(err_msg)
self.file_manager.out(err_msg, FILE_ERROR_OUT)
self.file_manager.delete_outfile(destination_file)
is_success = False
return is_success
class TransferSubtitles:
file_manager = None
ftp_config = None
ftp = None
def __init__(self):
self.file_manager = FileManager(SCRIPT_NAME, self.__class__.__name__,
UNIQUE_RUN_ID)
self.ftp_config = FTPConfig(
ROOT + "/resources/DownloadStorageFiles/ftp.ini", SCRIPT_NAME,
UNIQUE_RUN_ID)
def run(self, input_file, tag_id):
self.file_manager.log("SCRIPT RUN() START ------ args -> input_file:" +
input_file)
with open(input_file, "r") as f:
self.file_manager.log("OPEN input_file:" + input_file)
for line in f:
clean_line = str(line).replace('\n', '')
self.file_manager.log("READLN clean_line:" + clean_line)
line_items = str(clean_line).split(",")
if len(line_items) == 4:
media_id = line_items[0]
source_url = line_items[1]
destination_host = line_items[2]
destination_url = line_items[3]
if self.upload(source_url, destination_url,
destination_host):
self.file_manager.log("UPLOAD [" + media_id +
"] Success")
self.file_manager.out(media_id,
"success_ids_" + tag_id + ".txt")
self.file_manager.out(clean_line,
"success_" + tag_id + ".txt")
http_url = self.ftp_config.http_url + str(
destination_url).replace(
self.ftp_config.replace_path, "")
self.file_manager.log("INVALIDATE CACHE PARAM [" +
media_id + "] URL=" + http_url)
http_url = urllib.parse.quote_plus(http_url)
r = requests.get(
'https://api.asia.fox.com/akamai/purge?url=' +
http_url)
self.file_manager.log("INVALIDATE CACHE RESPONSE [" +
media_id + "] URL=" + r.text)
else:
self.file_manager.log("UPLOAD [" + media_id +
"] Failed")
self.file_manager.out(clean_line,
"failed_" + tag_id + ".txt")
else:
self.file_manager.log("PARSE [" + media_id + "] Failed")
self.file_manager.out(clean_line,
"failed_" + tag_id + ".txt")
self.file_manager.log("SCRIPT RUN() END ------")
return True
def upload(self, source_file, destination_file, profile_tag):
is_success = True
try:
if self.ftp_config.profile_tag != profile_tag:
if self.ftp is not None:
self.ftp.quit()
self.ftp_config.load(
profile_tag) # load storage_host as profile_tag
self.ftp = ftplib.FTP(self.ftp_config.host)
self.ftp.login(self.ftp_config.username,
self.ftp_config.password)
# local_filepath = self.file_manager.get_outfile(destination_file)
local_file = open(source_file, 'rb')
self.file_manager.log(
"Upload: " + source_file + " >> " +
self.ftp.storbinary('STOR ' + destination_file, local_file))
local_file.close()
except ftplib.error_perm:
self.file_manager.log("ftplib.error_perm destination_file:" +
destination_file)
is_success = False
except ftplib.all_errors:
self.file_manager.log("ftplib.all_errors destination_file:" +
destination_file)
is_success = False
return is_success