def main(argv):
src = None
dst = None
k_size = None
args = parser.parse_args(argv[1:])
if len(argv) <= 1:
parser.parse_args(['--help'])
return
## get kmer size
#if args.ksize:
# k_size = args.ksize
# get source dir
if args.src:
src = args.src
if not os.path.exists(src):
raise FileNotFoundError('{} does not exist.'.format(src))
# set target dir
# get destination dir
if args.dst:
dst = args.dst
else:
dst = src + '../0_arffs/' # default
print('version:', settings.DEV_VERSION)
print("source directory:", src)
print("destination directory:", dst)
#
# Rename and aggregating to the destination dir
#
all_dirs_info = FvManager.get_feature_dirs(src)
all_dirs_info.sort()
for dir in all_dirs_info:
version_info = VersionManager(version_str=dir)
src_arffs_dir = src + dir + '/0/1/'
files_list = FileList.ls(path=src_arffs_dir,
recursion=False,
mode=settings.FLS_FILE_ONLY,
ext=settings.FLS_EXT_ARFF)
for file in files_list:
if file.get_ext().lower() == settings.FLS_EXT_ARFF:
#print(file.full)
new_file = dst + file.name
FileManager.dir_create(dst)
copyfile(src=file.full, dst=new_file)
print(new_file)
class FTPConfig:
file_manager = None
host = ''
username = ''
password = ''
replace_path = ''
http_url = ''
config = ConfigParser()
profile_tag = ''
def __init__(self, config_file, script_name='', unique_run_id=''):
self.file_manager = FileManager(script_name, self.__class__.__name__,
unique_run_id)
self.config.read(config_file)
def load(self, profile_tag):
self.host = ""
self.username = ""
self.password = ""
self.replace_path = ""
self.http_url = ""
self.profile_tag = profile_tag
is_loaded = False
if profile_tag in self.config:
self.host = self.config.get(profile_tag, 'ftpHost', fallback='')
self.username = self.config.get(profile_tag,
'ftpUser',
fallback='')
self.password = self.config.get(profile_tag,
'ftpPass',
fallback='')
self.replace_path = self.config.get(profile_tag,
'ftpReplace',
fallback='')
self.http_url = self.config.get(profile_tag,
'httpUrl',
fallback='')
is_loaded = True
else:
self.file_manager.log("Unknown profile: " + profile_tag +
" ; Please check ftp.ini.")
self.file_manager.out("Unknown ftp profile: " + profile_tag,
"FTPConfig.error.txt")
return is_loaded
class WebClient:
fm = FileManager("", "WebClient", "WebClient")
def __init__(self, service_object_class, token, account_uri):
self.service_object_class = service_object_class
self.base_url = service_object_class.endpoint_uri
self.preset_params = service_object_class.preset_params
self.preset_params["token"] = token
self.preset_params["account"] = account_uri
self.preset_params["form"] = "json"
def get_parameters(self):
try:
if self.parameters is None:
self.queries({})
except:
self.queries({})
return self.parameters
def queries(self, in_dict):
self.parameters = ""
for key in self.preset_params.keys():
value = parse.quote(str(self.preset_params[key]), safe='')
self.parameters += key + "=" + value + "&"
for key in in_dict.keys():
value = parse.quote(str(in_dict[key]), safe='')
self.parameters += key + "=" + value + "&"
def request(self, data_arr):
rsp = False
request_url = "http://" + self.base_url + "?" + self.get_parameters()
self.fm.log_info("REQUEST url={} data={}".format(
request_url, data_arr))
# request_data = json.dumps({
# "entries": data_arr
# }).encode()
request_data = json.dumps(data_arr).encode()
req = request.Request(request_url, data=request_data)
req.add_header('Content-Type', 'application/json')
response_data = request.urlopen(req).read()
self.fm.log_info("RESPONSE data={}".format(response_data))
if not response_data is None:
# TODO need more handling here
response_json = json.loads(response_data)
try:
if not response_json.get("isException"):
print("PUBLISH is Success")
rsp = True
else:
print("PUBLISH is Exception")
except:
rsp = False
return rsp
def __init__(self):
self.file_manager = FileManager(SCRIPT_NAME, self.__class__.__name__, UNIQUE_RUN_ID)
self.ftp_config = FTPConfig(ROOT+"/resources/DownloadStorageFiles/ftp.ini", SCRIPT_NAME, UNIQUE_RUN_ID)
class PreparingSubtitlesDASH:
file_manager = None
ftp_config = None
ftp = None
language_code_map = dict({"en": "en",
"id": "id",
"ko": "ko",
"ms": "ms", #Malay
"th": "th",
"vi": "vi",
"ar": "ar",
"my": "my", #Burmese,
"zz": "zz", #(Other)
"zh": "zh",
"zho": "zh",
"zh-Hans": "zh-Hans",
"zh-Hant": "zh-Hant",
"mis": "zz"})
def __init__(self):
self.file_manager = FileManager(SCRIPT_NAME, self.__class__.__name__, UNIQUE_RUN_ID)
self.ftp_config = FTPConfig(ROOT+"/resources/DownloadStorageFiles/ftp.ini", SCRIPT_NAME, UNIQUE_RUN_ID)
def run(self, input_file, input_directory):
self.file_manager.log("FUNCTION run() args -> input_file:" + input_file +
", input_directory:" + input_directory)
with open(input_file, "r") as f:
self.file_manager.log("OPEN input_file:" + input_file)
for line in f:
media_id = str(line).replace('\n', '')
self.file_manager.log("READLN media_id:" + media_id)
if media_id:
local_input_directory = input_directory + media_id + "/"
manifest_extension = "mpd"
local_media_dash = local_input_directory + "manifest." + manifest_extension
# replace_list = local_media_directory + manifest_extension + "_replace.txt"
# rollback_list = local_media_directory + manifest_extension + "_rollback.txt"
remote_dash_storage_host = ""
remote_dash_basepath = ""
manifest_info_file = local_input_directory + manifest_extension + "_info.txt"
with open(manifest_info_file, "r") as dash_basepath_file:
self.file_manager.log("OPEN [" + media_id + "] manifest_info_file:" + manifest_info_file)
for dash_basepath_line in dash_basepath_file:
self.file_manager.log("READLN [" + media_id + "] dash_basepath_line:" + dash_basepath_line)
dash_basepath_line = str(dash_basepath_line).replace('\n', '')
remote_dash_storage_host = dash_basepath_line.split(",")[0]
remote_dash_basepath = dash_basepath_line.split(",")[1]
self.file_manager.log("STORE [" + media_id + "] " +
"remote_dash_storage_host:" + input_file +
", remote_dash_storage_host:" + input_directory)
count_match = 0
if os.path.exists(local_media_dash):
self.file_manager.log("CONTINUE [" + media_id + "] (exists) local_media_dash:" + local_media_dash)
manifest_dom = minidom.parse(local_media_dash)
for docAdaptationSet in manifest_dom.getElementsByTagName('AdaptationSet'):
if docAdaptationSet.attributes['contentType']:
if docAdaptationSet.attributes['contentType'].value == 'text':
count_match += 1
self.file_manager.log("AdaptationSet contentType is text")
subtitle_language = docAdaptationSet.attributes['lang'].value #lang="zh-Hans"
docRepresentation = self.getNodeByName(docAdaptationSet, "Representation")
docBaseURL = self.getNodeByName(docRepresentation, "BaseURL")
subtitle_uri = ""
if docBaseURL:
subtitle_uri = self.getNodeText(docBaseURL)
if subtitle_language and subtitle_uri:
self.file_manager.log("PROCESS [" + media_id + "]" +
" subtitle_language: " + subtitle_language +
", subtitle_uri: " + subtitle_uri)
local_replace_sourcepath = "replace/" + media_id + "/"
local_rollback_sourcepath = "rollback/" + media_id + "/"
self.file_manager.validate_dir(self.file_manager.out_path, local_replace_sourcepath) # ensure folder exists
self.file_manager.validate_dir(self.file_manager.out_path, local_rollback_sourcepath) # ensure folder exists
local_replace_sourcepath = self.file_manager.out_path + "/" + local_replace_sourcepath
local_rollback_sourcepath = self.file_manager.out_path + "/" + local_rollback_sourcepath
# Copy vtt from DownloadStorageFiles and rename
fixed_subtitle_input = local_input_directory + subtitle_language + ".vtt"
local_replace_sourcefile = local_replace_sourcepath + subtitle_uri
self.file_manager.log("COPY START [" + media_id + "] Replace preparation " +
" fixed_subtitle_input: " + fixed_subtitle_input +
", local_replace_sourcefile: " + local_replace_sourcefile)
copyfile(fixed_subtitle_input, local_replace_sourcefile)
self.file_manager.log("COPY END [" + media_id + "] Replace preparation " +
" fixed_subtitle_input: " + fixed_subtitle_input +
", local_replace_sourcefile: " + local_replace_sourcefile)
# Download remote vtt file
remote_location = remote_dash_basepath + subtitle_uri
local_rollback_sourcefile = local_rollback_sourcepath + subtitle_uri
self.file_manager.log("COPY START [" + media_id + "] Rollback preparation " +
" remote_location: " + remote_location +
", local_rollback_sourcefile: " + local_rollback_sourcefile)
if self.download(remote_location, local_rollback_sourcefile, remote_dash_storage_host):
self.file_manager.out(media_id + "," + local_replace_sourcefile + "," +
remote_dash_storage_host + "," +
remote_location,
manifest_extension + "_replace.txt")
self.file_manager.out(media_id + "," + local_rollback_sourcefile + "," +
remote_dash_storage_host + "," +
remote_location,
manifest_extension + "_rollback.txt")
else:
self.file_manager.log("FAILED [" + media_id + "]" +
" to download source vtt for backup")
self.file_manager.log("COPY END [" + media_id + "] Rollback preparation " +
" remote_location: " + remote_location +
", local_rollback_sourcefile: " + local_rollback_sourcefile)
else:
self.file_manager.log("SKIP [" + media_id + "] Subtitle has empty values" +
" subtitle_language: " + subtitle_language +
", subtitle_uri: " + subtitle_uri)
else:
self.file_manager.log("SKIP [" + media_id + "] (do not exist) local_media_dash:" + local_media_dash)
if count_match == 0:
self.file_manager.log("SKIP [" + media_id + "] Manifest doesn't have subtitles, mediaId: " + media_id)
return True
def getNodeByName(self, nodes, name):
if nodes:
for node in nodes.childNodes:
if node.nodeName == name:
return node
return None
def getNodeText(self, nodes):
if nodes:
for node in nodes.childNodes:
if node.wholeText:
return node.wholeText
return ""
def download(self, source_file, destination_file, profile_tag):
is_success = True
try:
if self.ftp_config.profile_tag != profile_tag:
if self.ftp is not None:
self.ftp.quit()
self.ftp_config.load(profile_tag) # load storage_host as profile_tag
self.ftp = ftplib.FTP(self.ftp_config.host)
self.ftp.login(self.ftp_config.username, self.ftp_config.password)
# local_filepath = self.file_manager.get_outfile(destination_file)
local_file = open(destination_file, 'wb')
self.file_manager.log("Download: " + source_file + " >> " +
self.ftp.retrbinary('RETR ' + source_file, local_file.write))
local_file.close()
except ftplib.error_perm:
err_msg = "ERROR FTP file does not exist, filepath:" + source_file
self.file_manager.log(err_msg)
self.file_manager.out(err_msg, FILE_ERROR_OUT)
self.file_manager.delete_outfile(destination_file)
is_success = False
except ftplib.all_errors:
err_msg = "ERROR FTP all errors, filepath:" + source_file
self.file_manager.log(err_msg)
self.file_manager.out(err_msg, FILE_ERROR_OUT)
self.file_manager.delete_outfile(destination_file)
is_success = False
return is_success
def __init__(self, config_file, script_name='', unique_run_id=''):
self.file_manager = FileManager(script_name, self.__class__.__name__,
unique_run_id)
self.config.read(config_file)
def __init__(self, sub_folder):
self.file_manager = FileManager(SCRIPT_NAME, self.__class__.__name__, sub_folder)
self.ftp_config = FTPConfig(ROOT+"/resources/DownloadStorageFiles/ftp.ini", SCRIPT_NAME, sub_folder)
class PreparingSubtitlesHLS3:
file_manager = None
ftp_config = None
ftp = None
language_code_map = dict({"en": "en",
"id": "id",
"ko": "ko",
"ms": "ms", #Malay
"th": "th",
"vi": "vi",
"ar": "ar",
"my": "my", #Burmese,
"zz": "zz", #(Other)
"zh": "zh",
"zho": "zh",
"zh-Hans": "zh-Hans",
"zh-Hant": "zh-Hant",
"mis": "zz"})
def __init__(self, sub_folder):
self.file_manager = FileManager(SCRIPT_NAME, self.__class__.__name__, sub_folder)
self.ftp_config = FTPConfig(ROOT+"/resources/DownloadStorageFiles/ftp.ini", SCRIPT_NAME, sub_folder)
def run(self, input_file, input_directory):
self.file_manager.log("FUNCTION run() args -> input_file:" + input_file +
", input_directory:" + input_directory)
with open(input_file, "r") as f:
self.file_manager.log("OPEN input_file:" + input_file)
for line in f:
media_id = str(line).replace('\n', '')
self.file_manager.log("READLN media_id:" + media_id)
if media_id:
local_input_directory = input_directory + media_id + "/"
manifest_extension = "m3u8"
local_media_hls = local_input_directory + "manifest." + manifest_extension
remote_hls_storage_host = ""
remote_hls_basepath = ""
manifest_info_file = local_input_directory + manifest_extension + "_info.txt"
with open(manifest_info_file, "r") as hls_basepath_file:
self.file_manager.log("OPEN [" + media_id + "] manifest_info_file:" + manifest_info_file)
for hls_basepath_line in hls_basepath_file:
self.file_manager.log("READLN [" + media_id + "] hls_basepath_line:" + hls_basepath_line)
hls_basepath_line = str(hls_basepath_line).replace('\n', '')
remote_hls_storage_host = hls_basepath_line.split(",")[0]
remote_hls_basepath = hls_basepath_line.split(",")[1]
self.file_manager.log("STORE [" + media_id + "] " +
"remote_hls_storage_host:" + input_file +
", remote_hls_storage_host:" + input_directory)
count_match = 0
if os.path.exists(local_media_hls):
self.file_manager.log("CONTINUE [" + media_id + "] (exists) local_media_hls:" + local_media_hls)
with open(local_media_hls) as manifest_hls_file:
for manifest_hls_line in manifest_hls_file:
manifest_hls_line = str(manifest_hls_line).replace('\n', '')
#EXT-X-MEDIA:TYPE=SUBTITLES
# 27chars
if manifest_hls_line.find("EXT-X-MEDIA:TYPE=SUBTITLES") >= 0:
count_match += 1
cue_time = datetime.datetime(100,1,1,0,0,0)
cue_time_minute = -1
subtitle_language = self.getValue(manifest_hls_line, "LANGUAGE")
subtitle_segment_uri = self.getValue(manifest_hls_line, "URI")
subtitle_segment_uri_layers = str(subtitle_segment_uri).split("/")
if subtitle_language and subtitle_segment_uri and len(subtitle_segment_uri_layers) == 2:
self.file_manager.log("PROCESS [" + media_id + "]" +
" subtitle_language: " + subtitle_language +
", subtitle_segment_uri: " + subtitle_segment_uri)
subtitle_segment_dir = subtitle_segment_uri_layers[0]
subtitle_segment_file = subtitle_segment_uri_layers[1]
remote_vtt_basepath = remote_hls_basepath + subtitle_segment_dir + "/"
local_replace_basepath = "replace/" + media_id + "/"
local_rollback_basepath = "rollback/" + media_id + "/"
local_replace_sourcepath = local_replace_basepath + subtitle_language + "/"
local_rollback_sourcepath = local_rollback_basepath + subtitle_language + "/"
self.file_manager.validate_dir(self.file_manager.out_path, local_replace_sourcepath) # ensure folder exists
self.file_manager.validate_dir(self.file_manager.out_path, local_rollback_sourcepath) # ensure folder exists
local_replace_base_fullpath = self.file_manager.out_path + "/" + local_replace_basepath
local_rollback_base_fullpath = self.file_manager.out_path + "/" + local_rollback_basepath
local_replace_source_fullpath = self.file_manager.out_path + "/" + local_replace_sourcepath
local_rollback_source_fullpath = self.file_manager.out_path + "/" + local_rollback_sourcepath
# download m3u8 subtitle segments to local (save as language code)
remote_location = remote_hls_basepath + subtitle_segment_uri
local_replace_subtitle_segment = local_replace_source_fullpath + subtitle_segment_file
if self.download(remote_location, local_replace_subtitle_segment, remote_hls_storage_host):
self.file_manager.log("DOWNLOAD [" + media_id + "] Subtitle Segment " +
" remote_hls_storage_host: " + remote_hls_storage_host +
", subtitle_language: " + subtitle_language +
", remote_location: " + remote_location +
", local_replace_subtitle_segment: " + local_replace_subtitle_segment)
fixed_subtitle_input = local_input_directory + subtitle_language + ".vtt"
# parse and stored fixed time interval into array list
list_of_time_intervals = []
with open(fixed_subtitle_input) as fixed_subtitle_input_file:
for fixed_subtitle_input_line in fixed_subtitle_input_file:
if fixed_subtitle_input_line.find(" --> ") >= 0:
list_of_time_intervals.append(fixed_subtitle_input_line)
time_interval_index = 0
# find remote webvtt to local
with open(local_replace_subtitle_segment) as local_replace_subtitle_segment_file:
for local_replace_subtitle_segment_line in local_replace_subtitle_segment_file:
local_replace_subtitle_segment_line = str(local_replace_subtitle_segment_line).replace('\n', '')
if local_replace_subtitle_segment_line.find(".webvtt") >= 0:
subtitle_uri = local_replace_subtitle_segment_line
# download remote webvtt to local for rollback
remote_vtt_location = remote_vtt_basepath + subtitle_uri
local_rollback_vtt_sourcefile = local_rollback_source_fullpath + subtitle_uri
self.file_manager.log("PROCESS [" + media_id + "] Rollback preparation - START " +
" remote_vtt_location: " + remote_vtt_location +
", local_rollback_vtt_sourcefile: " + local_rollback_vtt_sourcefile)
if self.download(remote_vtt_location, local_rollback_vtt_sourcefile, remote_hls_storage_host):
self.file_manager.out(media_id + "," + local_rollback_vtt_sourcefile + "," +
remote_hls_storage_host + "," +
remote_vtt_location,
manifest_extension + "_rollback.txt")
else:
self.file_manager.log("FAILED [" + media_id + "]" +
" to download source vtt for backup" +
", subtitle_language: " + subtitle_language +
", remote_hls_storage_host: " + remote_hls_storage_host +
", remote_vtt_location: " + remote_vtt_location)
self.file_manager.log("PROCESS [" + media_id + "] Rollback preparation - END ")
if os.path.exists(local_rollback_vtt_sourcefile):
# preparing replace files
local_replace_vtt_sourcefile = local_replace_source_fullpath + subtitle_uri
self.file_manager.log("PROCESS [" + media_id + "] Replace preparation - START" +
", local_replace_vtt_sourcefile: " + local_replace_vtt_sourcefile)
with open(local_rollback_vtt_sourcefile) as local_rollback_vtt_file:
# write to local_replace_vtt_sourcefile
cue_time_minute += 1
for local_rollback_vtt_line in local_rollback_vtt_file:
line_entry = local_rollback_vtt_line
if local_rollback_vtt_line.find(" --> ") >= 0:
line_entry = list_of_time_intervals[time_interval_index]
time_interval_index += 1
if local_rollback_vtt_line.find("LOCAL:00:00:00.000") >= 0:
idt = cue_time + datetime.timedelta(minutes=cue_time_minute)
header_local = "LOCAL:" + idt.strftime("%H:%M:%S.000")
line_entry = local_rollback_vtt_line.replace("LOCAL:00:00:00.000", header_local)
line_entry = line_entry.replace("\n","")
self.file_manager.out(line_entry, local_replace_sourcepath + subtitle_uri)
self.file_manager.out(media_id + "," + local_replace_vtt_sourcefile + "," +
remote_hls_storage_host + "," +
remote_vtt_location,
manifest_extension + "_replace.txt")
self.file_manager.log("PROCESS [" + media_id + "] Replace preparation - END")
else:
self.file_manager.log("FAILED [" + media_id + "]" +
" to download subtitle segment for further processing" +
" remote_hls_storage_host: " + remote_hls_storage_host +
", remote_location: " + remote_location)
else:
self.file_manager.log("SKIP [" + media_id + "] Subtitle has empty values" +
" subtitle_language: " + subtitle_language +
", subtitle_segment_uri: " + subtitle_segment_uri)
else:
self.file_manager.log("FAILED [" + media_id + "]" +
" cannot find local HLS file URI:" + local_media_hls)
if count_match == 0:
self.file_manager.log("SKIP [" + media_id + "] Manifest doesn't have subtitles, mediaId: " + media_id)
self.file_manager.log("--END -- :" + input_file)
return True
def getValue(self, line, key):
if line:
arr = str(line).split(",")
for item in arr:
if item.startswith(key):
keyval = item.split("=")
if len(keyval) == 2:
return keyval[1].replace("\"", "")
return ""
def download(self, source_file, destination_file, profile_tag):
is_success = True
try:
if self.ftp_config.profile_tag != profile_tag:
if self.ftp is not None:
self.ftp.quit()
self.ftp_config.load(profile_tag) # load storage_host as profile_tag
self.ftp = ftplib.FTP(self.ftp_config.host)
self.ftp.login(self.ftp_config.username, self.ftp_config.password)
# local_filepath = self.file_manager.get_outfile(destination_file)
local_file = open(destination_file, 'wb')
self.file_manager.log("Download: " + source_file + " >> " +
self.ftp.retrbinary('RETR ' + source_file, local_file.write))
local_file.close()
except ftplib.error_perm:
err_msg = "ERROR FTP file does not exist, filepath:" + source_file
self.file_manager.log(err_msg)
self.file_manager.out(err_msg, FILE_ERROR_OUT)
self.file_manager.delete_outfile(destination_file)
is_success = False
except ftplib.all_errors:
err_msg = "ERROR FTP all errors, filepath:" + source_file
self.file_manager.log(err_msg)
self.file_manager.out(err_msg, FILE_ERROR_OUT)
self.file_manager.delete_outfile(destination_file)
is_success = False
return is_success
class DownloadStorageFiles1:
file_manager = None
ftp_config = None
ftp = None
def __init__(self):
self.file_manager = FileManager(SCRIPT_NAME, self.__class__.__name__,
UNIQUE_RUN_ID)
self.ftp_config = FTPConfig(
ROOT + "/resources/DownloadStorageFiles/ftp.ini", SCRIPT_NAME,
UNIQUE_RUN_ID)
def download(self, input_file):
self.file_manager.log("--Download START -- :" + input_file)
with open(input_file, "r") as f:
for line in f:
line_path = str(line).replace('\n', '')
data = str(line_path).split(',')
if len(data) >= 3:
media_id = data[0]
self.file_manager.validate_dir(self.file_manager.out_path,
media_id +
"/") # ensure folder exists
storage_filehost = data[1]
storage_filepath = data[2]
source_filename = storage_filepath.split('/')[-1]
source_basepath = storage_filepath.replace(
source_filename, '')
# file_name = source_filename.split('.')[0]
file_extension = source_filename.split('.')[-1]
# local_filename = source_filename
local_filename = "manifest." + file_extension
if len(data) == 4:
local_filename = data[3] + "." + file_extension
else:
self.file_manager.out(
storage_filehost + "," + source_basepath,
media_id + "/" + file_extension + "_info.txt",
True)
try:
if self.ftp_config.profile_tag != storage_filehost:
if self.ftp is not None:
self.ftp.quit()
self.ftp_config.load(
storage_filehost
) # load storage_host as profile_tag
self.ftp = ftplib.FTP(self.ftp_config.host)
self.ftp.login(self.ftp_config.username,
self.ftp_config.password)
local_filepath = self.file_manager.get_outfile(
media_id + "/" + local_filename)
local_file = open(local_filepath, 'wb')
self.file_manager.log(
"Download: " + storage_filepath +
" >> " + self.ftp.retrbinary(
'RETR ' + storage_filepath, local_file.write))
local_file.close()
except ftplib.error_perm:
err_msg = "FTP file does not exist,mediaId:" + media_id + ",filepath:" + storage_filepath
self.file_manager.log(err_msg)
self.file_manager.out(err_msg, FILE_ERROR_OUT)
self.file_manager.delete_outfile(media_id + "/" +
local_filename)
except ftplib.all_errors:
err_msg = "FTP all errors,mediaId:" + media_id
self.file_manager.log(err_msg)
self.file_manager.out(err_msg, FILE_ERROR_OUT)
self.file_manager.delete_outfile(media_id + "/" +
local_filename)
else:
self.file_manager.log("Line data not enough info:" +
line_path)
self.file_manager.out(
"Line data not enough info:" + line_path,
FILE_ERROR_OUT)
self.file_manager.log("--Download END -- :" + input_file)
class DataServiceClient:
fm = FileManager("", "DataServiceClient", "DataServiceClient")
def __init__(self, data_object_class, token, account_uri):
self.data_object_class = data_object_class
self.base_url = data_object_class.endpoint_uri
self.data = {
"totalResults": 0,
"startIndex": 0,
"itemsPerPage": 0,
"entryCount": 0,
"entries": []
}
self.preset_params = data_object_class.preset_params
self.preset_params["token"] = token
self.preset_params["account"] = account_uri
self.preset_params["count"] = "true"
self.preset_params["form"] = "cjson"
def get_data(self):
return self.data
def get_total(self):
return self.data.get("totalResults", 0)
def get_entries(self):
return self.data.get("entries", [])
def get_size(self):
return self.data.get("entryCount", 0)
def get_parameters(self):
return self.parameters
def queries(self, in_dict):
self.parameters = ""
for key in self.preset_params.keys():
value = parse.quote(str(self.preset_params[key]), safe='')
self.parameters += key + "=" + value + "&"
for key in in_dict.keys():
value = parse.quote(str(in_dict[key]), safe='')
self.parameters += key + "=" + value + "&"
def request_get(self):
rsp = False
if self.parameters is None:
self.queries({})
self.data["entries"] = []
request_url = "http://" + self.base_url + "?" + self.parameters
self.fm.log_info("REQUEST url={}".format(request_url))
response_data = request.urlopen(request_url).read()
self.fm.log_info("RESPONSE data={}".format(response_data))
if not response_data is None:
response_json = json.loads(response_data)
if response_json is not None and response_json.get(
"entryCount") is not None:
self.data["totalResults"] = response_json.get(
"totalResults", 0)
self.data["startIndex"] = response_json.get("startIndex", 0)
self.data["itemsPerPage"] = response_json.get(
"itemsPerPage", 0)
self.data["entryCount"] = response_json.get("entryCount", 0)
for entry in response_json.get("entries", []):
item = self.data_object_class(entry)
self.data["entries"].append(item)
rsp = True
else:
self.fm.log_error("RESPONSE data={}".format(response_data))
return rsp
def request_post(self, data_arr):
rsp = False
if self.parameters is None:
self.queries({})
request_url = "http://" + self.base_url + "/feed?" + self.parameters
self.fm.log_info("REQUEST url={} data={}".format(
request_url, data_arr))
request_data = json.dumps({"entries": data_arr}).encode()
req = request.Request(request_url, data=request_data)
req.add_header('Content-Type', 'application/json')
response_data = request.urlopen(req).read()
self.fm.log_info("RESPONSE data={}".format(response_data))
if not response_data is None:
response_json = json.loads(response_data)
if response_json is not None and response_json.get(
"entryCount") is not None:
rsp = True
return rsp
class TransferSubtitles:
file_manager = None
ftp_config = None
ftp = None
def __init__(self):
self.file_manager = FileManager(SCRIPT_NAME, self.__class__.__name__,
UNIQUE_RUN_ID)
self.ftp_config = FTPConfig(
ROOT + "/resources/DownloadStorageFiles/ftp.ini", SCRIPT_NAME,
UNIQUE_RUN_ID)
def run(self, input_file, tag_id):
self.file_manager.log("SCRIPT RUN() START ------ args -> input_file:" +
input_file)
with open(input_file, "r") as f:
self.file_manager.log("OPEN input_file:" + input_file)
for line in f:
clean_line = str(line).replace('\n', '')
self.file_manager.log("READLN clean_line:" + clean_line)
line_items = str(clean_line).split(",")
if len(line_items) == 4:
media_id = line_items[0]
source_url = line_items[1]
destination_host = line_items[2]
destination_url = line_items[3]
if self.upload(source_url, destination_url,
destination_host):
self.file_manager.log("UPLOAD [" + media_id +
"] Success")
self.file_manager.out(media_id,
"success_ids_" + tag_id + ".txt")
self.file_manager.out(clean_line,
"success_" + tag_id + ".txt")
http_url = self.ftp_config.http_url + str(
destination_url).replace(
self.ftp_config.replace_path, "")
self.file_manager.log("INVALIDATE CACHE PARAM [" +
media_id + "] URL=" + http_url)
http_url = urllib.parse.quote_plus(http_url)
r = requests.get(
'https://api.asia.fox.com/akamai/purge?url=' +
http_url)
self.file_manager.log("INVALIDATE CACHE RESPONSE [" +
media_id + "] URL=" + r.text)
else:
self.file_manager.log("UPLOAD [" + media_id +
"] Failed")
self.file_manager.out(clean_line,
"failed_" + tag_id + ".txt")
else:
self.file_manager.log("PARSE [" + media_id + "] Failed")
self.file_manager.out(clean_line,
"failed_" + tag_id + ".txt")
self.file_manager.log("SCRIPT RUN() END ------")
return True
def upload(self, source_file, destination_file, profile_tag):
is_success = True
try:
if self.ftp_config.profile_tag != profile_tag:
if self.ftp is not None:
self.ftp.quit()
self.ftp_config.load(
profile_tag) # load storage_host as profile_tag
self.ftp = ftplib.FTP(self.ftp_config.host)
self.ftp.login(self.ftp_config.username,
self.ftp_config.password)
# local_filepath = self.file_manager.get_outfile(destination_file)
local_file = open(source_file, 'rb')
self.file_manager.log(
"Upload: " + source_file + " >> " +
self.ftp.storbinary('STOR ' + destination_file, local_file))
local_file.close()
except ftplib.error_perm:
self.file_manager.log("ftplib.error_perm destination_file:" +
destination_file)
is_success = False
except ftplib.all_errors:
self.file_manager.log("ftplib.all_errors destination_file:" +
destination_file)
is_success = False
return is_success
def get_dictionary(filepath):
Cracker.__dictionary = FileManager.get_file_lines(filepath)
for item_key in resp['item']:
item_list['item'][item_id][item_key] = resp['item'][item_key]
if 'bonuses' in resp:
for item_key in resp['bonuses']:
if item_key == 'slot' or item_key == 'attack_speed':
item_list['item'][item_id][item_key] = resp['bonuses'][item_key]
else:
if 'stats' not in item_list['item'][item_id]:
item_list['item'][item_id]['stats'] = {}
item_list['item'][item_id]['stats'][item_key] = resp['bonuses'][item_key]
item_list['item'][item_id]['wiki_mapped'] = True
_ItemIterator = ItemIterator()
_FileManager = FileManager()
_WikiParser = WikiParser()
# Load item list from file.
item_list = _FileManager.load_item_data(FileManager.ITEM_DATA_FILE)
def load_from_wiki(item):
response = _WikiParser.fetch_item(item['name'].replace(' ', '_'))
if response is None:
return {}
return _WikiParser.parse_response(response)
def populate_user_list_from_file(filepath):
for linux_password_string in FileManager.get_file_lines(filepath):
if linux_password_string.find(':$') > -1:
UserManager.create_user(linux_password_string)