def createSubDirectories(model_folder, local_path):
#sub_folders_513_dict = {}
model = ngCleanModelName(model_folder)
if not(path_manager.joinpath(local_path, model[0]).exists()):
createLocalFolders(local_path, path_manager.joinpath(local_path, model[0]))
if not(path_manager.joinpath(local_path, model[0], model[1]).exists()):
#sub_folders_513_dict[''.join(model)] = path_manager(createLocalFolders(path_manager.joinpath(local_path, model[0]), model[1]))
path = path_manager(createLocalFolders(path_manager.joinpath(local_path, model[0]), model[1]))
elif not(path_manager.joinpath(local_path, model[0], model[1]).exists()):
path = path_manager(createLocalFolders(path_manager.joinpath(local_path, model[0]), model[1]))
return path
def createLocalSubDirectories(sftp, remote_dir, local_path, MW_VERSION,
is_folder_zip):
current_path = []
remote_path = []
for dir in sftp.listdir_attr(remote_dir):
if dir.filename not in FOLDERS_TO_NOT_DOWNLOAD:
current_path.append(
path_manager(
localFolder.createLocalFolders(local_path, dir.filename)))
remote_path.append(
path_manager.joinpath(path_manager(remote_dir), dir.filename))
return downloadFiles(sftp, current_path, remote_path, MW_VERSION,
is_folder_zip)
def getLatestBuildNameFromServer(client, PATH_MW):
# Compare Modification time folder by folder
sftp = client.open_sftp()
try:
dir_MW = sftp.listdir_attr(PATH_MW.as_posix())
log.info(f'Getting latest modified folder...')
latest = 0
latest_folder = None
# Checking if a NET folder is there. Sometimes Build team put a folder with this name
for folder_attribute in sftp.listdir_attr(PATH_MW.as_posix()):
if ('NET' in str(folder_attribute.filename).split('_') and folder_attribute.st_mtime > latest):
dir_MW = path_manager.joinpath(path_manager.joinpath(PATH_MW, folder_attribute.filename))
log.info(f'NET folder available. Including in the path.')
for folder_attribute in sftp.listdir_attr(dir_MW.as_posix()):
if folder_attribute.st_mtime > latest:
latest = folder_attribute.st_mtime # Flag from Paramiko lib: Modification Time
latest_folder = folder_attribute.filename
log.info(f'Latest modified folder: {latest_folder}')
return path_manager(latest_folder)
except (EnvironmentError, IOError, OSError) as e:
print(e)
log.exception('An error occured to get latest build from server')
def createSubDirectories513(sftp, local_path, remote_path):
sub_folders_513_dict = {}
remote_list = []
for model_folder in sftp.listdir_attr(remote_path.as_posix()):
if (model_folder.filename in FOLDERS_TO_DOWNLOAD_513):
model = ngCleanModelName(model_folder.filename)
remote_list.append(
path_manager.joinpath(path_manager(remote_path),
model_folder.filename))
if not (''.join(model) in sub_folders_513_dict):
if (path_manager.joinpath(local_path, model[0]).exists()):
sub_folders_513_dict[''.join(model)] = path_manager(
localFolder.createLocalFolders(
path_manager.joinpath(local_path, model[0]),
model[1]))
else:
sub_folders_513_dict[''.join(model)] = path_manager(
localFolder.createLocalFolders(
localFolder.createLocalFolders(
local_path, model[0]), model[1]))
return sub_folders_513_dict, remote_list
from pathlib import Path as path_manager
import sys
import logging
debug = False # Used to avoid Traceback errors when Fails
MW_DICT = {
'v5.2.8': path_manager(r'Z:\01.Opentv5\00.Versions\NET\Release6.1'),
'v5.2.4': path_manager(r'Z:\01.Opentv5\00.Versions\NET\Release4.1'),
'v5.1.3': path_manager(r'Z:\01.Opentv5\00.Versions\NET\Release2.5')
}
##### Log environment ###############################################################################################################################################
log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
file_handler = logging.FileHandler('logs/folder_manager.log')
formatter = logging.Formatter(
'%(asctime)s: %(levelname)s : %(name)s : %(message)s')
file_handler.setFormatter(formatter)
log.addHandler(file_handler)
#####################################################################################################################################################################
def createBuildNameFolder(build_name, MW_VERSION):
LOCAL_PATH = MW_DICT.get(MW_VERSION.name)
if (LOCAL_PATH.exists()):
try:
print(f'Creating {build_name} folder')
path_manager.mkdir(path_manager.joinpath(LOCAL_PATH, build_name))
log.info(f'Creating {build_name} created')
return LOCAL_PATH / build_name
from pathlib import Path as path_manager
import sys
import logging
debug = True # Used to avoid Traceback errors when Fails
MW_DICT = {
'NET_R6': path_manager(r'Z:\01.Opentv5\00.Versions\NET\Release6.1'),
'NET_R4': path_manager(r'Z:\01.Opentv5\00.Versions\NET\Release4.1'),
'NET_R2': path_manager(r'Z:\01.Opentv5\00.Versions\NET\Release2.5')
}
##### Log environment ###############################################################################################################################################
log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
file_handler = logging.FileHandler('logs/folder_managerOS.log')
formatter = logging.Formatter('%(asctime)s: %(levelname)s : %(name)s : %(message)s')
file_handler.setFormatter(formatter)
log.addHandler(file_handler)
#####################################################################################################################################################################
def createBuildNameFolder(build_name, MW_VERSION):
LOCAL_PATH = MW_DICT.get(MW_VERSION)
if(LOCAL_PATH.exists()):
try:
print(f'Creating {build_name} folder')
path_manager.mkdir(path_manager.joinpath(LOCAL_PATH, build_name))
log.info(f'Creating {build_name} created')
return LOCAL_PATH / build_name
except (EnvironmentError, IOError, OSError) as e:
def getBuildNameFromDir(build_path):
# Server pattern: 'Build6.x.x.x_0123456/'
# After clean it: '6.x.x.x_0123456'
return path_manager(''.join(''.join(str(build_path).split('Build')).split('/')))
from downloadManager import startDownloadProcess as download from paramiko import SSHClient, AutoAddPolicy from pathlib import Path as path_manager import fileManager as localFile import folderManager as localFolder import logging ##### Connection ############################################################################################################################################### HOST = 'zrhftp.hq.k.grp' USER = '******' PWORD = '[email protected]' PORT = 22 ##### Variables ############################################################################################################################################### ROOT = path_manager('/nfs/OpentvOS') NET = path_manager('NET') MW_VERSION_513 = path_manager('v5.1.3') MW_VERSION_524 = path_manager('v5.2.4') MW_VERSION_528 = path_manager('v5.2.8') STB_MW_528_LIST = [ 'dgci362_unified_glibc_bc', 'dci738net_glibc_bc' ] MW_VERSION_LIST = [MW_VERSION_528, MW_VERSION_524, MW_VERSION_513] ##### Log ############################################################################################################################################### log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) file_handler = logging.FileHandler('logs/sftpManager.log') formatter = logging.Formatter( '%(asctime)s: %(levelname)s : %(name)s : %(message)s')
def startPathCreator(client, PATH_MW, STB_MODEL, local_build_folder,
MW_VERSION):
is_folder_zip = False #MW 5.2.4 could be zip or not
sftp = client.open_sftp()
if (STB_MODEL):
build_path = PATH_MW / STB_MODEL # /nfs/OpentvOS/v5.x.x/NET/Build6.x.x.x/brand/
elif not (STB_MODEL):
build_path = PATH_MW
print('Path:', build_path.as_posix())
log.info(f'Path created to start download process: {build_path} ')
#build_name_clean = getBuildNameFromDir(build_name) # Take build name without unnecessary words to persist on Build Number Control (txt)
try:
if (MW_VERSION.name == 'v5.2.8'):
### Handling MW 528 ###
#Get all Build_Production_XPTO folders (without attributes) from sftp (MW 5.2.8 only)
path_product_folder = []
for folders in sftp.listdir_attr(build_path.as_posix()):
if not (folders.filename == PRODUCT_TO_NOT_DOWNLOAD_MW_528):
path_product_folder.append(folders.filename)
new_local_path = path_manager(
localFolder.createLocalFolders(local_build_folder,
STB_MODEL)) # STB Model folder
# Loop to create all Build_Production_XPTO
for folder in path_product_folder:
localFolder.createLocalFolders(new_local_path, folder)
# Create local sub-folders from Build_Production_XPTO
# /nfs/OpentvOS/v5.2.8/NET/build_name/STB_Model/build_production_XPTO
for folder in path_product_folder:
createLocalSubDirectories(
sftp,
path_manager.joinpath(build_path,
path_manager(folder)).as_posix(),
path_manager.joinpath(new_local_path, folder), MW_VERSION,
is_folder_zip)
elif (MW_VERSION.name == 'v5.2.4'):
### Handling MW 524 ###
#Check if is a file compacted or a folder
path_product_folder = []
for folders in sftp.listdir_attr(build_path.as_posix()):
if (stat.S_ISREG(folders.st_mode)):
is_folder_zip = True
elif (stat.S_ISDIR(folders.st_mode)):
if not (folders.filename
== PRODUCT_TO_NOT_DOWNLOAD_MW_524):
path_product_folder.append(folders.filename)
if (is_folder_zip):
downloadFiles(sftp, local_build_folder, build_path, MW_VERSION,
is_folder_zip)
if not (is_folder_zip):
new_local_path = path_manager(
localFolder.createLocalFolders(
local_build_folder, STB_MODEL)) # STB Model folder
for folder in path_product_folder:
localFolder.createLocalFolders(new_local_path, folder)
# Create local sub-folders from Build_Production_XPTO
# /nfs/OpentvOS/v5.2.8/NET/build_name/STB_Model/build_production_XPTO
for folder in path_product_folder:
createLocalSubDirectories(
sftp,
path_manager.joinpath(build_path,
path_manager(folder)).as_posix(),
path_manager.joinpath(new_local_path, folder),
MW_VERSION, is_folder_zip)
elif (MW_VERSION.name == 'v5.1.3'):
path_folders = []
for folders in sftp.listdir_attr(build_path.as_posix()):
if (stat.S_ISREG(folders.st_mode)):
is_folder_zip = True
elif (stat.S_ISDIR(folders.st_mode)):
if (folders.filename in FOLDERS_TO_DOWNLOAD_513):
path_folders.append(folders.filename)
if (is_folder_zip):
downloadFiles(sftp, local_build_folder, build_path, MW_VERSION,
is_folder_zip)
if not (is_folder_zip):
folders_dict, remote_path = createSubDirectories513(
sftp, local_build_folder, build_path)
for remote_dir in remote_path:
model = ngCleanModelName(remote_dir.name)
new_local_path = path_manager(
localFolder.createLocalFolders(
folders_dict.get(''.join(model)), remote_dir.name))
downloadFiles(sftp, new_local_path, remote_dir, MW_VERSION,
is_folder_zip)
except (EnvironmentError, IOError, OSError) as e:
print(e)
log.exception('An error occured to start download process')
def downloadFiles(sftp, current_path, remote_path, MW_VERSION, is_folder_zip):
file_name_for_progress_bar = "" # Not necessary, just a improve to progress bar
callback_progressbar, progressbar = progressBarView(
ascii=False,
desc=file_name_for_progress_bar,
unit='b',
unit_scale=True)
# MW 5.2.8 has same folder structure since first version.
if (MW_VERSION.name == 'v5.2.8'):
i = 0 # To control the current_path folders list
for remote_dir in remote_path:
print(remote_dir)
for remote_file in sftp.listdir_attr(remote_dir.as_posix()):
file_name_for_progress_bar = remote_file.filename
# CAROUSEL names changes every new build
# The code below will avoid to download this file.
is_carousel = remote_file.filename.split(
'-') # CAROUSEL is the first word on filename
if remote_file.filename not in FILES_TO_NOT_DOWNLOAD and not is_carousel[
0] == 'CAROUSEL':
print(remote_file.filename)
sftp.get(
path_manager.joinpath(remote_dir,
remote_file.filename).as_posix(),
path_manager.joinpath(current_path[i],
remote_file.filename),
callback_progressbar)
log.info(
f'Files downloaded:{path_manager.joinpath(current_path[i], remote_file.filename)}'
)
i += 1
progressbar.close()
# MW 5.2.4 could have zip folder
elif (MW_VERSION.name == 'v5.2.4' and is_folder_zip == True):
for remote_file in sftp.listdir_attr(remote_path.as_posix()):
if remote_file.filename not in FILES_TO_NOT_DOWNLOAD:
print(remote_file.filename)
sftp.get(
path_manager.joinpath(remote_path,
remote_file.filename).as_posix(),
path_manager.joinpath(current_path, remote_file.filename),
callback_progressbar)
log.info(
f'Files downloaded:{path_manager.joinpath(current_path, remote_file.filename)}'
)
progressbar.close()
# Scenario that cover MW 5.2.4 when doesn't have zip folder
elif (MW_VERSION.name == 'v5.2.4' and is_folder_zip == False):
for remote_dir in remote_path:
print(remote_dir)
for remote_file in sftp.listdir_attr(remote_dir.as_posix()):
file_name_for_progress_bar = remote_file.filename
if remote_file.filename not in FILES_TO_NOT_DOWNLOAD:
print(remote_file.filename)
sftp.get(
path_manager.joinpath(remote_dir,
remote_file.filename).as_posix(),
path_manager.joinpath(current_path[i],
remote_file.filename),
callback_progressbar)
log.info(
f'Files downloaded:{path_manager.joinpath(current_path[i], remote_file.filename)}'
)
i += 1
progressbar.close()
elif (MW_VERSION.name == 'v5.1.3'):
for remote_file in sftp.listdir_attr(remote_path.as_posix()):
if str(remote_file.filename) not in FILES_TO_NOT_DOWNLOAD:
print(path_manager.joinpath(current_path,
remote_file.filename))
sftp.get(
path_manager.joinpath(path_manager(remote_path),
remote_file.filename).as_posix(),
path_manager.joinpath(current_path, remote_file.filename),
callback_progressbar)
log.info(
f'Files downloaded:{path_manager.joinpath(current_path, remote_file.filename)}'
)
progressbar.close()
from pathlib import Path as path_manager
import sys
import logging
debug = False # Used to avoid Traceback errors when Fails
PATH_FOLDER = path_manager(r"Z:\01.Opentv5\16.Version_control")
FILE_NAME_MW528 = "version_control_528.txt"
FILE_NAME_MW524 = "version_control_524.txt"
FILE_NAME_MW513 = "version_control_513.txt"
MW_VERSION_LIST = ['v5.2.8', 'v5.2.4', 'v5.1.3']
##### Log environment #################################################################
log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
file_handler = logging.FileHandler('logs/file_manager.log')
formatter = logging.Formatter(
'%(asctime)s: %(levelname)s : %(name)s : %(message)s')
file_handler.setFormatter(formatter)
log.addHandler(file_handler)
#######################################################################################
'''
0 - Go ahead and download
1 - It's already downloaded
2 - Go ahead and download. Version control wasn't created yet
3 - Fail
'''
def pathControl(LATEST_BUILD, MW_VERSION):
if (MW_VERSION.name in MW_VERSION_LIST):