def main():
check_min_supported_python_version()
configure_console_logger() # Default logs: to console
# Handle args:
args = sys.argv[1:]
ArgsReader.read_args(args)
# Init
Configuration.initialize_app(main_script_dir, main_script_path)
# Check if need to log to file as well
if Configuration.log_file and len(logging.getLogger(
).handlers) < 2: # Only two handlers are supported - Console & Log File
add_log_file_logging()
# Main dish
if Configuration.export_packages:
PackagesExporter.export_packages()
if Configuration.import_packages:
PackagesImporter.import_packages()
# Print nice summary
log_summary()
# Finish successfully
MyGlobals.terminate_program(0)
def wrapper(*args, **kwargs):
retry_count = MyGlobals.ftpRetriesCount
while retry_count:
try:
return func(*args, **kwargs)
except socket.timeout:
print(
'Timeout occurred while attempting to {}. Retry count left: {}/{}'
.format(action_description, retry_count,
MyGlobals.ftpRetriesCount))
retry_count -= 1
except OSError as errorMsg:
error_msg_str = str(errorMsg).lower()
timeout_str = 'timed out'
if timeout_str not in error_msg_str:
print('Failed to {}.\nError:\n{}'.format(
action_description, errorMsg))
return None
print(
'Timeout occurred while attempting to {}. Retry count left: {}/{}'
.format(action_description, retry_count,
MyGlobals.ftpRetriesCount))
retry_count -= 1
except BaseException as errorMsg:
print("Failed to {}.\nError:\n{}".format(
action_description, errorMsg))
return None
MyGlobals.sleep_for_a_while(
MyGlobals.sleepForBetweenActions_Default)
print('Timeouts over - returning')
return None
def setup_new_crontab_job():
if not MyCrontab.setup_script_as_crontab_job():
MyGlobals.terminate_program(2)
MyGlobals.terminate_program(
0,
'Success setting-up script to run as a crontab job\nIt will run automatically by the specified timing'
)
return
def prepare_ftp_and_os_indexes(ftp_con, src, dest):
if not change_ftp_dir(ftp_con, src):
return False
MyGlobals.mkdir_p(dest)
if MyGlobals.isVerbose:
print("Created dir: {}".format(dest))
os.chdir(dest)
return True
def _download_ftp_dir(ftp_con, ftp_src, dest):
print('Downloading dir: "{}" to: {}'.format(ftp_src, dest))
if not prepare_ftp_and_os_indexes(ftp_con, ftp_src, dest):
return False
file_list = get_file_list(ftp_con)
if file_list is None or file_list is False:
return False
ftp_src_dir_name = MyGlobals.get_dir_name(ftp_src)
print('Dir {} Files:\n -- {}\n\n'.format(ftp_src_dir_name,
'\n -- '.join(file_list)))
cur_ind = 0
end_ind = len(file_list)
download_process_ok = True
while (download_process_ok and cur_ind < end_ind):
MyGlobals.sleep_for_a_while(MyGlobals.sleepBetweenDownloads_Default)
cur_target_name = file_list[cur_ind]
# Testing if it's a directory - then recurse
try:
target_path = ftp_src + "/" + cur_target_name
# Check if a directory - if so then ftp-cd to it:
if MyGlobals.isVerbose:
print("Checking if {} is a directory".format(target_path))
is_dir = ftp_check_is_dir(ftp_con, target_path)
if is_dir is None:
print(
'Failed checking if {} is a directory'.format(target_path))
return False
elif is_dir:
if not change_ftp_dir(ftp_con, target_path):
return False
else:
raise ftplib.error_perm(
"target_path={} is a file".format(target_path))
download_process_ok = download_process_ok and _download_ftp_dir(
ftp_con, target_path, dest + '/' + cur_target_name)
if download_process_ok and not change_ftp_dir(ftp_con, ftp_src):
return False
# If got flag: --remove_src then Attempt to remove the dir after downloading it successfully
if download_process_ok and MyGlobals.isRemoveSrc:
download_process_ok = download_process_ok and ftp_delete_dir(
ftp_con, target_path)
except ftplib.error_perm:
# It's a file:
download_process_ok = download_process_ok and download_ftp_file(
ftp_con, cur_target_name, dest)
cur_ind += 1
return download_process_ok
def register_exported_files(py_pkg):
global last_exported_dir_files
exported_packags_path = path.join(Configuration.export_to, "*.*")
current_exported_dir_files = MyGlobals.get_files_list_from_path(
exported_packags_path)
new_created_files = MyGlobals.list_subtract(current_exported_dir_files,
last_exported_dir_files)
log_info("Newly created files: {}".format(new_created_files))
exported_packages_files_data[py_pkg.name] = new_created_files
last_exported_dir_files = current_exported_dir_files
def download_ftp_dir(ftp_con, ftp_src, dest):
dest = MyGlobals.remove_trailing_slash(dest)
ftp_src = MyGlobals.remove_trailing_slash(ftp_src)
src_path_dir_name = MyGlobals.get_dir_name(ftp_src)
download_result = _download_ftp_dir(ftp_con, ftp_src,
dest + '/' + src_path_dir_name)
if download_result and MyGlobals.isRemoveSrc:
return download_result and ftp_delete_dir(ftp_con, ftp_src)
return download_result
def index_packages_dir(dir_path):
log_info("Indexing offline packages dir: {}".format(dir_path))
try:
exit_code = commands.dir2pi(["dir2pi", dir_path, "--no-symlink"])
if exit_code != 0:
log_error("Error while indexing dir: {}\nExit with: {}".format(
dir_path, exit_code))
MyGlobals.terminate_program(1)
except BaseException as error_msg:
log_error("Error while indexing dir: {}\n{}".format(
dir_path, error_msg))
MyGlobals.terminate_program(1)
def add_log_file_logging():
try:
log_debug("Logging to file: {}".format(Configuration.log_file))
format_str = Configuration.log_fmt_str
formatter = logging.Formatter(format_str,
datefmt=Configuration.log_file_time_fmt)
f_handler = logging.FileHandler(Configuration.log_file, mode='w')
f_handler.setLevel(logging.INFO)
f_handler.setFormatter(formatter)
logging.getLogger().addHandler(f_handler)
except BaseException as error_msg:
log_error("Failed logging to file at: {}\n{}\nAborting..".format(
Configuration.log_file, error_msg))
MyGlobals.terminate_program(1)
def export_packages():
export_list = Configuration.export_packages.split(',')
for packages_src in export_list:
collect_packages_to_export(packages_src.strip())
print_packages_to_export_dict()
# Export the packages to: Configuration.export_to
export_collected_packages()
if not MyGlobals.is_dir(Configuration.export_to):
log_error(
"Finish exporting but missing exported packages dir: {}\nSomething went wrong. Check log above."
.format(Configuration.export_to))
MyGlobals.terminate_program(1)
MyDir2Pi.index_packages_dir(Configuration.export_to)
def collect_packages_to_export(packages_src):
if not packages_src:
log_warning("Skipping export of null or empty string value")
return
if MyGlobals.is_file(packages_src):
log_debug("Reading packages from file: {}".format(packages_src))
action_dict = MyGlobals.read_file_lines_as_list(packages_src)
if not action_dict["Result"]:
MyGlobals.terminate_program(1)
packages_list = action_dict["MoreInfo"]
add_packages_list_to_packages_to_export_dict(packages_list)
elif packages_src == "*":
add_all_pip_freeze_packages_to_packages_to_export_dict()
else:
add_package_to_packages_to_export_dict(
packages_src
) # Normal package request: 'pkg_name' or 'pkg_name==pkg_version'
def add_all_pip_freeze_packages_to_packages_to_export_dict():
pip = Configuration.pip_executable
opts = "freeze"
cmnd = "{} {}".format(pip, opts)
action_dict = MyGlobals.execute_command(cmnd)
if not action_dict["Result"]:
log_error(
"Failed to retrieve all installed pip packages using 'pip freeze' command"
)
MyGlobals.terminate_program(1)
pip_freeze_output = action_dict["MoreInfo"].decode(
Configuration.decode_commands_output_fmt).split("\n")
for pkg_str in pip_freeze_output:
if pkg_str and "==" not in pkg_str:
log_warning(
"pip freeze command invalid output: {}. Not in format of: pkg_name==pkg_version. Skipping this line"
.format(pkg_str))
continue
add_package_to_packages_to_export_dict(pkg_str)
def export_package(py_pkg):
export_msg = "Exporting package: {}".format(py_pkg.full_name)
log_info(export_msg)
pip = Configuration.pip_executable
import_opts = "download -d \"{}\" {}".format(Configuration.export_to,
py_pkg.full_name)
extra_pip_args = Configuration.extra_pip_args
cmnd = "{} {} {}".format(pip, import_opts, extra_pip_args)
return MyGlobals.execute_command(cmnd)
def read_args(args_list):
log_info("Reading received args: {}".format(args_list))
global received_args
if len(args_list) == 0:
parser.print_help()
MyGlobals.terminate_program(0)
try:
received_args = parser.parse_args()
except SystemExit as errorCode:
MyGlobals.terminate_program(errorCode)
except BaseException as errorMsg:
log_error("Error - Failed to parse received args: {}\n{}".format(args_list, errorMsg))
MyGlobals.terminate_program(1)
Configuration.pip_executable = received_args.pip_executable
Configuration.extra_pip_args = received_args.extra_pip_args
Configuration.is_verbose = received_args.verbose
Configuration.export_packages = received_args.export_packages
if received_args.export_to:
Configuration.export_to = received_args.export_to
Configuration.exported_files_json_file = path.join(received_args.export_to, "simple", "exported_files.json")
Configuration.import_packages = received_args.import_packages
Configuration.import_from = received_args.import_from
Configuration.log_file = received_args.log_file
log_debug("Finished reading args successfully")
def add_all_pip_available_packages_to_packages_to_import_dict():
simple_index_dir = Configuration.simple_index_dir
if MyGlobals.dir_exists(simple_index_dir):
dir_flat_files_list = os.listdir(simple_index_dir)
else:
dir_flat_files_list = os.listdir(Configuration.import_from)
dir_flat_files_list = convert_flat_files_names_list_to_packages_names(
dir_flat_files_list)
for pkg_name in dir_flat_files_list:
if pkg_name.lower() == "index.html":
continue
add_package_to_packages_to_import_dict(pkg_name)
def import_package(py_pkg):
import_msg = "Importing package: {}".format(py_pkg.full_name)
log_info(import_msg)
pip = Configuration.pip_executable
# import_opts = "install --no-index --find-links=\"{}\" {}".format(Configuration.import_from, py_pkg.full_name)
index_url = Configuration.index_url
if os.name == 'nt':
index_url = "file:///{}".format(index_url)
import_opts = "install --index-url=\"{}\" {}".format(
index_url, py_pkg.full_name)
extra_pip_args = Configuration.extra_pip_args
cmnd = "{} {} {}".format(pip, import_opts, extra_pip_args)
return MyGlobals.execute_command(cmnd)
def download_ftp_file(ftp_con, file_name, dest_path, create_dirs=False):
if MyGlobals.isVerbose:
cur_loc = ftp_get_pwd(ftp_con)
if cur_loc:
print('Downloading file: {} to: {}'.format(
cur_loc + '/' + file_name, dest_path))
else:
print('Downloading file: {} to: {}'.format(file_name, dest_path))
if create_dirs:
if not MyGlobals.mkdir_p(dest_path):
return None
if not change_local_dir(dest_path):
return False
download_result = _download_ftp_file(ftp_con, file_name, dest_path)
if download_result and MyGlobals.isRemoveSrc:
return ftp_delete_file(ftp_con, file_name)
return download_result
def pull_files_dirs_from_ftp():
# Get connection to FTP server
ftp_con = MyFtpLib.get_ftp_connection(MyGlobals.ftpAddr, MyGlobals.ftpPort,
MyGlobals.ftpActionsTimeoutSec)
if ftp_con is None:
MyGlobals.terminate_program(
2,
msg="Failed getting ftp connection to: {}:{}".format(
MyGlobals.ftpAddr, MyGlobals.ftpPort))
ftp_password = MyGlobals.ftpPassword
# If password is hashed - attempt to decipher it
if MyGlobals.isHashed:
ftp_password = MyPasswordDecipher.decipher_password_hash(ftp_password)
if ftp_password is None:
MyGlobals.terminate_program(2)
# Attempt to login
if not MyFtpLib.login_to_ftp_server(ftp_con, MyGlobals.ftpUser,
ftp_password):
MyGlobals.terminate_program(2)
# Download src path:
download_result = MyFtpLib.download_path(ftp_con, MyGlobals.ftpSourcePath,
MyGlobals.destPath)
# Check if successful
if not download_result:
MyGlobals.terminate_program(
2,
'FAILED - Downloading: {} to: {}'.format(MyGlobals.ftpSourcePath,
MyGlobals.destPath))
print('SUCCESS - Downloading: {} to: {}'.format(MyGlobals.ftpSourcePath,
MyGlobals.destPath))
return
def main():
received_args = MyGlobals.read_command_line_args(sys.argv[1:])
print('FTP Puller - Started')
print('Command Line: {} {}\n'.format(sys.argv[0], received_args))
if not MyGlobals.check_params(): # Check that params are ok
MyGlobals.terminate_program(1)
if MyGlobals.isPrintCronJobs:
print_crontab_jobs_with_comment()
if MyGlobals.isRemoveCronJobs: # If wants to remove old/running crontab jobs
remove_old_crontab_jobs()
if MyGlobals.isSetupAsCronjob and MyGlobals.receivedDownloadArgs: # Check if wants to setup this script as a crontab job
setup_new_crontab_job()
elif MyGlobals.receivedDownloadArgs: # Check if wants to run this script once
pull_files_dirs_from_ftp() # Start downloading
MyGlobals.terminate_program(0, 'FTP Puller - Finished')
def init():
global pip_executable
global extra_pip_args
global pip_2_tgz_executable
global dir_2_pi_executable
global export_packages
global export_to
global import_packages
global import_from
global exported_files_json_file
global log_file
global simple_index_dir
global index_url
# Expand env variables if any
pip_executable = MyGlobals.remove_surrounding_quotes(
MyGlobals.expand_variable(
pip_executable)).strip() if pip_executable else pip_executable
extra_pip_args = MyGlobals.remove_surrounding_quotes(
MyGlobals.expand_variable(
extra_pip_args)).strip() if extra_pip_args else extra_pip_args
export_packages = MyGlobals.remove_surrounding_quotes(
MyGlobals.expand_variable(
export_packages)).strip() if export_packages else export_packages
export_to = MyGlobals.remove_surrounding_quotes(
MyGlobals.expand_variable(
export_to)).strip() if export_to else export_to
import_packages = MyGlobals.remove_surrounding_quotes(
MyGlobals.expand_variable(
import_packages)).strip() if import_packages else import_packages
import_from = MyGlobals.remove_surrounding_quotes(
MyGlobals.expand_variable(
import_from)).strip() if import_from else import_from
exported_files_json_file = MyGlobals.remove_surrounding_quotes(
MyGlobals.expand_variable(exported_files_json_file)).strip(
) if exported_files_json_file else exported_files_json_file
log_file = MyGlobals.remove_surrounding_quotes(
MyGlobals.expand_variable(log_file)).strip() if log_file else log_file
simple_index_dir = os.path.join(import_from, "simple")
if MyGlobals.dir_exists(simple_index_dir):
simple_index_dir = MyGlobals.convert_windows_path_to_linux(
simple_index_dir)
index_url = simple_index_dir
if is_verbose:
logging.getLogger().setLevel(logging.NOTSET)
for log_hdnlr in logging.getLogger().handlers:
log_hdnlr.setLevel(logging.NOTSET)
def check_min_supported_python_version():
if sys.version_info[0] < 3:
print("Python 3 or a more recent version is required.")
MyGlobals.terminate_program(1)
def remove_old_crontab_jobs():
removed_result = MyCrontab.remove_current_ftp_puller_crontab_jobs()
if removed_result is False:
MyGlobals.terminate_program(2)
return
def print_crontab_jobs_with_comment():
print_result = MyCrontab.print_current_ftp_puller_crontab_jobs()
if print_result is False:
MyGlobals.terminate_program(2)
return
