def main(): check_min_supported_python_version() configure_console_logger() # Default logs: to console # Handle args: args = sys.argv[1:] ArgsReader.read_args(args) # Init Configuration.initialize_app(main_script_dir, main_script_path) # Check if need to log to file as well if Configuration.log_file and len(logging.getLogger( ).handlers) < 2: # Only two handlers are supported - Console & Log File add_log_file_logging() # Main dish if Configuration.export_packages: PackagesExporter.export_packages() if Configuration.import_packages: PackagesImporter.import_packages() # Print nice summary log_summary() # Finish successfully MyGlobals.terminate_program(0)
def wrapper(*args, **kwargs): retry_count = MyGlobals.ftpRetriesCount while retry_count: try: return func(*args, **kwargs) except socket.timeout: print( 'Timeout occurred while attempting to {}. Retry count left: {}/{}' .format(action_description, retry_count, MyGlobals.ftpRetriesCount)) retry_count -= 1 except OSError as errorMsg: error_msg_str = str(errorMsg).lower() timeout_str = 'timed out' if timeout_str not in error_msg_str: print('Failed to {}.\nError:\n{}'.format( action_description, errorMsg)) return None print( 'Timeout occurred while attempting to {}. Retry count left: {}/{}' .format(action_description, retry_count, MyGlobals.ftpRetriesCount)) retry_count -= 1 except BaseException as errorMsg: print("Failed to {}.\nError:\n{}".format( action_description, errorMsg)) return None MyGlobals.sleep_for_a_while( MyGlobals.sleepForBetweenActions_Default) print('Timeouts over - returning') return None
def setup_new_crontab_job(): if not MyCrontab.setup_script_as_crontab_job(): MyGlobals.terminate_program(2) MyGlobals.terminate_program( 0, 'Success setting-up script to run as a crontab job\nIt will run automatically by the specified timing' ) return
def prepare_ftp_and_os_indexes(ftp_con, src, dest): if not change_ftp_dir(ftp_con, src): return False MyGlobals.mkdir_p(dest) if MyGlobals.isVerbose: print("Created dir: {}".format(dest)) os.chdir(dest) return True
def _download_ftp_dir(ftp_con, ftp_src, dest): print('Downloading dir: "{}" to: {}'.format(ftp_src, dest)) if not prepare_ftp_and_os_indexes(ftp_con, ftp_src, dest): return False file_list = get_file_list(ftp_con) if file_list is None or file_list is False: return False ftp_src_dir_name = MyGlobals.get_dir_name(ftp_src) print('Dir {} Files:\n -- {}\n\n'.format(ftp_src_dir_name, '\n -- '.join(file_list))) cur_ind = 0 end_ind = len(file_list) download_process_ok = True while (download_process_ok and cur_ind < end_ind): MyGlobals.sleep_for_a_while(MyGlobals.sleepBetweenDownloads_Default) cur_target_name = file_list[cur_ind] # Testing if it's a directory - then recurse try: target_path = ftp_src + "/" + cur_target_name # Check if a directory - if so then ftp-cd to it: if MyGlobals.isVerbose: print("Checking if {} is a directory".format(target_path)) is_dir = ftp_check_is_dir(ftp_con, target_path) if is_dir is None: print( 'Failed checking if {} is a directory'.format(target_path)) return False elif is_dir: if not change_ftp_dir(ftp_con, target_path): return False else: raise ftplib.error_perm( "target_path={} is a file".format(target_path)) download_process_ok = download_process_ok and _download_ftp_dir( ftp_con, target_path, dest + '/' + cur_target_name) if download_process_ok and not change_ftp_dir(ftp_con, ftp_src): return False # If got flag: --remove_src then Attempt to remove the dir after downloading it successfully if download_process_ok and MyGlobals.isRemoveSrc: download_process_ok = download_process_ok and ftp_delete_dir( ftp_con, target_path) except ftplib.error_perm: # It's a file: download_process_ok = download_process_ok and download_ftp_file( ftp_con, cur_target_name, dest) cur_ind += 1 return download_process_ok
def register_exported_files(py_pkg): global last_exported_dir_files exported_packags_path = path.join(Configuration.export_to, "*.*") current_exported_dir_files = MyGlobals.get_files_list_from_path( exported_packags_path) new_created_files = MyGlobals.list_subtract(current_exported_dir_files, last_exported_dir_files) log_info("Newly created files: {}".format(new_created_files)) exported_packages_files_data[py_pkg.name] = new_created_files last_exported_dir_files = current_exported_dir_files
def download_ftp_dir(ftp_con, ftp_src, dest): dest = MyGlobals.remove_trailing_slash(dest) ftp_src = MyGlobals.remove_trailing_slash(ftp_src) src_path_dir_name = MyGlobals.get_dir_name(ftp_src) download_result = _download_ftp_dir(ftp_con, ftp_src, dest + '/' + src_path_dir_name) if download_result and MyGlobals.isRemoveSrc: return download_result and ftp_delete_dir(ftp_con, ftp_src) return download_result
def index_packages_dir(dir_path): log_info("Indexing offline packages dir: {}".format(dir_path)) try: exit_code = commands.dir2pi(["dir2pi", dir_path, "--no-symlink"]) if exit_code != 0: log_error("Error while indexing dir: {}\nExit with: {}".format( dir_path, exit_code)) MyGlobals.terminate_program(1) except BaseException as error_msg: log_error("Error while indexing dir: {}\n{}".format( dir_path, error_msg)) MyGlobals.terminate_program(1)
def add_log_file_logging(): try: log_debug("Logging to file: {}".format(Configuration.log_file)) format_str = Configuration.log_fmt_str formatter = logging.Formatter(format_str, datefmt=Configuration.log_file_time_fmt) f_handler = logging.FileHandler(Configuration.log_file, mode='w') f_handler.setLevel(logging.INFO) f_handler.setFormatter(formatter) logging.getLogger().addHandler(f_handler) except BaseException as error_msg: log_error("Failed logging to file at: {}\n{}\nAborting..".format( Configuration.log_file, error_msg)) MyGlobals.terminate_program(1)
def export_packages(): export_list = Configuration.export_packages.split(',') for packages_src in export_list: collect_packages_to_export(packages_src.strip()) print_packages_to_export_dict() # Export the packages to: Configuration.export_to export_collected_packages() if not MyGlobals.is_dir(Configuration.export_to): log_error( "Finish exporting but missing exported packages dir: {}\nSomething went wrong. Check log above." .format(Configuration.export_to)) MyGlobals.terminate_program(1) MyDir2Pi.index_packages_dir(Configuration.export_to)
def collect_packages_to_export(packages_src): if not packages_src: log_warning("Skipping export of null or empty string value") return if MyGlobals.is_file(packages_src): log_debug("Reading packages from file: {}".format(packages_src)) action_dict = MyGlobals.read_file_lines_as_list(packages_src) if not action_dict["Result"]: MyGlobals.terminate_program(1) packages_list = action_dict["MoreInfo"] add_packages_list_to_packages_to_export_dict(packages_list) elif packages_src == "*": add_all_pip_freeze_packages_to_packages_to_export_dict() else: add_package_to_packages_to_export_dict( packages_src ) # Normal package request: 'pkg_name' or 'pkg_name==pkg_version'
def add_all_pip_freeze_packages_to_packages_to_export_dict(): pip = Configuration.pip_executable opts = "freeze" cmnd = "{} {}".format(pip, opts) action_dict = MyGlobals.execute_command(cmnd) if not action_dict["Result"]: log_error( "Failed to retrieve all installed pip packages using 'pip freeze' command" ) MyGlobals.terminate_program(1) pip_freeze_output = action_dict["MoreInfo"].decode( Configuration.decode_commands_output_fmt).split("\n") for pkg_str in pip_freeze_output: if pkg_str and "==" not in pkg_str: log_warning( "pip freeze command invalid output: {}. Not in format of: pkg_name==pkg_version. Skipping this line" .format(pkg_str)) continue add_package_to_packages_to_export_dict(pkg_str)
def export_package(py_pkg): export_msg = "Exporting package: {}".format(py_pkg.full_name) log_info(export_msg) pip = Configuration.pip_executable import_opts = "download -d \"{}\" {}".format(Configuration.export_to, py_pkg.full_name) extra_pip_args = Configuration.extra_pip_args cmnd = "{} {} {}".format(pip, import_opts, extra_pip_args) return MyGlobals.execute_command(cmnd)
def read_args(args_list): log_info("Reading received args: {}".format(args_list)) global received_args if len(args_list) == 0: parser.print_help() MyGlobals.terminate_program(0) try: received_args = parser.parse_args() except SystemExit as errorCode: MyGlobals.terminate_program(errorCode) except BaseException as errorMsg: log_error("Error - Failed to parse received args: {}\n{}".format(args_list, errorMsg)) MyGlobals.terminate_program(1) Configuration.pip_executable = received_args.pip_executable Configuration.extra_pip_args = received_args.extra_pip_args Configuration.is_verbose = received_args.verbose Configuration.export_packages = received_args.export_packages if received_args.export_to: Configuration.export_to = received_args.export_to Configuration.exported_files_json_file = path.join(received_args.export_to, "simple", "exported_files.json") Configuration.import_packages = received_args.import_packages Configuration.import_from = received_args.import_from Configuration.log_file = received_args.log_file log_debug("Finished reading args successfully")
def add_all_pip_available_packages_to_packages_to_import_dict(): simple_index_dir = Configuration.simple_index_dir if MyGlobals.dir_exists(simple_index_dir): dir_flat_files_list = os.listdir(simple_index_dir) else: dir_flat_files_list = os.listdir(Configuration.import_from) dir_flat_files_list = convert_flat_files_names_list_to_packages_names( dir_flat_files_list) for pkg_name in dir_flat_files_list: if pkg_name.lower() == "index.html": continue add_package_to_packages_to_import_dict(pkg_name)
def import_package(py_pkg): import_msg = "Importing package: {}".format(py_pkg.full_name) log_info(import_msg) pip = Configuration.pip_executable # import_opts = "install --no-index --find-links=\"{}\" {}".format(Configuration.import_from, py_pkg.full_name) index_url = Configuration.index_url if os.name == 'nt': index_url = "file:///{}".format(index_url) import_opts = "install --index-url=\"{}\" {}".format( index_url, py_pkg.full_name) extra_pip_args = Configuration.extra_pip_args cmnd = "{} {} {}".format(pip, import_opts, extra_pip_args) return MyGlobals.execute_command(cmnd)
def download_ftp_file(ftp_con, file_name, dest_path, create_dirs=False): if MyGlobals.isVerbose: cur_loc = ftp_get_pwd(ftp_con) if cur_loc: print('Downloading file: {} to: {}'.format( cur_loc + '/' + file_name, dest_path)) else: print('Downloading file: {} to: {}'.format(file_name, dest_path)) if create_dirs: if not MyGlobals.mkdir_p(dest_path): return None if not change_local_dir(dest_path): return False download_result = _download_ftp_file(ftp_con, file_name, dest_path) if download_result and MyGlobals.isRemoveSrc: return ftp_delete_file(ftp_con, file_name) return download_result
def pull_files_dirs_from_ftp(): # Get connection to FTP server ftp_con = MyFtpLib.get_ftp_connection(MyGlobals.ftpAddr, MyGlobals.ftpPort, MyGlobals.ftpActionsTimeoutSec) if ftp_con is None: MyGlobals.terminate_program( 2, msg="Failed getting ftp connection to: {}:{}".format( MyGlobals.ftpAddr, MyGlobals.ftpPort)) ftp_password = MyGlobals.ftpPassword # If password is hashed - attempt to decipher it if MyGlobals.isHashed: ftp_password = MyPasswordDecipher.decipher_password_hash(ftp_password) if ftp_password is None: MyGlobals.terminate_program(2) # Attempt to login if not MyFtpLib.login_to_ftp_server(ftp_con, MyGlobals.ftpUser, ftp_password): MyGlobals.terminate_program(2) # Download src path: download_result = MyFtpLib.download_path(ftp_con, MyGlobals.ftpSourcePath, MyGlobals.destPath) # Check if successful if not download_result: MyGlobals.terminate_program( 2, 'FAILED - Downloading: {} to: {}'.format(MyGlobals.ftpSourcePath, MyGlobals.destPath)) print('SUCCESS - Downloading: {} to: {}'.format(MyGlobals.ftpSourcePath, MyGlobals.destPath)) return
def main(): received_args = MyGlobals.read_command_line_args(sys.argv[1:]) print('FTP Puller - Started') print('Command Line: {} {}\n'.format(sys.argv[0], received_args)) if not MyGlobals.check_params(): # Check that params are ok MyGlobals.terminate_program(1) if MyGlobals.isPrintCronJobs: print_crontab_jobs_with_comment() if MyGlobals.isRemoveCronJobs: # If wants to remove old/running crontab jobs remove_old_crontab_jobs() if MyGlobals.isSetupAsCronjob and MyGlobals.receivedDownloadArgs: # Check if wants to setup this script as a crontab job setup_new_crontab_job() elif MyGlobals.receivedDownloadArgs: # Check if wants to run this script once pull_files_dirs_from_ftp() # Start downloading MyGlobals.terminate_program(0, 'FTP Puller - Finished')
def init(): global pip_executable global extra_pip_args global pip_2_tgz_executable global dir_2_pi_executable global export_packages global export_to global import_packages global import_from global exported_files_json_file global log_file global simple_index_dir global index_url # Expand env variables if any pip_executable = MyGlobals.remove_surrounding_quotes( MyGlobals.expand_variable( pip_executable)).strip() if pip_executable else pip_executable extra_pip_args = MyGlobals.remove_surrounding_quotes( MyGlobals.expand_variable( extra_pip_args)).strip() if extra_pip_args else extra_pip_args export_packages = MyGlobals.remove_surrounding_quotes( MyGlobals.expand_variable( export_packages)).strip() if export_packages else export_packages export_to = MyGlobals.remove_surrounding_quotes( MyGlobals.expand_variable( export_to)).strip() if export_to else export_to import_packages = MyGlobals.remove_surrounding_quotes( MyGlobals.expand_variable( import_packages)).strip() if import_packages else import_packages import_from = MyGlobals.remove_surrounding_quotes( MyGlobals.expand_variable( import_from)).strip() if import_from else import_from exported_files_json_file = MyGlobals.remove_surrounding_quotes( MyGlobals.expand_variable(exported_files_json_file)).strip( ) if exported_files_json_file else exported_files_json_file log_file = MyGlobals.remove_surrounding_quotes( MyGlobals.expand_variable(log_file)).strip() if log_file else log_file simple_index_dir = os.path.join(import_from, "simple") if MyGlobals.dir_exists(simple_index_dir): simple_index_dir = MyGlobals.convert_windows_path_to_linux( simple_index_dir) index_url = simple_index_dir if is_verbose: logging.getLogger().setLevel(logging.NOTSET) for log_hdnlr in logging.getLogger().handlers: log_hdnlr.setLevel(logging.NOTSET)
def check_min_supported_python_version(): if sys.version_info[0] < 3: print("Python 3 or a more recent version is required.") MyGlobals.terminate_program(1)
def remove_old_crontab_jobs(): removed_result = MyCrontab.remove_current_ftp_puller_crontab_jobs() if removed_result is False: MyGlobals.terminate_program(2) return
def print_crontab_jobs_with_comment(): print_result = MyCrontab.print_current_ftp_puller_crontab_jobs() if print_result is False: MyGlobals.terminate_program(2) return