def do_get_qat(self, arg): """Retrieves the latest QAT installer from Box (config in settings.json)""" ftps = FTP_TLS(host=self.settings['box_host'], user=self.settings['box_user'], passwd=self.settings['box_pwd']) builds = list(ftps.mlsd(self.settings['ci_root'])) builds.sort(key=lambda entry: entry[1]['modify'], reverse=True) out.info("Retrieving build number " + builds[2][0]) build_num = builds[2][0] prev_build_num = builds[3][0] path_to_installer = self.path_to_installer_for_build(build_num) if platform == "linux" or platform == "linux2": local_filename = self.dot_dir + '/pentaho-business-analytics.bin' elif platform == "darwin": local_filename = self.dot_dir + '/pentaho-business-analytics.app.tar.gz' rm(local_filename) try: self.retrieve_file(ftps, local_filename, path_to_installer) except: out.error('Failed to get build ' + build_num + '.\nTrying to get build ' + prev_build_num) path_to_installer = self.path_to_installer_for_build( prev_build_num) try: self.retrieve_file(ftps, local_filename, path_to_installer) except: out.error('Failed to get build ' + prev_build_num)
def find_versions_above( mirror: Dict[str, str], target_version: Version ) -> Generator[Mapping[str, Union[str, Version]], None, None]: print("Using mirror:", mirror['host']) tls = get_use_tls() if tls: ftp = FTP_TLS(mirror['host']) else: ftp = FTP(mirror['host']) ftp.login(user=mirror.get('user', None), passwd=mirror.get('passwd', ''), acct=mirror.get('acct', '')) ftp.cwd(mirror['path']) for filename, attrs in sorted(ftp.mlsd(), key=lambda tup: tup[0]): if not attrs["type"] == "file": continue if not filename.endswith(".tar.gz"): continue pkgver = get_package_version(filename) if not pkgver: continue if pkgver > target_version: fileinfo = mirror.copy() fileinfo["filename"] = filename fileinfo["version"] = pkgver fileinfo["tls"] = tls yield fileinfo
def upload_archive_file(self, local_filename, remote_filename, target_dir): yield("Uploading {} to FTP in directory: {}, filename: {}".format(local_filename, target_dir, remote_filename)) local_filesize = os.stat(local_filename).st_size self.upload_total = os.stat(local_filename).st_size self.upload_current = 0 if self.settings.ftps['no_certificate_check']: context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) context.verify_mode = ssl.CERT_NONE context.check_hostname = False else: context = ssl.create_default_context() ftps = FTP_TLS( host=self.settings.ftps['address'], user=self.settings.ftps['user'], passwd=self.settings.ftps['passwd'], context=context, source_address=self.settings.ftps[ 'source_address'], timeout=self.settings.timeout_timer ) ftps.cwd(target_dir) ftps.encoding = 'utf8' ftps.prot_p() for line in ftps.mlsd(facts=["size"]): if(line[0] == remote_filename and local_filesize == int(line[1]["size"])): yield("File exists and size is equal.") ftps.close() return with open(local_filename, "rb") as file: for retry_count in range(3): try: ftps.storbinary( 'STOR %s' % remote_filename, file, callback=lambda data, args=self.print_method: self.print_progress(data, args) ) except (ConnectionResetError, socket.timeout, TimeoutError): yield("Upload failed, retrying...") else: break yield("\nFile uploaded.") ftps.close()
class PostDownloader(object): def __init__(self, settings: 'Settings', logger, web_queue=None) -> None: self.settings = settings self.web_queue = web_queue self.logger = logger self.ftps: Optional[FTP_TLS] = None self.current_ftp_dir: Optional[str] = None self.current_download: DataDict = { 'filename': '', 'blocksize': 0, 'speed': 0, 'index': 0, 'total': 0, } def process_downloaded_archive(self, archive: Archive) -> None: if os.path.isfile(archive.zipped.path): except_at_open = False return_error = None try: my_zip = ZipFile( archive.zipped.path, 'r') return_error = my_zip.testzip() my_zip.close() except (BadZipFile, NotImplementedError): except_at_open = True if except_at_open or return_error: if 'panda' in archive.source_type: self.logger.error( "For archive: {}, file check on downloaded zipfile failed on file: {}, " "forcing download as panda_archive to fix it.".format(archive, archive.zipped.path) ) crc32 = calc_crc32( archive.zipped.path) Archive.objects.add_or_update_from_values({'crc32': crc32}, pk=archive.pk) if self.web_queue and archive.gallery: temp_settings = Settings(load_from_config=self.settings.config) temp_settings.allow_downloaders_only(['panda_archive'], True, True, True) self.web_queue.enqueue_args_list((archive.gallery.get_link(),), override_options=temp_settings) return else: self.logger.warning( "For archive: {}, File check on downloaded zipfile: {}. " "Check the file manually.".format(archive, archive.zipped.path) ) crc32 = calc_crc32( archive.zipped.path) filesize = get_zip_filesize( archive.zipped.path) filecount = filecount_in_zip( archive.zipped.path) values = {'crc32': crc32, 'filesize': filesize, 'filecount': filecount, } updated_archive = Archive.objects.add_or_update_from_values( values, pk=archive.pk) if archive.gallery and updated_archive.filesize != updated_archive.gallery.filesize: if Archive.objects.filter(gallery=updated_archive.gallery, filesize=updated_archive.gallery.filesize): self.logger.info( "For archive: {} size does not match gallery, " "but there's already another archive that matches.".format(updated_archive) ) return if 'panda' in archive.source_type: self.logger.info( "For archive: {} size does not match gallery, " "downloading again from panda_archive.".format(updated_archive) ) if self.web_queue: temp_settings = Settings(load_from_config=self.settings.config) temp_settings.allow_downloaders_only(['panda_archive'], True, True, True) self.web_queue.enqueue_args_list( (updated_archive.gallery.get_link(), ), override_options=temp_settings ) else: self.logger.warning( "For archive: {} size does not match gallery. Check the file manually.".format(archive) ) def write_file_update_progress(self, cmd: str, callback: Callable, filesize: int = 0, blocksize: int = 8192, rest: bool = None) -> str: self.ftps.voidcmd('TYPE I') # type: ignore with self.ftps.transfercmd(cmd, rest) as conn: # type: ignore self.current_download['filesize'] = filesize self.current_download['downloaded'] = 0 self.current_download['filename'] = cmd.replace('RETR ', '') start = time.clock() while 1: data = conn.recv(blocksize) if not data: break downloaded = len(data) self.current_download['downloaded'] += downloaded current = time.clock() if current > start: self.current_download['speed'] = self.current_download['downloaded'] / ((current - start) * 1024) callback(data) self.current_download['filename'] = '' self.current_download['speed'] = 0 self.current_download['filesize'] = 0 # shutdown ssl layer if _SSLSocket is not None and isinstance(conn, _SSLSocket): conn.unwrap() return self.ftps.voidresp() # type: ignore def start_connection(self) -> None: if self.settings.ftps['no_certificate_check']: context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) context.verify_mode = ssl.CERT_NONE context.check_hostname = False else: context = ssl.create_default_context() self.ftps = FTP_TLS( host=self.settings.ftps['address'], user=self.settings.ftps['user'], passwd=self.settings.ftps['passwd'], context=context, source_address=self.settings.ftps['source_address'], timeout=self.settings.timeout_timer ) # Hath downloads self.ftps.prot_p() def set_current_dir(self, self_dir: str) -> None: self.current_ftp_dir = self_dir if not self.ftps: return None self.ftps.cwd(self_dir) def download_all_missing(self, archives: Iterable[Archive] = None) -> None: files_torrent = [] files_hath = [] if not archives: found_archives: Iterable[Archive] = list(Archive.objects.filter_by_dl_remote()) else: found_archives = archives if not found_archives: return for archive in found_archives: if 'torrent' in archive.match_type: files_torrent.append(archive) elif 'hath' in archive.match_type: files_hath.append(archive) if len(files_torrent) + len(files_hath) == 0: return self.start_connection() if not self.ftps: self.logger.error( "Cannot download the archives, the FTP connection is not initialized." ) return None # Hath downloads if len(files_hath) > 0: self.set_current_dir(self.settings.providers['panda'].remote_hath_dir) # self.ftps.encoding = 'utf8' files_matched_hath = [] for line in self.ftps.mlsd(facts=["type"]): if line[1]["type"] != 'dir': continue m = re.search(r'.*?\[(\d+)\]$', line[0]) if m: for archive in files_hath: if m.group(1) == archive.gallery.gid: files_matched_hath.append( (line[0], archive.zipped.path, int(archive.filesize), archive)) for matched_file_hath in files_matched_hath: total_remote_size = 0 remote_ftp_tuples = [] for img_file_tuple in self.ftps.mlsd(path=matched_file_hath[0], facts=["type", "size"]): if img_file_tuple[1]["type"] != 'file' or img_file_tuple[0] == 'galleryinfo.txt': continue total_remote_size += int(img_file_tuple[1]["size"]) remote_ftp_tuples.append((img_file_tuple[0], img_file_tuple[1]["size"])) if total_remote_size != matched_file_hath[2]: self.logger.info( "For archive: {archive}, remote folder: {folder} " "has not completed the download ({current}/{total}), skipping".format( archive=matched_file_hath[3], folder=matched_file_hath[0], current=filesizeformat(total_remote_size), total=filesizeformat(matched_file_hath[2]) ) ) continue self.logger.info( "For archive: {archive}, downloading and creating zip " "for folder {filename}, {image_count} images".format( archive=matched_file_hath[3], filename=matched_file_hath[1], image_count=len(remote_ftp_tuples) )) dir_path = mkdtemp() self.current_download['total'] = len(remote_ftp_tuples) for count, remote_file in enumerate(sorted(remote_ftp_tuples), start=1): for retry_count in range(10): try: with open(os.path.join(dir_path, remote_file[0]), "wb") as file: self.current_download['index'] = count self.write_file_update_progress( 'RETR %s' % (str(matched_file_hath[0]) + "/" + remote_file[0]), file.write, int(remote_file[1]) ) except (ConnectionResetError, socket.timeout, TimeoutError): self.logger.error("Hath download failed for file {} of {}, restarting connection...".format( count, len(remote_ftp_tuples)) ) self.ftps.close() self.start_connection() self.set_current_dir(self.settings.providers['panda'].remote_hath_dir) else: break with ZipFile(os.path.join(self.settings.MEDIA_ROOT, matched_file_hath[1]), 'w') as archive_file: for (root_path, _, file_names) in os.walk(dir_path): for current_file in file_names: archive_file.write( os.path.join(root_path, current_file), arcname=os.path.basename(current_file)) shutil.rmtree(dir_path, ignore_errors=True) self.process_downloaded_archive(matched_file_hath[3]) # Torrent downloads if len(files_torrent) > 0: self.set_current_dir(self.settings.ftps['remote_torrent_dir']) self.ftps.encoding = 'utf8' files_matched_torrent = [] for line in self.ftps.mlsd(facts=["type", "size"]): if not line[0]: continue if 'type' not in line[1]: continue if line[1]["type"] != 'dir' and line[1]["type"] != 'file': continue for archive in files_torrent: if archive.gallery: cleaned_torrent_name = os.path.splitext( os.path.basename(archive.zipped.path))[0].replace(' [' + archive.gallery.gid + ']', '') else: cleaned_torrent_name = os.path.splitext(os.path.basename(archive.zipped.path))[0] if replace_illegal_name(os.path.splitext(line[0])[0]) in cleaned_torrent_name: if line[1]["type"] == 'dir': files_matched_torrent.append((line[0], line[1]["type"], 0, archive)) else: files_matched_torrent.append((line[0], line[1]["type"], int(line[1]["size"]), archive)) for matched_file_torrent in files_matched_torrent: if matched_file_torrent[1] == 'dir': dir_path = mkdtemp() remote_ftp_files = list(self.ftps.mlsd(path=matched_file_torrent[0], facts=["type", "size"])) self.current_download['total'] = len(remote_ftp_files) self.logger.info( "For archive: {archive}, downloading and creating zip " "for folder {filename}, {image_count} images".format( archive=matched_file_torrent[3], filename=matched_file_torrent[0], image_count=len(remote_ftp_files) )) for count, img_file_tuple in enumerate(remote_ftp_files): if img_file_tuple[1]["type"] != 'file': continue for retry_count in range(10): try: with open(os.path.join(dir_path, img_file_tuple[0]), "wb") as file: self.current_download['index'] = count self.write_file_update_progress( 'RETR %s' % (str(matched_file_torrent[0]) + "/" + img_file_tuple[0]), file.write, int(img_file_tuple[1]["size"]) ) except (ConnectionResetError, socket.timeout, TimeoutError): self.logger.error("Torrent download failed for folder, restarting connection...") self.ftps.close() self.start_connection() self.set_current_dir(self.settings.ftps['remote_torrent_dir']) else: break with ZipFile(matched_file_torrent[3].zipped.path, 'w') as archive_file: for (root_path, _, file_names) in os.walk(dir_path): for current_file in file_names: archive_file.write( os.path.join(root_path, current_file), arcname=os.path.basename(current_file)) shutil.rmtree(dir_path, ignore_errors=True) else: self.logger.info( "For archive: {archive} downloading remote file: {remote} to local file: {local}".format( archive=matched_file_torrent[3], remote=matched_file_torrent[0], local=matched_file_torrent[3].zipped.path ) ) self.current_download['total'] = 1 for retry_count in range(10): try: with open(matched_file_torrent[3].zipped.path, "wb") as file: self.current_download['index'] = 1 self.write_file_update_progress( 'RETR %s' % matched_file_torrent[0], file.write, matched_file_torrent[2]) except (ConnectionResetError, socket.timeout, TimeoutError): self.logger.error("Torrent download failed for archive, restarting connection...") self.ftps.close() self.start_connection() self.set_current_dir(self.settings.ftps['remote_torrent_dir']) else: break if self.settings.convert_rar_to_zip and os.path.splitext(matched_file_torrent[0])[1].lower() == ".rar": self.logger.info( "For archive: {}, converting rar: {} to zip".format( matched_file_torrent[3], matched_file_torrent[3].zipped.path ) ) convert_rar_to_zip(matched_file_torrent[3].zipped.path) self.process_downloaded_archive(matched_file_torrent[3]) self.ftps.close() def copy_all_missing(self, mode, archives: Iterable[Archive] = None): files_torrent = [] files_hath = [] if not archives: found_archives: Iterable[Archive] = list(Archive.objects.filter_by_dl_remote()) else: found_archives = archives if not found_archives: return for archive in found_archives: if not os.path.isfile(archive.zipped.path): if 'torrent' in archive.match_type: files_torrent.append(archive) elif 'hath' in archive.match_type: files_hath.append(archive) if len(files_torrent) + len(files_hath) == 0: return # Hath downloads if len(files_hath) > 0: files_matched_hath = [] for matched_file in os.listdir(self.settings.providers['panda'].local_hath_folder): if os.path.isfile(os.path.join(self.settings.providers['panda'].local_hath_folder, matched_file)): continue m = re.search(r'.*?\[(\d+)\]$', matched_file) if m: for archive in files_hath: if m.group(1) == archive.gallery.gid: files_matched_hath.append( [matched_file, archive.zipped.path, int(archive.filesize), archive]) for img_dir in files_matched_hath: total_remote_size = 0 remote_files = [] directory = os.path.join(self.settings.providers['panda'].local_hath_folder, img_dir[0]) for img_file in os.listdir(directory): if not os.path.isfile(os.path.join(directory, img_file)) or img_file == 'galleryinfo.txt': continue total_remote_size += os.stat( os.path.join(directory, img_file)).st_size remote_files.append( os.path.join(directory, img_file)) if total_remote_size != img_dir[2]: self.logger.info( "For archive: {archive}, folder: {folder} " "has not completed the download ({current}/{total}), skipping".format( archive=img_dir[3], folder=img_dir[0], current=filesizeformat(total_remote_size), total=filesizeformat(img_dir[2]) ) ) continue self.logger.info( "For archive: {archive}, creating zip " "for folder {filename}, {image_count} images".format( archive=img_dir[3], filename=img_dir[1], image_count=len(remote_files) )) dir_path = mkdtemp() for img_file_original in remote_files: img_file = os.path.split(img_file_original)[1] if mode == 'local_move': shutil.move(img_file_original, os.path.join(dir_path, img_file)) else: shutil.copy(img_file_original, os.path.join(dir_path, img_file)) with ZipFile(os.path.join(self.settings.MEDIA_ROOT, img_dir[1]), 'w') as archive_file: for (root_path, _, file_names) in os.walk(dir_path): for current_file in file_names: archive_file.write( os.path.join(root_path, current_file), arcname=os.path.basename(current_file)) shutil.rmtree(dir_path, ignore_errors=True) self.process_downloaded_archive(img_dir[3]) # Torrent downloads if len(files_torrent) > 0: files_matched_torrent = [] for filename in os.listdir(self.settings.torrent['download_dir']): for archive in files_torrent: if archive.gallery: cleaned_torrent_name = os.path.splitext( os.path.basename(archive.zipped.path))[0].replace(' [' + archive.gallery.gid + ']', '') else: cleaned_torrent_name = os.path.splitext(os.path.basename(archive.zipped.path))[0] if replace_illegal_name(os.path.splitext(filename)[0]) in cleaned_torrent_name: files_matched_torrent.append([filename, not os.path.isfile( os.path.join(self.settings.torrent['download_dir'], filename)), archive]) for matched_file in files_matched_torrent: target = os.path.join(self.settings.torrent['download_dir'], matched_file[0]) if matched_file[1]: self.logger.info( "For archive: {archive}, creating zip for folder: {filename}".format( archive=matched_file[2], filename=matched_file[0], )) dir_path = mkdtemp() for img_file in os.listdir(target): if not os.path.isfile(os.path.join(target, img_file)): continue if mode == 'local_move': shutil.move(os.path.join(target, img_file), os.path.join(dir_path, img_file)) else: shutil.copy(os.path.join(target, img_file), os.path.join(dir_path, img_file)) with ZipFile(matched_file[2].zipped.path, 'w') as archive_file: for (root_path, _, file_names) in os.walk(dir_path): for current_file in file_names: archive_file.write( os.path.join(root_path, current_file), arcname=os.path.basename(current_file)) shutil.rmtree(dir_path, ignore_errors=True) else: self.logger.info( "For archive: {archive}, downloading file: {filename}".format( archive=matched_file[2], filename=matched_file[0], )) if mode == 'local_move': shutil.move(target, matched_file[2].zipped.path) else: shutil.copy(target, matched_file[2].zipped.path) if self.settings.convert_rar_to_zip and os.path.splitext(matched_file[0])[1].lower() == ".rar": self.logger.info( "For archive: {}, converting rar: {} to zip".format( matched_file[2], matched_file[2].zipped.path ) ) convert_rar_to_zip(matched_file[2].zipped.path) self.process_downloaded_archive(matched_file[2]) def transfer_all_missing(self, archives: Iterable[Archive] = None) -> None: if self.settings.download_handler.startswith('local'): self.copy_all_missing(self.settings.download_handler, archives) else: for retry_count in range(3): try: self.download_all_missing(archives) except (ConnectionResetError, socket.timeout, TimeoutError) as e: self.logger.error( "Download failed, restarting connection. Retry: {} of 3. Error: {}".format(retry_count + 1, e) ) else: return self.logger.error("Download failed, restart limit reached (3), ending")
from ftplib import FTP_TLS from ftplib import FTP import os import time import ipaddress import getpass input_server = input("Enter the IP of the server you'd like to connect to:") login_name = input("Enter the username:"******"Enter the password:"******"""Enter the full path to the file including file extension of the file you'd like to upload. Example:\"C:\\Users\\Andreas Stenberg\\Desktop\\FTP-skript.py\"n:""") file = open(file_choice, "rb") try:#Frågar efter server-IP och användarnamn och lösenord ftp.auth() ftp.login(login_name, login_pwd) print(ftp.getwelcome()) ftp.prot_p() ftp.mlsd() ftp.storbinary("STOR {}".format(file_choice), file) time.sleep(5) ftp.quit() file.close() except: ftp.quit()