def handle_input_files(self, **kwargs): self.task.change_status(status_msg='Uploading input files', status_code=151) self.logger.debug(self.log_prefix + 'Uploading input files') files = SkyLabFile.objects.filter( type=1, task=self.task) # input files for this task self.logger.debug(self.log_prefix + 'Opening SFTP client') sftp = self.shell._open_sftp_client() # open sftp client self.logger.debug(self.log_prefix + 'Opened SFTP client') #no timeouts will be implemented for ray since input files used are too large # sftp.get_channel().settimeout(900.0) # # self.logger.debug(self.log_prefix + "Set timeout to {0}".format(sftp.get_channel().gettimeout())) for f in files: sftp.chdir(self.remote_task_dir) # cd /mirror/task_xx self.logger.debug(self.log_prefix + 'Uploading ' + f.filename) mkdir_p(sftp, f.upload_path) # mimics mkdir -p f.upload_path sftp.putfo(f.file, f.filename, callback=self.sftp_file_transfer_callback ) # At this point, you are f.upload_path self.logger.debug(self.log_prefix + 'Uploaded ' + f.filename) sftp.close() self.logger.debug(self.log_prefix + 'Closed SFTP client')
def handle_input_files(self, **kwargs): self.task.change_status(status_msg='Uploading input files', status_code=151) self.logger.debug(self.log_prefix + 'Uploading input files') files = SkyLabFile.objects.filter(type=1, task=self.task) # input files for this task self.logger.debug(self.log_prefix + 'Opening SFTP client') sftp = self.shell._open_sftp_client() # open sftp client self.logger.debug(self.log_prefix + 'Opened SFTP client') sftp.get_channel().settimeout(180.0) # set timeout to 3 mins self.logger.debug(self.log_prefix + "Set timeout to {0}".format(sftp.get_channel().gettimeout())) for f in files: sftp.chdir(self.remote_task_dir) # cd /mirror/task_xx mkdir_p(sftp, f.upload_path) # mimics mkdir -p f.upload_path while True: try: self.logger.debug(self.log_prefix + "Uploading " + f.filename) sftp.putfo(f.file, f.filename, callback=self.sftp_file_transfer_callback) # copy file object to cluster as f.filename in the current dir self.logger.debug(self.log_prefix + "Uploaded " + f.filename) break except (socket.timeout, EOFError): self.logger.debug(self.log_prefix + "Retrying for " + f.filename) time.sleep(2) sftp.close() self.logger.debug(self.log_prefix + 'Closed SFTP client')
def handle_input_files(self, **kwargs): self.task.change_status(status_msg='Uploading input files', status_code=151) self.logger.debug(self.log_prefix + 'Uploading input files') files = SkyLabFile.objects.filter(type=1, task=self.task) # input files for this task self.logger.debug(self.log_prefix + 'Opening SFTP client') sftp = self.shell._open_sftp_client() # open sftp client self.logger.debug(self.log_prefix + 'Opened SFTP client') #no timeouts will be implemented for ray since input files used are too large # sftp.get_channel().settimeout(900.0) # # self.logger.debug(self.log_prefix + "Set timeout to {0}".format(sftp.get_channel().gettimeout())) for f in files: sftp.chdir(self.remote_task_dir) # cd /mirror/task_xx self.logger.debug(self.log_prefix + 'Uploading ' + f.filename ) mkdir_p(sftp, f.upload_path) # mimics mkdir -p f.upload_path sftp.putfo(f.file, f.filename, callback=self.sftp_file_transfer_callback) # At this point, you are f.upload_path self.logger.debug(self.log_prefix + 'Uploaded ' + f.filename) sftp.close() self.logger.debug(self.log_prefix + 'Closed SFTP client')