def handle_input_files(self, **kwargs):
self.task.change_status(status_msg='Uploading input files',
status_code=151)
self.logger.debug(self.log_prefix + 'Uploading input files')
files = SkyLabFile.objects.filter(
type=1, task=self.task) # input files for this task
self.logger.debug(self.log_prefix + 'Opening SFTP client')
sftp = self.shell._open_sftp_client() # open sftp client
self.logger.debug(self.log_prefix + 'Opened SFTP client')
#no timeouts will be implemented for ray since input files used are too large
# sftp.get_channel().settimeout(900.0) #
# self.logger.debug(self.log_prefix + "Set timeout to {0}".format(sftp.get_channel().gettimeout()))
for f in files:
sftp.chdir(self.remote_task_dir) # cd /mirror/task_xx
self.logger.debug(self.log_prefix + 'Uploading ' + f.filename)
mkdir_p(sftp, f.upload_path) # mimics mkdir -p f.upload_path
sftp.putfo(f.file,
f.filename,
callback=self.sftp_file_transfer_callback
) # At this point, you are f.upload_path
self.logger.debug(self.log_prefix + 'Uploaded ' + f.filename)
sftp.close()
self.logger.debug(self.log_prefix + 'Closed SFTP client')
def handle_input_files(self, **kwargs):
self.task.change_status(status_msg='Uploading input files', status_code=151)
self.logger.debug(self.log_prefix + 'Uploading input files')
files = SkyLabFile.objects.filter(type=1, task=self.task) # input files for this task
self.logger.debug(self.log_prefix + 'Opening SFTP client')
sftp = self.shell._open_sftp_client() # open sftp client
self.logger.debug(self.log_prefix + 'Opened SFTP client')
sftp.get_channel().settimeout(180.0) # set timeout to 3 mins
self.logger.debug(self.log_prefix + "Set timeout to {0}".format(sftp.get_channel().gettimeout()))
for f in files:
sftp.chdir(self.remote_task_dir) # cd /mirror/task_xx
mkdir_p(sftp, f.upload_path) # mimics mkdir -p f.upload_path
while True:
try:
self.logger.debug(self.log_prefix + "Uploading " + f.filename)
sftp.putfo(f.file, f.filename, callback=self.sftp_file_transfer_callback) # copy file object to cluster as f.filename in the current dir
self.logger.debug(self.log_prefix + "Uploaded " + f.filename)
break
except (socket.timeout, EOFError):
self.logger.debug(self.log_prefix + "Retrying for " + f.filename)
time.sleep(2)
sftp.close()
self.logger.debug(self.log_prefix + 'Closed SFTP client')
def handle_input_files(self, **kwargs):
self.task.change_status(status_msg='Uploading input files', status_code=151)
self.logger.debug(self.log_prefix + 'Uploading input files')
files = SkyLabFile.objects.filter(type=1, task=self.task) # input files for this task
self.logger.debug(self.log_prefix + 'Opening SFTP client')
sftp = self.shell._open_sftp_client() # open sftp client
self.logger.debug(self.log_prefix + 'Opened SFTP client')
sftp.get_channel().settimeout(180.0) # set timeout to 3 mins
self.logger.debug(self.log_prefix + "Set timeout to {0}".format(sftp.get_channel().gettimeout()))
for f in files:
sftp.chdir(self.remote_task_dir) # cd /mirror/task_xx
mkdir_p(sftp, f.upload_path) # mimics mkdir -p f.upload_path
while True:
try:
self.logger.debug(self.log_prefix + "Uploading " + f.filename)
sftp.putfo(f.file, f.filename, callback=self.sftp_file_transfer_callback) # copy file object to cluster as f.filename in the current dir
self.logger.debug(self.log_prefix + "Uploaded " + f.filename)
break
except (socket.timeout, EOFError):
self.logger.debug(self.log_prefix + "Retrying for " + f.filename)
time.sleep(2)
sftp.close()
self.logger.debug(self.log_prefix + 'Closed SFTP client')
def handle_input_files(self, **kwargs):
self.task.change_status(status_msg='Uploading input files', status_code=151)
self.logger.debug(self.log_prefix + 'Uploading input files')
files = SkyLabFile.objects.filter(type=1, task=self.task) # input files for this task
self.logger.debug(self.log_prefix + 'Opening SFTP client')
sftp = self.shell._open_sftp_client() # open sftp client
self.logger.debug(self.log_prefix + 'Opened SFTP client')
#no timeouts will be implemented for ray since input files used are too large
# sftp.get_channel().settimeout(900.0) #
# self.logger.debug(self.log_prefix + "Set timeout to {0}".format(sftp.get_channel().gettimeout()))
for f in files:
sftp.chdir(self.remote_task_dir) # cd /mirror/task_xx
self.logger.debug(self.log_prefix + 'Uploading ' + f.filename )
mkdir_p(sftp, f.upload_path) # mimics mkdir -p f.upload_path
sftp.putfo(f.file, f.filename, callback=self.sftp_file_transfer_callback) # At this point, you are f.upload_path
self.logger.debug(self.log_prefix + 'Uploaded ' + f.filename)
sftp.close()
self.logger.debug(self.log_prefix + 'Closed SFTP client')
