def _run_interface(self, runtime): settings_files = self._get_settings() for ants_settings in settings_files: interface_result = None self._config_ants(ants_settings) NIWORKFLOWS_LOG.info( 'Retry #%d, commandline: \n%s', self.retry, self.norm.cmdline) try: interface_result = self.norm.run() except Exception as exc: NIWORKFLOWS_LOG.warn( 'Retry #%d failed: %s.', self.retry, exc) errfile = op.join(runtime.cwd, 'stderr.nipype') outfile = op.join(runtime.cwd, 'stdout.nipype') shutil.move(errfile, errfile + '.%03d' % self.retry) shutil.move(outfile, outfile + '.%03d' % self.retry) if interface_result is not None: runtime.returncode = 0 self._results.update(interface_result.outputs.get()) NIWORKFLOWS_LOG.info( 'Successful spatial normalization (retry #%d).', self.retry) return runtime self.retry += 1 raise RuntimeError( 'Robust spatial normalization failed after %d retries.' % (self.retry - 1))
def _generate_error_report(self, errno=None): """ Saves an html snippet """ # as of now we think this will be the same for every interface NIWORKFLOWS_LOG.warn('Report was not generated') errorstr = '<div><span class="error">Failed to generate report!</span>.\n' if errno: errorstr += (' <span class="error">Interface returned exit ' 'code %d</span>.\n') % errno errorstr += '</div>\n' with open(self._out_report, 'w' if PY3 else 'wb') as outfile: outfile.write(errorstr)
def _chunk_read_(response, local_file, chunk_size=8192, report_hook=None, initial_size=0, total_size=None, verbose=1): """Download a file chunk by chunk and show advancement :param urllib.response.addinfourl response: response to the download request in order to get file size :param str local_file: hard disk file where data should be written :param int chunk_size: size of downloaded chunks. Default: 8192 :param bool report_hook: whether or not to show downloading advancement :param int initial_size: if resuming, indicate the initial size of the file :param int total_size: Expected final size of download (None means it is unknown). :param int verbose: verbosity level (0 means no message). :returns: the downloaded file path. :rtype: string """ try: if total_size is None: total_size = response.info().get('Content-Length').strip() total_size = int(total_size) + initial_size except Exception as exc: if verbose > 2: NIWORKFLOWS_LOG.warn('Total size of chunk could not be determined') if verbose > 3: NIWORKFLOWS_LOG.warn("Full stack trace: %s", str(exc)) total_size = None bytes_so_far = initial_size t_0 = time_last_display = time.time() while True: chunk = response.read(chunk_size) bytes_so_far += len(chunk) time_last_read = time.time() if (report_hook and # Refresh report every half second or when download is # finished. (time_last_read > time_last_display + 0.5 or not chunk)): _chunk_report_(bytes_so_far, total_size, initial_size, t_0) time_last_display = time_last_read if chunk: local_file.write(chunk) else: break return
def _fetch_file(url, dataset_dir, filetype=None, resume=True, overwrite=False, md5sum=None, username=None, password=None, retry=0, verbose=1, temp_downloads=None): """Load requested file, downloading it if needed or requested. :param str url: contains the url of the file to be downloaded. :param str dataset_dir: path of the data directory. Used for data storage in the specified location. :param bool resume: if true, try to resume partially downloaded files :param overwrite: if bool true and file already exists, delete it. :param str md5sum: MD5 sum of the file. Checked if download of the file is required :param str username: username used for basic HTTP authentication :param str password: password used for basic HTTP authentication :param int verbose: verbosity level (0 means no message). :returns: absolute path of downloaded file :rtype: str ..note:: If, for any reason, the download procedure fails, all downloaded files are removed. """ if not overwrite and os.listdir(dataset_dir): return True data_dir, _ = op.split(dataset_dir) if temp_downloads is None: temp_downloads = op.join(NIWORKFLOWS_CACHE_DIR, 'downloads') # Determine data path if not op.exists(temp_downloads): os.makedirs(temp_downloads) # Determine filename using URL parse = urlparse(url) file_name = op.basename(parse.path) if file_name == '': file_name = _md5_hash(parse.path) if filetype is not None: file_name += filetype temp_full_name = op.join(temp_downloads, file_name) temp_part_name = temp_full_name + ".part" if overwrite: shutil.rmtree(dataset_dir, ignore_errors=True) if op.exists(temp_full_name): if overwrite: os.remove(temp_full_name) t_0 = time.time() local_file = None initial_size = 0 # Download data request = Request(url) request.add_header('Connection', 'Keep-Alive') if username is not None and password is not None: if not url.startswith('https'): raise ValueError( 'Authentication was requested on a non secured URL ({0!s}).' 'Request has been blocked for security reasons.'.format(url)) # Note: HTTPBasicAuthHandler is not fitted here because it relies # on the fact that the server will return a 401 error with proper # www-authentication header, which is not the case of most # servers. encoded_auth = base64.b64encode((username + ':' + password).encode()) request.add_header(b'Authorization', b'Basic ' + encoded_auth) if verbose > 0: displayed_url = url.split('?')[0] if verbose == 1 else url NIWORKFLOWS_LOG.info('Downloading data from %s ...', displayed_url) if resume and op.exists(temp_part_name): # Download has been interrupted, we try to resume it. local_file_size = op.getsize(temp_part_name) # If the file exists, then only download the remainder request.add_header("Range", "bytes={}-".format(local_file_size)) try: data = urlopen(request) content_range = data.info().get('Content-Range') if (content_range is None or not content_range.startswith( 'bytes {}-'.format(local_file_size))): raise IOError('Server does not support resuming') except Exception: # A wide number of errors can be raised here. HTTPError, # URLError... I prefer to catch them all and rerun without # resuming. if verbose > 0: NIWORKFLOWS_LOG.warn( 'Resuming failed, try to download the whole file.') return _fetch_file(url, dataset_dir, resume=False, overwrite=overwrite, md5sum=md5sum, username=username, password=password, verbose=verbose) local_file = open(temp_part_name, "ab") initial_size = local_file_size else: try: data = urlopen(request) except (HTTPError, URLError): if retry < MAX_RETRIES: if verbose > 0: NIWORKFLOWS_LOG.warn( 'Download failed, retrying (attempt %d)', retry + 1) time.sleep(5) return _fetch_file(url, dataset_dir, resume=False, overwrite=overwrite, md5sum=md5sum, username=username, password=password, verbose=verbose, retry=retry + 1) else: raise local_file = open(temp_part_name, "wb") _chunk_read_(data, local_file, report_hook=(verbose > 0), initial_size=initial_size, verbose=verbose) # temp file must be closed prior to the move if not local_file.closed: local_file.close() shutil.move(temp_part_name, temp_full_name) delta_t = time.time() - t_0 if verbose > 0: # Complete the reporting hook sys.stderr.write(' ...done. ({0:.0f} seconds, {1:.0f} min)\n'.format( delta_t, delta_t // 60)) if md5sum is not None: if _md5_sum_file(temp_full_name) != md5sum: raise ValueError("File {} checksum verification has failed." " Dataset fetching aborted.".format(local_file)) if filetype is None: fname, filetype = op.splitext(op.basename(temp_full_name)) if filetype == '.gz': fname, ext = op.splitext(fname) filetype = ext + filetype if filetype.startswith('.'): filetype = filetype[1:] if filetype == 'tar': sp.check_call(['tar', 'xf', temp_full_name], cwd=data_dir) os.remove(temp_full_name) return True return True