def _run_interface(self, runtime):
settings_files = self._get_settings()
for ants_settings in settings_files:
interface_result = None
self._config_ants(ants_settings)
NIWORKFLOWS_LOG.info(
'Retry #%d, commandline: \n%s', self.retry, self.norm.cmdline)
try:
interface_result = self.norm.run()
except Exception as exc:
NIWORKFLOWS_LOG.warn(
'Retry #%d failed: %s.', self.retry, exc)
errfile = op.join(runtime.cwd, 'stderr.nipype')
outfile = op.join(runtime.cwd, 'stdout.nipype')
shutil.move(errfile, errfile + '.%03d' % self.retry)
shutil.move(outfile, outfile + '.%03d' % self.retry)
if interface_result is not None:
runtime.returncode = 0
self._results.update(interface_result.outputs.get())
NIWORKFLOWS_LOG.info(
'Successful spatial normalization (retry #%d).', self.retry)
return runtime
self.retry += 1
raise RuntimeError(
'Robust spatial normalization failed after %d retries.' % (self.retry - 1))
def _generate_error_report(self, errno=None):
""" Saves an html snippet """
# as of now we think this will be the same for every interface
NIWORKFLOWS_LOG.warn('Report was not generated')
errorstr = '<div><span class="error">Failed to generate report!</span>.\n'
if errno:
errorstr += (' <span class="error">Interface returned exit '
'code %d</span>.\n') % errno
errorstr += '</div>\n'
with open(self._out_report, 'w' if PY3 else 'wb') as outfile:
outfile.write(errorstr)
def _chunk_read_(response,
local_file,
chunk_size=8192,
report_hook=None,
initial_size=0,
total_size=None,
verbose=1):
"""Download a file chunk by chunk and show advancement
:param urllib.response.addinfourl response: response to the download
request in order to get file size
:param str local_file: hard disk file where data should be written
:param int chunk_size: size of downloaded chunks. Default: 8192
:param bool report_hook: whether or not to show downloading advancement
:param int initial_size: if resuming, indicate the initial size of the file
:param int total_size: Expected final size of download (None means it
is unknown).
:param int verbose: verbosity level (0 means no message).
:returns: the downloaded file path.
:rtype: string
"""
try:
if total_size is None:
total_size = response.info().get('Content-Length').strip()
total_size = int(total_size) + initial_size
except Exception as exc:
if verbose > 2:
NIWORKFLOWS_LOG.warn('Total size of chunk could not be determined')
if verbose > 3:
NIWORKFLOWS_LOG.warn("Full stack trace: %s", str(exc))
total_size = None
bytes_so_far = initial_size
t_0 = time_last_display = time.time()
while True:
chunk = response.read(chunk_size)
bytes_so_far += len(chunk)
time_last_read = time.time()
if (report_hook and
# Refresh report every half second or when download is
# finished.
(time_last_read > time_last_display + 0.5 or not chunk)):
_chunk_report_(bytes_so_far, total_size, initial_size, t_0)
time_last_display = time_last_read
if chunk:
local_file.write(chunk)
else:
break
return
def _fetch_file(url,
dataset_dir,
filetype=None,
resume=True,
overwrite=False,
md5sum=None,
username=None,
password=None,
retry=0,
verbose=1,
temp_downloads=None):
"""Load requested file, downloading it if needed or requested.
:param str url: contains the url of the file to be downloaded.
:param str dataset_dir: path of the data directory. Used for data
storage in the specified location.
:param bool resume: if true, try to resume partially downloaded files
:param overwrite: if bool true and file already exists, delete it.
:param str md5sum: MD5 sum of the file. Checked if download of the file
is required
:param str username: username used for basic HTTP authentication
:param str password: password used for basic HTTP authentication
:param int verbose: verbosity level (0 means no message).
:returns: absolute path of downloaded file
:rtype: str
..note::
If, for any reason, the download procedure fails, all downloaded files are
removed.
"""
if not overwrite and os.listdir(dataset_dir):
return True
data_dir, _ = op.split(dataset_dir)
if temp_downloads is None:
temp_downloads = op.join(NIWORKFLOWS_CACHE_DIR, 'downloads')
# Determine data path
if not op.exists(temp_downloads):
os.makedirs(temp_downloads)
# Determine filename using URL
parse = urlparse(url)
file_name = op.basename(parse.path)
if file_name == '':
file_name = _md5_hash(parse.path)
if filetype is not None:
file_name += filetype
temp_full_name = op.join(temp_downloads, file_name)
temp_part_name = temp_full_name + ".part"
if overwrite:
shutil.rmtree(dataset_dir, ignore_errors=True)
if op.exists(temp_full_name):
if overwrite:
os.remove(temp_full_name)
t_0 = time.time()
local_file = None
initial_size = 0
# Download data
request = Request(url)
request.add_header('Connection', 'Keep-Alive')
if username is not None and password is not None:
if not url.startswith('https'):
raise ValueError(
'Authentication was requested on a non secured URL ({0!s}).'
'Request has been blocked for security reasons.'.format(url))
# Note: HTTPBasicAuthHandler is not fitted here because it relies
# on the fact that the server will return a 401 error with proper
# www-authentication header, which is not the case of most
# servers.
encoded_auth = base64.b64encode((username + ':' + password).encode())
request.add_header(b'Authorization', b'Basic ' + encoded_auth)
if verbose > 0:
displayed_url = url.split('?')[0] if verbose == 1 else url
NIWORKFLOWS_LOG.info('Downloading data from %s ...', displayed_url)
if resume and op.exists(temp_part_name):
# Download has been interrupted, we try to resume it.
local_file_size = op.getsize(temp_part_name)
# If the file exists, then only download the remainder
request.add_header("Range", "bytes={}-".format(local_file_size))
try:
data = urlopen(request)
content_range = data.info().get('Content-Range')
if (content_range is None or not content_range.startswith(
'bytes {}-'.format(local_file_size))):
raise IOError('Server does not support resuming')
except Exception:
# A wide number of errors can be raised here. HTTPError,
# URLError... I prefer to catch them all and rerun without
# resuming.
if verbose > 0:
NIWORKFLOWS_LOG.warn(
'Resuming failed, try to download the whole file.')
return _fetch_file(url,
dataset_dir,
resume=False,
overwrite=overwrite,
md5sum=md5sum,
username=username,
password=password,
verbose=verbose)
local_file = open(temp_part_name, "ab")
initial_size = local_file_size
else:
try:
data = urlopen(request)
except (HTTPError, URLError):
if retry < MAX_RETRIES:
if verbose > 0:
NIWORKFLOWS_LOG.warn(
'Download failed, retrying (attempt %d)', retry + 1)
time.sleep(5)
return _fetch_file(url,
dataset_dir,
resume=False,
overwrite=overwrite,
md5sum=md5sum,
username=username,
password=password,
verbose=verbose,
retry=retry + 1)
else:
raise
local_file = open(temp_part_name, "wb")
_chunk_read_(data,
local_file,
report_hook=(verbose > 0),
initial_size=initial_size,
verbose=verbose)
# temp file must be closed prior to the move
if not local_file.closed:
local_file.close()
shutil.move(temp_part_name, temp_full_name)
delta_t = time.time() - t_0
if verbose > 0:
# Complete the reporting hook
sys.stderr.write(' ...done. ({0:.0f} seconds, {1:.0f} min)\n'.format(
delta_t, delta_t // 60))
if md5sum is not None:
if _md5_sum_file(temp_full_name) != md5sum:
raise ValueError("File {} checksum verification has failed."
" Dataset fetching aborted.".format(local_file))
if filetype is None:
fname, filetype = op.splitext(op.basename(temp_full_name))
if filetype == '.gz':
fname, ext = op.splitext(fname)
filetype = ext + filetype
if filetype.startswith('.'):
filetype = filetype[1:]
if filetype == 'tar':
sp.check_call(['tar', 'xf', temp_full_name], cwd=data_dir)
os.remove(temp_full_name)
return True
return True