def setUp(self):
self.filename = 'filenameonlytobeusedfortestingreally.conphig'
try:
os.remove(self.filename)
except:
pass
self.profile = ProfileConfig(self.filename)
def get_socket(host,port,bufsize):
# Find host,port from profiles.config
profile = ProfileConfig()
if host is None:
host = profile.get_config_val('matlab.host')
if port is None:
port = profile.get_config_val('matlab.port')
if bufsize is None:
bufsize = profile.get_config_val('matlab.bufsize')
# Create a socket (SOCK_STREAM means a TCP socket)
sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
return(sock,host,port,bufsize)
def get_socket(host, port, bufsize):
'''Open a socket to the machine running MATLAB.
Parameters
----------
host : str
IP address of machine running MATLAB.
port : int
port to connect to.
bufsize : int
Buffer size to use for communication.
Returns
-------
sock : socket.socket
TCP socket for communication
host : str
host ip address
port : int
Port we're communicating over
bufsize : int
Buffer size to use during communication
Notes
-----
If values are not provided (i.e., None) the values for
host, port, bufsize will be taken from the active profile in
profiles.config.
'''
# Find host,port from profiles.config
profile = ProfileConfig()
if host is None:
host = profile.get_config_val('matlab.host')
if port is None:
port = profile.get_config_val('matlab.port')
if bufsize is None:
bufsize = profile.get_config_val('matlab.bufsize')
# Create a socket (SOCK_STREAM means a TCP socket)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
return (sock, host, port, bufsize)
def start_server():
# Find host,port from profiles.config
profile = ProfileConfig()
host = profile.get_config_val('matlab.host')
port = profile.get_config_val('matlab.port')
# Start an instance of MATLAB
try:
matlab = MATLAB()
server = socketserver.TCPServer((host,port),MyTCPHandler)
server.matlab = matlab
# Activate the server; this will keep running until you
# interrupt the program with Ctrl-C
logging.info('Server running on %s:%d' % (host,port))
logging.info('Interrupt the server with Ctrl-C')
server.serve_forever()
finally:
logging.info('Just a sec, stopping matlab and freeing up ports...')
matlab.exit()
def start_server():
'''Start the server so the client can connect.
Notes
=====
This server must be running on the remote before client can be used to
connect to it.
Examples
========
To run this server, simply run:
.. code-block:: bash
python3 mr_utils/matlab/start_server.py
'''
# Find host,port from profiles.config
profile = ProfileConfig()
host = profile.get_config_val('matlab.host')
port = profile.get_config_val('matlab.port')
# Start an instance of MATLAB
try:
matlab = MATLAB()
server = socketserver.TCPServer((host, port), MyTCPHandler)
server.matlab = matlab
# Activate the server; this will keep running until you
# interrupt the program with Ctrl-C
logging.info('Server running on %s:%d', host, port)
logging.info('Interrupt the server with Ctrl-C')
server.serve_forever()
finally:
logging.info('Just a sec, stopping matlab and freeing up ports...')
matlab.exit()
def client(num_out,
cmd,
files,
host=None,
username=None,
password=None,
root_dir=None):
'''BART client.
Parameters
==========
num_out : int
Number of expected variables returned.
cmd : str
BART command to be run.
files : list
Any files to be provided to BART.
host : str, optional
IP address of machine we want to connect to.
username : str, optional
username to sign in with.
password : str, optional
password to use for sign in (will be plain-text!)
root_dir : str, optional
Root directory to run BART out of.
'''
# Grab credentials
profile = ProfileConfig()
if host is None:
host = profile.get_config_val('bart.host')
if username is None:
username = profile.get_config_val('bart.user')
if password is None:
password = profile.get_config_val('bart.password')
if root_dir is None:
root_dir = profile.get_config_val('bart.root_dir')
# Connect to host
ssh_client = paramiko.SSHClient()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_client.connect(hostname=host, username=username, password=password)
# Send the command
stdin, stdout, stderr = ssh_client.exec_command(
("python3 -c 'import sys; sys.path.insert(0,\"%s/python\");"
"from bart import bart; import numpy as np;"
"print(bart)' " % (root_dir)))
print(stdout.readlines())
ssh_client.close()
def client(num_out,
cmd,
files,
host=None,
username=None,
password=None,
root_dir=None):
# Grab credentials
profile = ProfileConfig()
if host is None:
host = profile.get_config_val('bart.host')
if username is None:
username = profile.get_config_val('bart.user')
if password is None:
password = profile.get_config_val('bart.password')
if root_dir is None:
root_dir = profile.get_config_val('bart.root_dir')
# print(cmd)
ssh_client = paramiko.SSHClient()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_client.connect(hostname=host, username=username, password=password)
# tmp_files = []
# for file in files:
# tmp_files.append(NamedTemporaryFile().name)
# stdin,stdout,stderr = ssh_client.exec_command('%s/bart %s' % (root_dir,cmd))
# print("python3 -c 'import sys; sys.path.insert(0,\"%s/python\"); from bart import bart; bart(%d,\"%s\")' " % (root_dir,num_out,cmd))
stdin, stdout, stderr = ssh_client.exec_command(
"python3 -c 'import sys; sys.path.insert(0,\"%s/python\"); from bart import bart; import numpy as np; print(bart)' "
% (root_dir))
print(stdout.readlines())
# print(stderr.readlines())
ssh_client.close()
class ProfileConfigTestCase(unittest.TestCase):
def setUp(self):
self.filename = 'filenameonlytobeusedfortestingreally.conphig'
try:
os.remove(self.filename)
except:
pass
self.profile = ProfileConfig(self.filename)
def tearDown(self):
os.remove(self.filename)
def test_create_profile(self):
self.profile.create_profile('mine')
with self.assertRaises(RuntimeError):
self.profile.create_profile('mine')
def test_get_val(self):
self.assertTrue(
self.profile.get_config_val('gadgetron.host') == 'localhost')
def test_activate_profile(self):
self.profile.create_profile('mine')
self.profile.activate_profile('default')
self.assertTrue(self.profile.active_profile == 'default')
self.profile.activate_profile('mine')
self.assertTrue(self.profile.active_profile == 'mine')
def test_set_config(self):
self.profile.create_profile('workcomp')
self.profile.activate_profile('workcomp')
self.profile.set_config({'gadgetron.port': 8080})
self.assertTrue(self.profile.get_config_val('gadgetron.port') == 8080)
def s2i_client(
filename,
put_file=True,
get_file=True,
cleanup_raw=True,
cleanup_processed=True,
remote_dir='/tmp',
host=None,
port=22,
username=None,
ssh_key=None,
password=None,
debug_level=logging.INFO):
'''Runs siemens_to_ismrmrd on a remote computer.
Main idea: allow users to use siemens_to_ismrmrd even if they don't have
it installed locally. They will, however, require SSH access to computer
that does have it installed.
Client puts file on server using SFTP, runs siemens_to_ismrmrd over SSH,
and gets the file back using SFTP. Username, password, hostname, and port
is retrieved from the active profile in profiles.config. Default port is
22. If no password is found, the RSA SSH key will be used from either the
specified directory in profiles.config or, if empty, use '~/.ssh/id_rsa'.
filename -- Raw data (.dat) file on the local machine (if put_file is True)
or on the remote machine (if put_file is False).
put_file -- Whether or not to copy the raw data file from local to remote.
get_file -- Whether or not to copy the processed file from machine to local.
cleanup_raw -- Whether or not to delete raw data on remote.
cleanup_processed -- Whether or not to delete processed data on remote.
remote_dir -- Working directory on remote (default in /tmp).
host -- hostname of remote machine.
port -- Port of remote machine to connect to.
username -- Username to use for SSH/SFTP connections.
ssh_key -- RSA private key file to use for SSH/SFTP connections.
password -- Password to use fr SSH/SFTP connections (stored in plaintext).
debug_level -- Level of verbosity; see python logging module.
'''
# Setup logging
logging.basicConfig(format='%(levelname)s: %(message)s',level=debug_level)
# Grab credentials
profile = ProfileConfig()
if host is None:
host = profile.get_config_val('siemens_to_ismrmrd.host')
logging.info('profiles.config: using hostname %s' % host)
if port is None:
host = profile.get_config_val('siemens_to_ismrmrd.port')
logging.info('profiles.config: using port %s' % str(port))
if username is None:
username = profile.get_config_val('siemens_to_ismrmrd.user')
logging.info('profiles.config: using username %s' % username)
if password is None:
password = profile.get_config_val('siemens_to_ismrmrd.password')
# If blank, assume no password
if password == '':
password = None
else:
logging.warning('profiles.config: using password stored in plaintext!')
logging.warning('Suggested to use RSA key for connections!')
# So now look for the RSA key
if ssh_key is None:
ssh_key = profile.get_config_val('siemens_to_ismrmrd.ssh_key')
if ssh_key == '':
ssh_key = '%s/.ssh/id_rsa' % os.environ['HOME']
logging.info('Using defaut RSA key %s' % ssh_key)
else:
logging.info('profiles.config: using RSA key %s' % ssh_key)
try:
ssh_conn = FastTransport((host,port))
ssh_conn.use_compression(True)
if ssh_key is not None:
ssh_conn.connect(pkey=paramiko.RSAKey.from_private_key_file(ssh_key),username=username)
else:
ssh_conn.connect(username=username,password=password)
sftp = paramiko.SFTPClient.from_transport(ssh_conn)
if put_file:
# Upload file to the server
remote_filename = '%s/siemens_to_ismrmrd_%s' % (remote_dir,str(uuid.uuid4()))
logging.info('Starting transfer of %s to %s:%s:%s...',filename,host,str(port),remote_filename)
with TqdmWrap(ascii=True,unit='b',unit_scale=True) as pbar:
# this can be pretty slow
sftp.put(filename,remote_filename,callback=pbar.viewBar)
else:
# No file to transfer, then the filename is the name of the file
# we need in the working directory on the remote.
remote_filename = '%s/%s' % (remote_dir,filename)
logging.info('Not transferring, looking for remote file %s:%s:%s',host,str(port),remote_filename)
# Run siemens_to_ismrmrd on remote
ssh_client = paramiko.SSHClient()
ssh_client.load_system_host_keys()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
if ssh_key is not None:
ssh_client.connect(hostname=host,port=port,username=username)
else:
ssh_client.connect(hostname=host,port=port,username=username,password=password)
logging.info('SSH client succesfully connected!')
processed_filename = '%s_processed' % remote_filename
cmd = 'siemens_to_ismrmrd -f %s -o %s' % (remote_filename,processed_filename)
logging.info('Running \"%s\" on remote...' % cmd)
stdin,stdout,stderr = ssh_client.exec_command(cmd)
for line in stdout.read().decode().splitlines():
logging.info(line)
if len(stderr.readlines()):
logging.error(stderr.read().decode())
# Copy the processed file
if get_file:
tmp_name = NamedTemporaryFile().name
logging.info('Transferring processed file back to local machine...')
with TqdmWrap(ascii=True,unit='b',unit_scale=True) as pbar:
sftp.get(processed_filename,tmp_name,callback=pbar.viewBar)
dset = ismrmrd.Dataset(tmp_name,'/dataset',False)
else:
logging.info('Not transferring processed file back from remote')
dset = None
# Clean files from server
if cleanup_raw:
logging.info('Cleaning up raw data on remote')
sftp.remove(remote_filename)
if cleanup_processed:
logging.info('Cleaning up processed data on remote')
sftp.remove(processed_filename)
finally:
ssh_client.close()
sftp.close()
ssh_conn.close()
return(dset)
class ProfileConfigTestCase(unittest.TestCase):
'''Sanity checks for generation of profiles.config.'''
def setUp(self):
self.filename = 'filenameonlytobeusedfortestingreally.conphig'
try:
os.remove(self.filename)
except OSError:
pass
self.profile = ProfileConfig(self.filename)
def tearDown(self):
os.remove(self.filename)
def test_create_profile(self):
'''Create dummy profile and verify its existence.'''
self.profile.create_profile('mine')
with self.assertRaises(RuntimeError):
self.profile.create_profile('mine')
def test_get_val(self):
'''Make sure default value is accessible through get_config_val.'''
self.assertTrue(
self.profile.get_config_val('gadgetron.host') == 'localhost')
def test_activate_profile(self):
'''Make sure that profiles can be activated.'''
self.profile.create_profile('mine')
self.profile.activate_profile('default')
self.assertTrue(self.profile.active_profile == 'default')
self.profile.activate_profile('mine')
self.assertTrue(self.profile.active_profile == 'mine')
def test_set_config(self):
'''Set a configuration value.'''
self.profile.create_profile('workcomp')
self.profile.activate_profile('workcomp')
self.profile.set_config({'gadgetron.port': 8080})
self.assertTrue(self.profile.get_config_val('gadgetron.port') == 8080)
def client(data,
address=None,
port=None,
outfile=None,
in_group='/dataset',
out_group=None,
config='default.xml',
config_local=None,
script=None,
existing_modules=['numpy', 'scipy', 'h5py'],
script_dir=None,
verbose=False):
'''Send acquisitions to Gadgetron.
This client allows you to connect to a Gadgetron server and process data.
Parameters
==========
data : str or array_like
Input file with file extension or numpy array.
address : str, optional
Hostname of Gadgetron. If not set, taken from profile config.
port : int, optional
Port to connect to. If not set, taken from profile config.
outfile : str, optional
If provided, output will be saved to file with this name.
in_group : str, optional
If input is hdf5, input data group name.
out_group : str, optional
Output group name if file is written.
config : xml_str, optional
Remote configuration file.
config_local : xml_str, optional
Local configuration file.
script : str, optional
File path to the Python script to be bundled and transfered.
existing_modules : list, optional
Python packages to exclude from bundling.
script_dir : str, optional
Directory to send script on remote machine.
verbose : bool, optional
Verbose mode.
Returns
=======
data : array_like
Image from Gadgetron
header : xml
Header from Gadgetron
Raises
======
NotImplementedError
`script` bundling is not currently implemented.
Exception
`data` is not provided in the correct format.
Notes
=====
out_group=None will use the current date as the group name.
'''
# Make sure we have an out_group label
if out_group is None:
out_group = str(datetime.datetime.now())
# If user wanted to be verbose, let's give them verbose, otherwise just
# tell them about warnings
if verbose:
logging.basicConfig(format='%(levelname)s: %(message)s',
level=logging.DEBUG)
else:
logging.basicConfig(format='%(levelname)s: %(message)s',
level=logging.WARNING)
# First thing's first, we need to send the python script over!
if script is not None:
raise NotImplementedError()
# from mr_utils.utils import package_script
# package_script(script, existing_modules=existing_modules)
# The magic happens in the connector
logging.debug('Instantiating Connector')
con = gt.Connector()
## Register all the readers we need:
# The readers need to know where to output the data that gadgetron sends
# back. The output is an hdf5 file, but if we don't want that file and
# only care about the numpy array, then we'll create a temporary file to
# store the output in and then kill it when we're done.
if outfile is None:
outfile = NamedTemporaryFile().name
logging.debug('Writing to filename: %s', outfile)
# Image message readers
for im_id in [
gt.GADGET_MESSAGE_ISMRMRD_IMAGE_REAL_USHORT,
gt.GADGET_MESSAGE_ISMRMRD_IMAGE_REAL_FLOAT,
gt.GADGET_MESSAGE_ISMRMRD_IMAGE_CPLX_FLOAT,
gt.GADGET_MESSAGE_ISMRMRD_IMAGE
]:
con.register_reader(im_id, gt.ImageMessageReader(outfile, out_group))
# Images with attributes
for im_id in [
gt.GADGET_MESSAGE_ISMRMRD_IMAGEWITHATTRIB_REAL_USHORT,
gt.GADGET_MESSAGE_ISMRMRD_IMAGEWITHATTRIB_REAL_FLOAT,
gt.GADGET_MESSAGE_ISMRMRD_IMAGEWITHATTRIB_CPLX_FLOAT
]:
con.register_reader(im_id,
gt.ImageAttribMessageReader(outfile, out_group))
# DICOM
con.register_reader(gt.GADGET_MESSAGE_DICOM,
gt.BlobMessageReader(out_group, 'dcm'))
con.register_reader(gt.GADGET_MESSAGE_DICOM_WITHNAME,
gt.BlobAttribMessageReader('', 'dcm'))
# Connect to Gadgetron - if no host, port were supplied then look at the
# active profile to get the values
if (address is None) or (port is None):
from mr_utils.config import ProfileConfig
profile = ProfileConfig()
if address is None:
address = profile.get_config_val('gadgetron.host')
if port is None:
port = profile.get_config_val('gadgetron.port')
logging.debug('Connecting to Gadgetron @ %s:%d', address, port)
con.connect(address, port)
# Find the configuration file we need
if config_local:
logging.debug('Sending gadgetron configuration script...')
con.send_gadgetron_configuration_script(config_local)
else:
logging.debug('Sending gadgetron configuration filename %s', config)
con.send_gadgetron_configuration_file(config)
# Decide what the input was
if isinstance(data, ismrmrd.Dataset):
# User has already given us the ismrmrd dataset that gadgetron expects
dset = data
## I would like to figure out a way to pass in a numpy array with header!
# If we've given a filename:
elif isinstance(data, str):
# Find the file extension
ext = pathlib.Path(data).suffix
if ext == '.h5':
# Load the dataset from hdf5 file
dset = ismrmrd.Dataset(data, in_group, False)
elif ext == '.dat':
# Load the dataset from raw
dset = load_raw(data, use='s2i', as_ismrmrd=True)
else:
raise Exception('data was not ismrmrd.Dataset or raw data!')
if not dset:
parser.error('Not a valid dataset: %s' % data)
# Strip the dataset header and send to gadgetron
xml_config = dset.read_xml_header()
con.send_gadgetron_parameters(xml_config)
# Next, send each acquisition to gadgetron
for idx in trange(dset.number_of_acquisitions(), desc='Send', leave=False):
acq = dset.read_acquisition(idx)
try:
con.send_ismrmrd_acquisition(acq)
except socket.error as msg:
logging.error('Failed to send acquisition %d', idx)
print(msg)
return None
logging.debug('Sending close message to Gadgetron')
con.send_gadgetron_close()
con.wait()
# Convert the hdf5 file into something we can use and send it back
with h5py.File(outfile, 'r') as f:
# Group might not be image_0
data = f[out_group][list(f[out_group].keys())[0]]['data'][:]
header = f[out_group][list(f[out_group].keys())[0]]['header'][:]
# data = f[out_group]['image_0']['data'][:]
# header = f[out_group]['image_0']['header'][:]
return (data, header)
## Let's make an ssh tunnel if we find it in profiles.config
from mr_utils.config import ProfileConfig
from paramiko.transport import Transport
import socket
if __name__ == '__main__':
profile = ProfileConfig()
ssh_host = profile.get_config_val('ssh.host')
ssh_user = profile.get_config_val('ssh.user')
host = profile.get_config_val('gadgetron.host')
port = profile.get_config_val('gadgetron.port')
print(ssh_host, ssh_user)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((ssh_host, 22))
transport = Transport(sock)
transport.connect(username=ssh_user, password=None)
chan = transport.open_channel('session', dest_addr=(host, port))
print(chan)
chan.close()
transport.close()
sock.close()
def client(data,
address=None,
port=None,
outfile=None,
in_group='/dataset',
out_group=str(datetime.datetime.now()),
config='default.xml',
config_local=None,
loops=1,
verbose=False):
'''Send acquisitions to Gadgetron.
This client allows you to connect to a Gadgetron server and process data.
data -- Input file with file extension or numpy array.
address -- Hostname of Gadgetron. If not set, taken from profile config.
port -- Port to connect to. If not set, taken from profile config.
outfile -- If provided, output will be saved to file with this name.
in_group -- If input is hdf5, input data group name.
out_group -- Output group name if file is written.
config -- Remote configuration file.
config_local -- Local configuration file.
loops -- Number of loops.
verbose -- Verbose mode.
'''
# Initialize logging
console = logging.StreamHandler()
console.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
console.setLevel(logging.DEBUG)
logger.addHandler(console)
# If user wanted to be verbose, let's give them verbose, otherwise just
# tell them about warnings
if verbose:
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.WARNING)
# The magic happens in the connector
logger.debug('Instantiating Connector')
con = gt.Connector()
## Register all the readers we need:
# The readers need to know where to output the data that gadgetron sends
# back. The output is an hdf5 file, but if we don't want that file and
# only care about the numpy array, then we'll create a temporary file to
# store the output in and then kill it when we're done.
if outfile is None:
outfile = NamedTemporaryFile().name
logger.debug('Writing to filename: %s' % outfile)
# Image message readers
for id in [
gt.GADGET_MESSAGE_ISMRMRD_IMAGE_REAL_USHORT,
gt.GADGET_MESSAGE_ISMRMRD_IMAGE_REAL_FLOAT,
gt.GADGET_MESSAGE_ISMRMRD_IMAGE_CPLX_FLOAT,
gt.GADGET_MESSAGE_ISMRMRD_IMAGE
]:
con.register_reader(id, gt.ImageMessageReader(outfile, out_group))
# Images with attributes
for id in [
gt.GADGET_MESSAGE_ISMRMRD_IMAGEWITHATTRIB_REAL_USHORT,
gt.GADGET_MESSAGE_ISMRMRD_IMAGEWITHATTRIB_REAL_FLOAT,
gt.GADGET_MESSAGE_ISMRMRD_IMAGEWITHATTRIB_CPLX_FLOAT
]:
con.register_reader(id,
gt.ImageAttribMessageReader(outfile, out_group))
# DICOM
con.register_reader(gt.GADGET_MESSAGE_DICOM,
gt.BlobMessageReader(out_group, 'dcm'))
con.register_reader(gt.GADGET_MESSAGE_DICOM_WITHNAME,
gt.BlobAttribMessageReader('', 'dcm'))
# Connect to Gadgetron - if no host, port were supplied then look at the
# active profile to get the values
if (address is None) or (port is None):
from mr_utils.config import ProfileConfig
profile = ProfileConfig()
if address is None:
address = profile.get_config_val('gadgetron.host')
if port is None:
port = profile.get_config_val('gadgetron.port')
logger.debug('Connecting to Gadgetron @ %s:%d' % (address, port))
con.connect(address, port)
# Find the configuration file we need
if config_local:
logger.debug('Sending gadgetron configuration script %s' %
config_local)
con.send_gadgetron_configuration_script(config_local)
else:
logger.debug('Sending gadgetron configuration filename %s' % config)
con.send_gadgetron_configuration_file(config)
# Decide what the input was
if type(data) is ismrmrd.Dataset:
# User has already given us the ismrmrd dataset that gadgetron expects
dset = data
## I would like to figure out a way to pass in a numpy array with header!
# If we've given a filename:
elif type(data) is str:
# Find the file extension
ext = pathlib.Path(data).suffix
if ext == '.h5':
# Load the dataset from hdf5 file
dset = ismrmrd.Dataset(data, in_group, False)
elif ext == '.dat':
# Load the dataset from raw
dset = load_raw(data, use='s2i', as_ismrmrd=True)
else:
raise Exception('data was not ismrmrd.Dataset or raw data!')
if not dset:
parser.error('Not a valid dataset: %s' % data)
# Strip the dataset header and send to gadgetron
xml_config = dset.read_xml_header()
con.send_gadgetron_parameters(xml_config)
# Next, send each acquisition to gadgetron
for idx in range(dset.number_of_acquisitions()):
logger.debug('Sending acquisition %d' % idx)
acq = dset.read_acquisition(idx)
try:
con.send_ismrmrd_acquisition(acq)
except:
logger.error('Failed to send acquisition %d' % idx)
return
logger.debug('Sending close message to Gadgetron')
con.send_gadgetron_close()
con.wait()
# Convert the hdf5 file into something we can use and send it back
with h5py.File(outfile, 'r') as f:
data = f[out_group]['image_0']['data'][:]
header = f[out_group]['image_0']['header'][:]
return (data, header)