def iter_images(filename, index=None, header=None):
''' Read a set of SPIDER images
:Parameters:
filename : str or file object
Filename or open stream for a file
index : int, optional
Index of image to start, if None, start with the first image (Default: None)
header : dict, optional
Output dictionary to place header values
:Returns:
out : array
Array with image information from the file
'''
f = util.uopen(filename, 'rb')
if index is None: index = 0
try:
h = read_web_header(f)
#if header is not None: util.update_header(header, h, web2ara, 'web')
count = count_images(h)
offset, ar_args = array_from_header(h)
f.seek(int(offset))
if not hasattr(index, '__iter__'): index = xrange(index, count)
else: index = index.astype(numpy.int)
for i in index:
yield util.read_image(f, *ar_args)
finally:
util.close(filename, f)
def valid_image(filename):
''' Test if the image is valid
:Parameters:
filename : str
Input filename to test
:Returns:
flag : bool
True if image is valid
'''
f = util.uopen(filename, 'rb')
try:
h = read_spider_header(f)
h_len = int(h['labbyt'])
d_len = int(h['nx']) * int(h['ny']) * int(h['nz'])
i_len = d_len * 4
count = count_images(h)
if count > 1 or h['istack'] == 2:
return file_size(f) == (h_len + count * (h_len+i_len))
else:
return file_size(f) == (h_len + count * i_len)
finally:
util.close(filename, f)
def valid_image(filename):
''' Test if the image is valid
:Parameters:
filename : str
Input filename to test
:Returns:
flag : bool
True if image is valid
'''
f = util.uopen(filename, 'rb')
try:
h = read_spider_header(f)
h_len = int(h['labbyt'])
d_len = int(h['nx']) * int(h['ny']) * int(h['nz'])
i_len = d_len * 4
count = count_images(h)
if count > 1 or h['istack'] == 2:
return file_size(f) == (h_len + count * (h_len + i_len))
else:
return file_size(f) == (h_len + count * i_len)
finally:
util.close(filename, f)
def write_image(filename, img, index=None, header=None, inplace=False):
''' Write an image array to a file in the MRC format
:Parameters:
filename : str
Name of the output file
img : array
Image array
index : int, optional
Index to write image in the stack
header : dict, optional
Dictionary of header values
inplace : bool
Write new image to stack without removing the stack
'''
if header is None and hasattr(img, 'header'): header=img.header
mode = 'rb+' if index is not None and (index > 0 or inplace and index > -1) else 'wb+'
f = util.uopen(filename, mode)
if header is None or not is_format_header(header):
header = create_header(img.shape, img.dtype, img.order, header)
try:
if inplace:
f.seek(int(header.itemsize+int(header['extended'])+index*img.ravel().shape[0]*img.dtype.itemsize))
elif f != filename:
f.seek(0)
header.tofile(f)
if index > 0: f.seek(int(header.itemsize+int(header['extended'])+index*img.ravel().shape[0]*img.dtype.itemsize))
img.tofile(f)
finally:
util.close(filename, f)
def valid_image(filename, no_strict_mrc=False):
''' Test if the image is valid
:Parameters:
filename : str
Input filename to test
no_strict_mrc : bool
Perform strict MRC header checking (recommended) - Only
EPU MRC files and Yifan's frame alignment require this
to be off.
:Returns:
flag : bool
True if image is valid
'''
f = util.uopen(filename, 'rb')
try:
h = read_mrc_header(f, no_strict_mrc)
total = file_size(f)
dtype = numpy.dtype(mrc2numpy[h['mode'][0]])
return total == (1024 + int(h['nsymbt']) + int(h['nx'][0]) *
int(h['ny'][0]) * int(h['nz'][0]) * dtype.itemsize)
finally:
util.close(filename, f)
def read_image(filename, index=None, header=None, cache=None):
''' Read an image from the specified file in the WEB format
:Parameters:
filename : str or file object
Filename or open stream for a file
index : int, optional
Index of image to get, if None, first image (Default: None)
header : dict, optional
Output dictionary to place header values
:Returns:
out : array
Array with image information from the file
'''
idx = 0 if index is None else index
f = util.uopen(filename, 'rb')
try:
h = read_web_header(f)
#if header is not None: util.update_header(header, h, web2ara, 'web')
if idx >= count_images(h): raise IOError, "Index exceeds number of images in stack: %d < %d"%(idx, count_images(h))
offset, ar_args = array_from_header(h)
f.seek(offset + idx * ar_args[1] * ar_args[0].itemsize)
out = util.read_image(f, *ar_args)
finally:
util.close(filename, f)
return out
def read_web_header(filename, index=None):
''' Read the WEB header
:Parameters:
filename : str or file object
Filename or open stream for a file
index : int, ignored
Index of image to get the header, if None, the stack header (Default: None)
:Returns:
out : array
Array with header information in the file
'''
f = util.uopen(filename, 'rb')
m=None
try:
#curr = f.tell()
h = numpy.fromfile(f, dtype=header_dtype, count=1)
if not is_readable(h): h = h.byteswap().newbyteorder()
if not is_readable(h): raise IOError, "Not an WEB file"
if h['extended_ident'] == 'WEBMETADATA':
count = h['extended'][0]/metadata_dtype.itemsize
if (count*metadata_dtype.itemsize) != h['extended'][0]:
_logger.warn("Unable to read metadata - size mismatch: %d *%d = %d != %d"%(count, metadata_dtype.itemsize, (count*metadata_dtype.itemsize), h['extended'][0]))
else:
m = numpy.fromfile(f, dtype=metadata_dtype, count=count)
finally:
util.close(filename, f)
return h, m
def req_json(method, url, params='', headers={}, expects=[200]):
parsed = urlparse(url)
host = parsed.netloc
uri = parsed.path
if (parsed.query != ''):
uri = uri + '?' + parsed.query
conn = None
if (parsed.scheme == 'https'):
conn = httplib.HTTPSConnection(parsed.hostname, parsed.port or 443)
else:
conn = httplib.HTTPConnection(parsed.hostname, parsed.port or 80)
if parsed.username != None:
token = parsed.username + ':' + (parsed.password or '')
token = 'Basic ' + base64.b64encode(token)
headers['Authorization'] = token
success = False
try:
conn.request(method, uri, params, headers)
response = conn.getresponse()
if response.status not in expects:
raise Error('Status: %d %s %sri' %
(response.status, response.reason, response.read()))
string = response.read()
if not string:
return None
ret = json.loads(string)
success = True
return ret
finally:
util.close(conn)
def read_mrc_header(filename, index=None, no_strict_mrc=False):
''' Read the MRC header
:Parameters:
filename : str or file object
Filename or open stream for a file
index : int, ignored
Index of image to get the header, if None, the stack header (Default: None)
no_strict_mrc : bool
Perform strict MRC header checking (recommended) - Only
EPU MRC files and Yifan's frame alignment require this
to be off.
:Returns:
out : array
Array with header information in the file
'''
f = util.uopen(filename, 'rb')
try:
#curr = f.tell()
h = util.fromfile(f, dtype=header_image_dtype, count=1)
if not is_readable(h, no_strict_mrc): h = h.newbyteorder()
if not is_readable(h, no_strict_mrc): raise IOError, "Not MRC header"
finally:
util.close(filename, f)
return h
def numTest():
n = Num(520).nums([
4, 10, 15, 38, 54, 57, 62, 83, 100, 100, 174, 190, 215, 225, 233, 250,
260, 270, 299, 300, 306, 333, 350, 375, 443, 475, 525, 583, 780, 1000
])
print(n.mu)
assert (util.close(n.mu, 270.3))
assert (util.close(n.sd, 231.946))
def location_handler(event):
"""
This is the handler function for the Location intent.
:param event: the input event (data) received from AWS Lex
:return: an ElicitIntent reponse
"""
# Extract the inputTranscript from the input event
trans = event['inputTranscript']
# Also extract the slot values, in case we have to check for something in them
place = event['currentIntent']['slots'].get('place', '')
place_two = event['currentIntent']['slots'].get('place_two', '')
# Also, to be sure, we are checking in case they were assigned 'null' in the JSON file
if not place:
place = ''
if not place_two:
place_two = ''
# And send it to the checker to find the right function for handling
func = _checker(trans, place, place_two)
# Save the response from the function
ans = func(event)
# Default answer if all failed
if not ans: # pragma: no cover
# We are not convering this in the tests as this would require unproper JSON files to begin with.
# Currently we have everything configured as we want, so simulating this would be a little hard.
ans = "Unfortunately I can't seem to find the location"
return util.close({}, 'Fulfilled', ans)
def weather_handler():
weather = Weather()
data = weather.read_data()
# card_title = "Weather in otaniemi"
message = "The temperature is around " + str(data[0])
return util.close({}, 'Fulfilled', message)
def login():
oauth_base.delete_file()
config = __config()
server = HTTPServer(('localhost', 0), OAuthHandler)
_, port = server.socket.getsockname()
url = 'https://accounts.google.com/o/oauth2/auth?' + urllib.urlencode({
'client_id': config['client_id'],
'redirect_uri': 'http://localhost:%s/redirect_uri' % (port),
'response_type': 'code',
'scope': ' '.join(config['scopes']),
'approval_prompt': 'force',
'access_type': 'offline'
})
try:
webbrowser.open(url, new=1, autoraise=True)
server.handle_request()
finally:
util.close(server.socket)
def read_spider_header(filename, index=None):
''' Read the SPIDER header
:Parameters:
filename : str or file object
Filename or open stream for a file
index : int, optional
Index of image to get the header, if None, the stack header (Default: None)
:Returns:
out : array
Array with header information in the file
'''
f = util.uopen(filename, 'rb')
try:
#curr = f.tell()
h = numpy.fromfile(f, dtype=header_dtype, count=1)
if not is_readable(h):
h = h.newbyteorder()
if not is_readable(h): raise IOError, "Not a SPIDER file"
if index is not None:
h_len = int(h['labbyt'])
i_len = int(h['nx']) * int(h['ny']) * int(h['nz']) * 4
count = max(int(h['istack']), 1)
if index >= count:
raise IOError, "Index exceeds number of images in stack: %d < %d" % (
index, count)
#offset = index * (h_len+i_len)
offset = h_len + index * (h_len + i_len) if int(
h['istack']) > 0 else 0
try:
f.seek(offset)
except:
_logger.error("Offset: %s" % str(offset))
raise
h = numpy.fromfile(f, dtype=h.dtype, count=1)
finally:
util.close(filename, f)
return h
def hello_handler():
"""
This intent simply responds to user's greeting Libby.
We decided to have this as users usually test voice assitants with simply 'Hello'
"""
# Let's get a random chance for the reponse
prob = random.uniform(0.0, 1.0) * 100
# If the response is smaller that _perc_chance, we give out a 'random' response
if prob < _perc_chance:
return util.close({}, 'Fulfilled',
_randoms[random.randint(0,
len(_randoms) - 1)])
# Otherwise a normal 'proper' one
return util.close({}, 'Fulfilled',
_responses[random.randint(0,
len(_responses) - 1)])
def weather_handler():
weather = Weather()
data = weather.read_data()
# card_title = "Weather in otaniemi"
message = {
'contentType': 'PlaintText',
'content': "The temperature is around " + str(data[0])
}
return util.close({}, 'Fulfilled', message)
def read_spider_header(filename, index=None):
''' Read the SPIDER header
:Parameters:
filename : str or file object
Filename or open stream for a file
index : int, optional
Index of image to get the header, if None, the stack header (Default: None)
:Returns:
out : array
Array with header information in the file
'''
f = util.uopen(filename, 'rb')
try:
#curr = f.tell()
h = numpy.fromfile(f, dtype=header_dtype, count=1)
if not is_readable(h):
h = h.newbyteorder()
if not is_readable(h): raise IOError, "Not a SPIDER file"
if index is not None:
h_len = int(h['labbyt'])
i_len = int(h['nx']) * int(h['ny']) * int(h['nz']) * 4
count = max(int(h['istack']), 1)
if index >= count: raise IOError, "Index exceeds number of images in stack: %d < %d"%(index, count)
#offset = index * (h_len+i_len)
offset = h_len + index * (h_len+i_len) if int(h['istack']) > 0 else 0
try:
f.seek(offset)
except:
_logger.error("Offset: %s"%str(offset))
raise
h = numpy.fromfile(f, dtype=h.dtype, count=1)
finally:
util.close(filename, f)
return h
def valid_image(filename):
''' Test if the image is valid
:Parameters:
filename : str
Input filename to test
:Returns:
flag : bool
True if image is valid
'''
f = util.uopen(filename, 'rb')
try:
h = read_web_header(f)
offset, ar_args = array_from_header(h)
return file_size(f) == (offset + h['count'] * ar_args[1] * ar_args[0].itemsize)
finally:
util.close(filename, f)
def find_info(book_id, field='buildings'):
"""
Finds info from some book defined by book_id and constructs an output
message. Default information searched is the building(and only
information which can be searched atm).
:param book_id: Id of the book
:param field: Field which teh user is looking for
:return: Response to AWS server in JSON format
"""
request = record(book_id, field=['id', 'shortTitle', field])['json']
if request['status'] == 'OK':
message = locate_book(request['records'][0]['buildings'])
title = request['records'][0]['shortTitle']
message = "".join([title, message])
return util.close({
'book_id': book_id,
'author': None
}, 'Fulfilled', message)
else:
return util.close({'author': None}, 'Fulfilled',
"Something went wrong")
def predict_single(self,
instance,
cands_by_action_index,
follower_weight,
verbose=False,
train_utterances=None,
comparison_stats=None):
states = instance.states[:self.num_actions_to_take + 1]
actions = instance.actions[:self.num_actions_to_take]
ref_utterances = instance.utterances[:self.num_actions_to_take]
if verbose:
def ref_speaker_scores(speaker):
dy.renew_cg()
return [
-1 * l.npvalue()
for l in speaker.loss(states,
actions,
ref_utterances,
testing=True,
loss_per_action=True)
]
# average reference-wise scores across speakers
ref_speaker_scores = np.array([
ref_speaker_scores(speaker) for speaker in self.speakers
]).mean(axis=0)
pred_utterances = []
dy.renew_cg()
inf_states, observe_fns, act_fns = zip(*[
follower._inf_state_observe_and_act(states[0], testing=True)
for follower in self.followers
])
score_fn = make_score_fn(follower_weight)
# def follower_score(follower, all_utterances):
# dy.renew_cg()
# return -1 * follower.loss(states, actions, all_utterances, testing=True, last_utterance_loss=True).npvalue()
for action_ix in range(self.num_actions_to_take):
rescored_candidates = []
for candidate in cands_by_action_index[action_ix]:
assert candidate.instance == instance
# all_utterances = pred_utterances + [candidate.utterance]
marginal_ret = [
follower._marginal_loss(observe, act, inf_state,
actions[action_ix],
candidate.utterance)
for follower, inf_state, observe, act in zip(
self.followers, inf_states, observe_fns, act_fns)
]
follower_scores = [
-1 * l.npvalue() for (l, inf_state) in marginal_ret
]
new_inf_states = [inf_state for (l, inf_state) in marginal_ret]
rescored_candidates.append(
candidate._replace(follower_score=np.mean(follower_scores),
inference_states=new_inf_states))
best_cand = max(rescored_candidates, key=score_fn)
if verbose:
ref_speaker_score = ref_speaker_scores[action_ix]
ref_follower_score = np.mean([
-1 * follower._marginal_loss(
observe, act, inf_state, actions[action_ix],
ref_utterances[action_ix])[0].npvalue()
for follower, inf_state, observe, act in zip(
self.followers, inf_states, observe_fns, act_fns)
])
ref_combined_score = follower_weight * ref_follower_score + (
1 - follower_weight) * ref_speaker_score
for cand in rescored_candidates:
if ref_utterances[action_ix] == cand.utterance:
assert util.close(ref_speaker_score,
cand.speaker_score)
assert util.close(ref_follower_score,
cand.follower_score)
print(instance)
print("\t".join(" ".join(utt) for utt in pred_utterances))
combined_speaker_follower_scores = [
score_fn(cand) for cand in rescored_candidates
]
segment_utts = [cand.utterance for cand in rescored_candidates]
compare_pragmatic_segment(
instance,
action_ix,
segment_utts,
combined_speaker_follower_scores,
train_utterances,
ref_speaker_score=ref_speaker_score,
ref_follower_score=ref_follower_score,
ref_combined_score=ref_combined_score)
if comparison_stats is not None:
if ref_speaker_score > best_cand.speaker_score:
comparison_stats['ref_speaker_higher'] += 1
if ref_follower_score > best_cand.follower_score:
comparison_stats['ref_follower_higher'] += 1
comparison_stats['compared'] += 1
pred_utterances.append(best_cand.utterance)
inf_states = best_cand.inference_states
return pred_utterances
def close(self):
remainder = self.zip.flush()
if write(self.fd, remainder) != len(remainder):
raise self.exc
return close(self.fd)
def close(self):
close(self._fd)
def write_image(filename, img, index=None, header=None, inplace=False):
''' Write an image array to a file in the MRC format
:Parameters:
filename : str
Name of the output file
img : array
Image array
index : int, optional
Index to write image in the stack
header : dict, optional
Dictionary of header values
inplace : bool
Write new image to stack without removing the stack
'''
#float64
#complex64
if header is None and hasattr(img, 'header'): header=img.header
dtype = numpy.complex64 if numpy.iscomplexobj(img) else numpy.float32
try: img = img.astype(dtype)
except: raise TypeError, "Unsupported type for SPIDER writing: %s"%str(img.dtype)
mode = 'rb+' if index is not None and (index > 0 or inplace and index > -1) else 'wb+'
try:
f = util.uopen(filename, mode)
except:
_logger.error("Mode: %s - Index: %s"%(str(mode), str(index)))
raise
try:
if header is None or not hasattr(header, 'dtype') or not is_format_header(header):
h = numpy.zeros(1, header_dtype)
even = header['fourier_even'] if header is not None and 'fourier_even' in header else None
util.update_header(h, spi_defaults, ara2spi)
header=util.update_header(h, header, ara2spi, 'spi')
# Image size in header
header['nx'] = img.T.shape[0]
header['ny'] = img.T.shape[1] if img.ndim > 1 else 1
header['nz'] = img.T.shape[2] if img.ndim > 2 else 1
header['lenbyt'] = img.shape[0]*4
header['labrec'] = 1024 / int(header['lenbyt'])
if 1024%int(header['lenbyt']) != 0:
header['labrec'] = int(header['labrec'])+1
header['labbyt'] = int(header['labrec'] ) * int(header['lenbyt'])
imgsize = img.ravel().shape[0]*4
headsize = int(header['labbyt'])
header['irec'] = header['labrec']+header['nx']
#
#header['irec']
if numpy.iscomplexobj(img):
header['iform'] = 3 if img.ndim == 3 else 1
# determine even or odd Fourier - assumes other dim are padded appropriately
if even is None:
v = int(round(float(img.shape[1])/img.shape[0]))
v = img.shape[1]/v
even = (v%2)==0
if even:
header['iform'] = -22 if img.ndim == 3 else -12
else:
header['iform'] = -21 if img.ndim == 3 else -11
else:
header['iform'] = 3 if img.ndim == 3 else 1
fheader = numpy.zeros(int(header['labbyt'])/4, dtype=numpy.float32)
for name, idx in _header_map.iteritems():
fheader[idx-1]=float(header[name])
if inplace:
f.seek(index * (imgsize + headsize)+headsize+headsize)
else:
if index is not None:
fheader[_header_map['maxim']-1] = index+1
fheader[_header_map['imgnum']-1] = index+1
fheader[_header_map['istack']-1] = 2
f.seek(0)
fheader.tofile(f)
fheader[_header_map['istack']-1] = 0
f.seek(index * (imgsize + headsize)+headsize)
fheader[_header_map['maxim']-1] = 0
fheader.tofile(f)
img.tofile(f)
finally:
util.close(filename, f)
def read_image(filename, index=None, header=None):
''' Read an image from the specified file in the SPIDER format
:Parameters:
filename : str or file object
Filename or open stream for a file
index : int, optional
Index of image to get, if None, first image (Default: None)
header : dict, optional
Dictionary to hold header values
:Returns:
out : array
Array with image information from the file
'''
f = util.uopen(filename, 'rb')
h = None
try:
if index is None: index = 0
h = read_spider_header(f)
dtype = numpy.dtype(spi2numpy[float(h['iform'])])
#if header_dtype.newbyteorder()==h.dtype: dtype = dtype.newbyteorder() # - changed
#if header is not None: util.update_header(header, h, spi2ara, 'spi')
tmp = read_header(h)
if header is not None: header.update(tmp)
h_len = int(h['labbyt'])
d_len = int(h['nx']) * int(h['ny']) * int(h['nz'])
i_len = d_len * 4
count = count_images(h)
if index >= count:
raise IOError, "Index exceeds number of images in stack: %d < %d" % (
index, count)
if count > 1 and int(h['istack']) == 0:
raise ValueError, "Improperly formatted SPIDER header - not stack but contains mutliple images"
offset = h_len * 2 + index * (h_len + i_len) if int(
h['istack']) > 0 else h_len
if count > 1 or h['istack'] == 2:
if file_size(f) != (h_len + count * (h_len + i_len)):
raise ValueError, "file size != header: %d != %d - count: %d -- nx:%d,ny:%d,nz:%d" % (
file_size(f), (h_len + count * (h_len + i_len)), count,
int(h['nx']), int(h['ny']), int(h['nz']))
else:
if file_size(f) != (h_len + count * i_len):
f.seek(h_len + index * (h_len + i_len))
h2 = read_spider_header(f)
raise ValueError, "file size != header: %d != %d - %d + %d * %d -- %d,%d == %d,%d -- count: " % (
file_size(f), (h_len + count * i_len), h_len, count, i_len,
int(h['istack']), int(h['imgnum']), int(
h2['istack']), int(h2['imgnum']), int(h['maxim']))
try:
f.seek(offset)
except:
_logger.error("Offset: %s" % str(offset))
raise
out = numpy.fromfile(f, dtype=dtype, count=d_len)
if header_dtype.newbyteorder()[0] == h.dtype[0]: out = out.byteswap()
#assert(out.ravel().shape[0]==d_len)
if int(h['nz']) > 1:
out = out.reshape(int(h['nz']), int(h['ny']), int(h['nx']))
elif int(h['ny']) > 1:
try:
out = out.reshape(int(h['ny']), int(h['nx']))
except:
_logger.error("%d != %d*%d = %d" %
(out.ravel().shape[0], int(h['nx']), int(
h['ny']), int(h['nx']) * int(h['ny'])))
raise
finally:
util.close(filename, f)
#if header_image_dtype.newbyteorder()==h.dtype:out = out.byteswap()
return out
def parse_subject(request, subject, session_attributes={}):
"""
:param request: JSON data from the Finna API
:param subject: Subject or search term of the current session
:param session_attributes: session attributes for current session if user
has given some
:return: Response to AWS server in JSON format
"""
author = session_attributes.get('author')
if subject is "":
if author:
return parse_author(request, {'author': author})
return util.elicit_intent(
session_attributes, "I'm sorry. I was not "
"able to catch what book"
" you wanted to find. "
"Could you please repeat.")
if request['status'] == 'OK':
result_count = request['resultCount']
# no books was found with the subject
if result_count == 0:
message = "Oh I'm so sorry, no books was found with search term: "\
+ subject
if author:
message += " written by " + str(author)
return util.close({
'subject': subject,
'author': author
}, 'Fulfilled', message)
elif result_count == 1:
return find_info(request['records'][0]['id'])
# if less than five books was found, will be checked all different
# locations in books way that there is not same locations twice.
elif result_count < 5:
real_count = 0
find_locations = []
while real_count < result_count:
buildings = request['records'][real_count]['buildings']
for layer in buildings:
if re.compile("1/AALTO/([a-z])*/").match(layer['value']):
if layer['translated'] not in find_locations:
find_locations.append(layer['translated'])
real_count += 1
# if author was also found.
if author:
message = subject + " books by " + str(author) \
+ " can be found in " + util.make_string_list(find_locations)
else:
message = subject + " books can be found in " + \
util.make_string_list(find_locations)
return util.close({
'subject': subject,
'author': author
}, 'Fulfilled', message)
# there is more than five results, this ask more information from user
else:
if not author:
message = "I found " + str(result_count) + " books with term " \
+ subject + ". Please specify an author or a year, " \
" so I can narrow down the search."
else:
message = "I found " + str(result_count) + " books with " \
+ subject + " by author " + author + ". Can you " \
"give the publication date for example to narrow " \
"down the search."
return util.elicit_intent({
'subject': subject,
'author': author
}, message)
else:
return util.close({'author': None}, 'Fulfilled',
"Something went wrong")
def iter_images(filename, index=None, header=None):
''' Read a set of SPIDER images
:Parameters:
filename : str or file object
Filename or open stream for a file
index : int, optional
Index of image to start, if None, start with the first image (Default: None)
header : dict, optional
Dictionary to hold header values
:Returns:
out : array
Array with image information from the file
'''
f = util.uopen(filename, 'rb')
if index is None: index = 0
try:
h = read_spider_header(f)
dtype = numpy.dtype(spi2numpy[float(h['iform'])])
#if header_dtype.newbyteorder()==h.dtype: dtype = dtype.newbyteorder()
#if header is not None: util.update_header(header, h, spi2ara, 'spi')
tmp=read_header(h)
if header is not None: header.update(tmp)
h_len = int(h['labbyt'])
d_len = int(h['nx']) * int(h['ny']) * int(h['nz'])
i_len = d_len * 4
count = count_images(h)
if numpy.any(index >= count): raise IOError, "Index exceeds number of images in stack: %s < %d"%(str(index), count)
#offset = h_len + 0 * (h_len+i_len)
size = ( h_len + count * (h_len+i_len) ) if int(h['istack']) > 0 else (h_len + i_len)
if file_size(f) != size:
raise ValueError, "file size != header: %d != %d - %d -- %d,%d,%d"%(file_size(f), (h_len + count * (h_len+i_len)), count, int(h['nx']), int(h['ny']), int(h['nz']))
try:
f.seek(h_len)
except:
_logger.error("Offset: %s"%str(h_len))
raise
if int(h['istack']) == 0: # This file contains a single image!
out = numpy.fromfile(f, dtype=dtype, count=d_len)
if header_dtype.newbyteorder()[0]==h.dtype[0]: out = out.byteswap()
if int(h['nz']) > 1: out = out.reshape(int(h['nz']), int(h['ny']), int(h['nx']))
elif int(h['ny']) > 1: out = out.reshape(int(h['ny']), int(h['nx']))
yield out
return
if not hasattr(index, '__iter__'): index = xrange(index, count)
else: index = index.astype(numpy.int)
last=-1
if count > 1 and int(h['istack']) == 0: raise ValueError, "Improperly formatted SPIDER header - not stack but contains mutliple images"
for i in index:
if i < 0: raise ValueError, "Cannot have a negative index"
if i != (last+1):
offset = h_len*2 + i * (h_len+i_len)
try:
f.seek(int(offset))
except:
_logger.error("Offset: %s"%str(offset))
_logger.error("i: %s"%str(i))
raise
else:
f.seek(h_len, 1)
last=i
out = numpy.fromfile(f, dtype=dtype, count=d_len)
if header_dtype.newbyteorder()[0]==h.dtype[0]: out = out.byteswap()
if int(h['nz']) > 1:
try:
out = out.reshape(int(h['nz']), int(h['ny']), int(h['nx']))
except:
_logger.error("%d, %d, %d == %d == %d", int(h['nz']), int(h['ny']), int(h['nx']), numpy.prod((int(h['nz']), int(h['ny']), int(h['nx']))), out.shape[0])
raise
elif int(h['ny']) > 1:
try:
out = out.reshape(int(h['ny']), int(h['nx']))
except:
_logger.error("(%d < %d) -- %d, %d == %d == %d", i, count, int(h['ny']), int(h['nx']), numpy.prod((int(h['ny']), int(h['nx']))), out.shape[0])
raise
yield out
finally:
util.close(filename, f)
def write_image(filename, img, index=None, header=None, inplace=False):
''' Write an image array to a file in the MRC format
:Parameters:
filename : str
Name of the output file
img : array
Image array
index : int, optional
Index to write image in the stack
header : dict, optional
Dictionary of header values
inplace : bool
Write new image to stack without removing the stack
'''
#float64
#complex64
if header is None and hasattr(img, 'header'): header = img.header
dtype = numpy.complex64 if numpy.iscomplexobj(img) else numpy.float32
try:
img = img.astype(dtype)
except:
raise TypeError, "Unsupported type for SPIDER writing: %s" % str(
img.dtype)
mode = 'rb+' if index is not None and (
index > 0 or inplace and index > -1) else 'wb+'
try:
f = util.uopen(filename, mode)
except:
_logger.error("Mode: %s - Index: %s" % (str(mode), str(index)))
raise
try:
if header is None or not hasattr(
header, 'dtype') or not is_format_header(header):
h = numpy.zeros(1, header_dtype)
even = header[
'fourier_even'] if header is not None and 'fourier_even' in header else None
util.update_header(h, spi_defaults, ara2spi)
header = util.update_header(h, header, ara2spi, 'spi')
# Image size in header
header['nx'] = img.T.shape[0]
header['ny'] = img.T.shape[1] if img.ndim > 1 else 1
header['nz'] = img.T.shape[2] if img.ndim > 2 else 1
header['lenbyt'] = img.shape[0] * 4
header['labrec'] = 1024 / int(header['lenbyt'])
if 1024 % int(header['lenbyt']) != 0:
header['labrec'] = int(header['labrec']) + 1
header['labbyt'] = int(header['labrec']) * int(header['lenbyt'])
imgsize = img.ravel().shape[0] * 4
headsize = int(header['labbyt'])
header['irec'] = header['labrec'] + header['nx']
#
#header['irec']
if numpy.iscomplexobj(img):
header['iform'] = 3 if img.ndim == 3 else 1
# determine even or odd Fourier - assumes other dim are padded appropriately
if even is None:
v = int(round(float(img.shape[1]) / img.shape[0]))
v = img.shape[1] / v
even = (v % 2) == 0
if even:
header['iform'] = -22 if img.ndim == 3 else -12
else:
header['iform'] = -21 if img.ndim == 3 else -11
else:
header['iform'] = 3 if img.ndim == 3 else 1
fheader = numpy.zeros(int(header['labbyt']) / 4, dtype=numpy.float32)
for name, idx in _header_map.iteritems():
fheader[idx - 1] = float(header[name])
if inplace:
f.seek(index * (imgsize + headsize) + headsize + headsize)
else:
if index is not None:
fheader[_header_map['maxim'] - 1] = index + 1
fheader[_header_map['imgnum'] - 1] = index + 1
fheader[_header_map['istack'] - 1] = 2
f.seek(0)
fheader.tofile(f)
fheader[_header_map['istack'] - 1] = 0
f.seek(index * (imgsize + headsize) + headsize)
fheader[_header_map['maxim'] - 1] = 0
fheader.tofile(f)
img.tofile(f)
finally:
util.close(filename, f)
def iter_images(filename, index=None, header=None):
''' Read a set of SPIDER images
:Parameters:
filename : str or file object
Filename or open stream for a file
index : int, optional
Index of image to start, if None, start with the first image (Default: None)
header : dict, optional
Dictionary to hold header values
:Returns:
out : array
Array with image information from the file
'''
f = util.uopen(filename, 'rb')
if index is None: index = 0
try:
h = read_spider_header(f)
dtype = numpy.dtype(spi2numpy[float(h['iform'])])
#if header_dtype.newbyteorder()==h.dtype: dtype = dtype.newbyteorder()
#if header is not None: util.update_header(header, h, spi2ara, 'spi')
tmp = read_header(h)
if header is not None: header.update(tmp)
h_len = int(h['labbyt'])
d_len = int(h['nx']) * int(h['ny']) * int(h['nz'])
i_len = d_len * 4
count = count_images(h)
if numpy.any(index >= count):
raise IOError, "Index exceeds number of images in stack: %s < %d" % (
str(index), count)
#offset = h_len + 0 * (h_len+i_len)
size = (h_len + count *
(h_len + i_len)) if int(h['istack']) > 0 else (h_len + i_len)
if file_size(f) != size:
raise ValueError, "file size != header: %d != %d - %d -- %d,%d,%d" % (
file_size(f), (h_len + count * (h_len + i_len)), count,
int(h['nx']), int(h['ny']), int(h['nz']))
try:
f.seek(h_len)
except:
_logger.error("Offset: %s" % str(h_len))
raise
if int(h['istack']) == 0: # This file contains a single image!
out = numpy.fromfile(f, dtype=dtype, count=d_len)
if header_dtype.newbyteorder()[0] == h.dtype[0]:
out = out.byteswap()
if int(h['nz']) > 1:
out = out.reshape(int(h['nz']), int(h['ny']), int(h['nx']))
elif int(h['ny']) > 1:
out = out.reshape(int(h['ny']), int(h['nx']))
yield out
return
if not hasattr(index, '__iter__'): index = xrange(index, count)
else: index = index.astype(numpy.int)
last = -1
if count > 1 and int(h['istack']) == 0:
raise ValueError, "Improperly formatted SPIDER header - not stack but contains mutliple images"
for i in index:
if i < 0: raise ValueError, "Cannot have a negative index"
if i != (last + 1):
offset = h_len * 2 + i * (h_len + i_len)
try:
f.seek(int(offset))
except:
_logger.error("Offset: %s" % str(offset))
_logger.error("i: %s" % str(i))
raise
else:
f.seek(h_len, 1)
last = i
out = numpy.fromfile(f, dtype=dtype, count=d_len)
if header_dtype.newbyteorder()[0] == h.dtype[0]:
out = out.byteswap()
if int(h['nz']) > 1:
try:
out = out.reshape(int(h['nz']), int(h['ny']), int(h['nx']))
except:
_logger.error(
"%d, %d, %d == %d == %d", int(h['nz']), int(h['ny']),
int(h['nx']),
numpy.prod((int(h['nz']), int(h['ny']), int(h['nx']))),
out.shape[0])
raise
elif int(h['ny']) > 1:
try:
out = out.reshape(int(h['ny']), int(h['nx']))
except:
_logger.error("(%d < %d) -- %d, %d == %d == %d", i, count,
int(h['ny']), int(h['nx']),
numpy.prod((int(h['ny']), int(h['nx']))),
out.shape[0])
raise
yield out
finally:
util.close(filename, f)
def close(self):
remainder = self.zip.flush()
if write(self.fd, remainder) != len(remainder):
raise self.exc
return close(self.fd)
def close(self):
close(self._fd)
def iter_images(filename, index=None, header=None, no_strict_mrc=False):
''' Read a set of SPIDER images
:Parameters:
filename : str or file object
Filename or open stream for a file
index : int, optional
Index of image to start, if None, start with the first image (Default: None)
header : dict, optional
Output dictionary to place header values
no_strict_mrc : bool
Perform strict MRC header checking (recommended) - Only
EPU MRC files and Yifan's frame alignment require this
to be off.
:Returns:
out : array
Array with image information from the file
'''
f = util.uopen(filename, 'rb')
if index is None: index = 0
try:
h = read_mrc_header(f, no_strict_mrc)
count = count_images(h)
#if header is not None: util.update_header(header, h, mrc2ara, 'mrc')
tmp = read_header(h)
if header is not None: header.update(tmp)
d_len = h['nx'][0] * h['ny'][0]
dtype = numpy.dtype(mrc2numpy[h['mode'][0]])
offset = 1024 + int(h['nsymbt']) + 0 * d_len * dtype.itemsize
try:
f.seek(int(offset))
except:
_logger.error("%s -- %s" %
(str(offset), str(offset.__class__.__name__)))
raise
if not hasattr(index, '__iter__'): index = xrange(index, count)
else: index = index.astype(numpy.int)
last = 0
total = file_size(f)
if total != (1024 + int(h['nsymbt']) + int(h['nx'][0]) *
int(h['ny'][0]) * int(h['nz'][0]) * dtype.itemsize):
raise util.InvalidHeaderException, "file size != header: %d != %d -- %d" % (
total,
(1024 + int(h['nsymbt']) + int(h['nx'][0]) * int(h['ny'][0]) *
int(h['nz'][0]) * dtype.itemsize), int(h['nsymbt']))
for i in index:
if i != (last + 1):
f.seek(
int(1024 + int(h['nsymbt']) + i * d_len * dtype.itemsize))
out = util.fromfile(f, dtype=dtype, count=d_len)
out = reshape_data(out, h, index, count)
if header_image_dtype.newbyteorder()[0] == h.dtype[0]:
out = out.byteswap()
yield out
finally:
util.close(filename, f)
def read_image(filename,
index=None,
header=None,
cache=None,
no_strict_mrc=False,
force_volume=False):
''' Read an image from the specified file in the MRC format
:Parameters:
filename : str or file object
Filename or open stream for a file
index : int, optional
Index of image to get, if None, first image (Default: None)
header : dict, optional
Output dictionary to place header values
no_strict_mrc : bool
Perform strict MRC header checking (recommended) - Only
EPU MRC files and Yifan's frame alignment require this
to be off.
force_volume : bool
For image to be read as a volume
:Returns:
out : array
Array with image information from the file
'''
idx = 0 if index is None else index
f = util.uopen(filename, 'rb')
try:
h = read_mrc_header(f, no_strict_mrc)
#if header is not None: util.update_header(header, h, mrc2ara, 'mrc')
tmp = read_header(h, force_volume=force_volume)
if header is not None: header.update(tmp)
count = count_images(h)
if idx >= count:
raise IOError, "Index exceeds number of images in stack: %d < %d" % (
idx, count)
if index is None and (count == h['nx'][0] or force_volume):
d_len = h['nx'][0] * h['ny'][0] * h['nz'][0]
else:
d_len = h['nx'][0] * h['ny'][0]
dtype = numpy.dtype(mrc2numpy[h['mode'][0]])
offset = 1024 + int(h['nsymbt']) + idx * d_len * dtype.itemsize
total = file_size(f)
if total != (1024 + int(h['nsymbt']) + int(h['nx'][0]) *
int(h['ny'][0]) * int(h['nz'][0]) * dtype.itemsize):
raise util.InvalidHeaderException, "file size != header: %d != %d -- %s, %d" % (
total,
(1024 + int(h['nsymbt']) + int(h['nx'][0]) * int(h['ny'][0]) *
int(h['nz'][0]) * dtype.itemsize), str(idx), int(h['nsymbt']))
f.seek(int(offset))
out = util.fromfile(f, dtype=dtype, count=d_len)
out = reshape_data(out, h, index, count, force_volume)
if header_image_dtype.newbyteorder()[0] == h.dtype[0]:
out = out.byteswap()
finally:
util.close(filename, f)
#assert(numpy.alltrue(numpy.logical_not(numpy.isnan(out))))
#if header_image_dtype.newbyteorder()==h.dtype:out = out.byteswap()
return out
def exit(self, widget, data=None):
self.stop()
close()
def symTest():
s = Sym().syms(
['y', 'y', 'y', 'y', 'y', 'y', 'y', 'y', 'y', 'n', 'n', 'n', 'n', 'n'])
assert (util.close(s.symEnt(), 0.9403))
print(s.symEnt())
def read_image(filename, index=None, header=None):
''' Read an image from the specified file in the SPIDER format
:Parameters:
filename : str or file object
Filename or open stream for a file
index : int, optional
Index of image to get, if None, first image (Default: None)
header : dict, optional
Dictionary to hold header values
:Returns:
out : array
Array with image information from the file
'''
f = util.uopen(filename, 'rb')
h = None
try:
if index is None: index = 0
h = read_spider_header(f)
dtype = numpy.dtype(spi2numpy[float(h['iform'])])
#if header_dtype.newbyteorder()==h.dtype: dtype = dtype.newbyteorder() # - changed
#if header is not None: util.update_header(header, h, spi2ara, 'spi')
tmp=read_header(h)
if header is not None: header.update(tmp)
h_len = int(h['labbyt'])
d_len = int(h['nx']) * int(h['ny']) * int(h['nz'])
i_len = d_len * 4
count = count_images(h)
if index >= count: raise IOError, "Index exceeds number of images in stack: %d < %d"%(index, count)
if count > 1 and int(h['istack']) == 0: raise ValueError, "Improperly formatted SPIDER header - not stack but contains mutliple images"
offset = h_len*2 + index * (h_len+i_len) if int(h['istack']) > 0 else h_len
if count > 1 or h['istack'] == 2:
if file_size(f) != (h_len + count * (h_len+i_len)):
raise ValueError, "file size != header: %d != %d - count: %d -- nx:%d,ny:%d,nz:%d"%(file_size(f), (h_len + count * (h_len+i_len)), count, int(h['nx']), int(h['ny']), int(h['nz']))
else:
if file_size(f) != (h_len + count * i_len):
f.seek(h_len + index * (h_len+i_len))
h2 = read_spider_header(f)
raise ValueError, "file size != header: %d != %d - %d + %d * %d -- %d,%d == %d,%d -- count: "%(file_size(f), (h_len + count * i_len), h_len, count, i_len, int(h['istack']), int(h['imgnum']), int(h2['istack']), int(h2['imgnum']), int(h['maxim']) )
try:
f.seek(offset)
except:
_logger.error("Offset: %s"%str(offset))
raise
out = numpy.fromfile(f, dtype=dtype, count=d_len)
if header_dtype.newbyteorder()[0]==h.dtype[0]: out = out.byteswap()
#assert(out.ravel().shape[0]==d_len)
if int(h['nz']) > 1: out = out.reshape(int(h['nz']), int(h['ny']), int(h['nx']))
elif int(h['ny']) > 1:
try:
out = out.reshape(int(h['ny']), int(h['nx']))
except:
_logger.error("%d != %d*%d = %d"%(out.ravel().shape[0], int(h['nx']), int(h['ny']), int(h['nx'])*int(h['ny'])))
raise
finally:
util.close(filename, f)
#if header_image_dtype.newbyteorder()==h.dtype:out = out.byteswap()
return out
def write_image(filename, img, index=None, header=None, inplace=False):
''' Write an image array to a file in the MRC format
:Parameters:
filename : str
Name of the output file
img : array
Image array
index : int, optional
Index to write image in the stack
header : dict, optional
Dictionary of header values
inplace : bool
Write new image to stack without removing the stack
'''
if header is None and hasattr(img, 'header'): header = img.header
try:
img = img.astype(mrc2numpy[numpy2mrc[img.dtype.type]])
except:
raise TypeError, "Unsupported type for MRC writing: %s" % str(
img.dtype)
mode = 'rb+' if index is not None and (
index > 0 or inplace and index > -1) else 'wb+'
f = util.uopen(filename, mode)
if header is None or not hasattr(header,
'dtype') or not is_format_header(header):
h = numpy.zeros(1, header_image_dtype)
util.update_header(h, mrc_defaults, ara2mrc)
pix = header.get('apix', 1.0) if header is not None else 1.0
header = util.update_header(h, header, ara2mrc, 'mrc')
header['nx'] = img.T.shape[0]
header['ny'] = img.T.shape[1] if img.ndim > 1 else 1
if header['nz'] == 0:
header['nz'] = img.shape[2] if img.ndim > 2 else 1
header['mode'] = numpy2mrc[img.dtype.type]
header['mx'] = header['nx']
header['my'] = header['ny']
header['mz'] = header['nz']
header['xlen'] = header['nx'] * pix
header['ylen'] = header['ny'] * pix
header['zlen'] = header['nz'] * pix
header['alpha'] = 90
header['beta'] = 90
header['gamma'] = 90
header['mapc'] = 1
header['mapr'] = 2
header['maps'] = 3
header['amin'] = numpy.min(img)
header['amax'] = numpy.max(img)
header['amean'] = numpy.mean(img)
header['map'] = 'MAP'
header['byteorder'] = byteorderint2[sys.byteorder] #'DA\x00\x00'
header['nlabels'] = 1
header['label0'] = 'Created by Arachnid'
#header['byteorder'] = numpy.fromstring('\x44\x41\x00\x00', dtype=header['byteorder'].dtype)
#header['rms'] = numpy.std(img)
if img.ndim == 3:
header['nxstart'] = header['nx'] / -2
header['nystart'] = header['ny'] / -2
header['nzstart'] = header['nz'] / -2
if index is not None:
stack_count = index + 1
header['nz'] = stack_count
header['mz'] = stack_count
header['zlen'] = stack_count
#header['zorigin'] = stack_count/2.0
try:
if inplace:
f.seek(
int(1024 + int(h['nsymbt']) +
index * img.ravel().shape[0] * img.dtype.itemsize))
elif f != filename:
f.seek(0)
header.tofile(f)
if index > 0:
f.seek(
int(1024 + int(h['nsymbt']) +
index * img.ravel().shape[0] * img.dtype.itemsize))
img.tofile(f)
finally:
util.close(filename, f)
def handle_read(self):
data = self.recv(1024)
if data == '':
self.handle_close()
return
log.l.LogIt('RTC006', 'D', 'cmd i: %s', (data))
args = string.split(data)
if len(args) == 0:
return;
self.mode = args[0]
self.target = args[1]
args = args[2:]
log.l.LogIt('RTC007', '1', '%s', (str(args)))
if len(args) == 0:
return
#--------------------
if args[0] == 'close':
self.senddata.append(util.close(self.mode == 'router', self.target, args[1:]))
#--------------------
elif args[0] == 'data':
if self.mode == 'domain':
self.senddata.append('command: '+args[0]+': address only to router')
if self.mode == 'router':
self.senddata.append(util.data(args[1:]))
elif args[0] == 'dump':
if self.mode == 'domain':
self.senddata.append('command: '+args[0]+': address only to router')
if self.mode == 'router':
self.senddata.append(util.dump())
#--------------------
elif args[0] == 'event':
if self.mode == 'domain':
self.senddata.append('command: '+args[0]+': address only to router')
if self.mode == 'router':
self.senddata.append(util.event(args[1:]))
#--------------------
elif args[0] == 'fb' or args[0] == 'fallback' or args[0] == 'secondary':
if len(args) > 1 and args[1] == 'auto':
auto = 1
else:
auto = 0
if self.mode == 'domain':
cfg.domain[self.target]['fallback'] = 1
self.senddata.append(util.switch_secondary(self.target, auto))
if self.mode == 'router':
for i in cfg.domain.keys():
if cfg.domain[i]['apr'] == cfg.name:
cfg.domain[i]['fallback'] = 1
self.senddata.append(i+' '+util.switch_secondary(i, auto))
#--------------------
elif args[0] == 'ff' or args[0] == 'fallforward' or args[0] == 'primary':
if len(args) > 1 and args[1] == 'auto':
auto = 1
else:
auto = 0
if self.mode == 'domain':
cfg.domain[self.target]['fallback'] = 0
self.senddata.append(util.switch_primary(self.target, auto))
if self.mode == 'router':
for i in cfg.domain.keys():
if cfg.domain[i]['apr'] == cfg.name:
cfg.domain[i]['fallback'] = 0
self.senddata.append(i+' '+util.switch_primary(i, auto))
#--------------------
elif args[0] == 'pvc':
if self.mode == 'domain':
self.senddata.append('command: '+args[0]+': address only to router')
if self.mode == 'router':
self.senddata.append(util.pvc(args[1:]))
#--------------------
elif args[0] == 'refresh':
evt_hdlr.refresh()
self.senddata.append('status refreshed')
#--------------------
elif args[0] == 'set':
self.senddata = util.set(self.mode == 'router', self.target, args[1:])
#--------------------
elif args[0] == 'sna':
if self.mode == 'domain':
self.senddata.append('command: '+args[0]+': address only to router')
if self.mode == 'router':
self.senddata.append(util.sna(args[1:]))
#--------------------
elif args[0] == 'status':
if self.mode == 'domain':
self.senddata.append(util.status(self.target))
if self.mode == 'router':
for i in cfg.domain.keys():
if cfg.domain[i]['apr'] == cfg.name:
self.senddata.append(i+' '+util.status(i))
if len(self.senddata) == 0:
self.senddata.append('not active')
#--------------------
elif args[0] == 'stop':
if self.mode == 'domain':
self.senddata.append('command: '+args[0]+': address only to router')
if self.mode == 'router':
log.l.LogIt('RTC008', 'I', 'command termination', ())
cfg.stopping = 1
msg = '%s terminating' % self.target
self.senddata.append(msg)
for i in cfg.domain.keys():
util.closeall(i)
#--------------------
elif args[0] == 'trace':
if self.mode == 'domain':
self.senddata.append('command: '+args[0]+': address only to router')
if self.mode == 'router':
if len(args) > 1:
log.l.SetTraceLevel(int(args[1]))
self.senddata.append('trace level %s' % (args[1]))
log.l.LogIt('RTC009', 'I', 'command trace %s', (args[1]))
else:
level = log.l.GetTraceLevel()
self.senddata.append('trace level %d' % (level))
log.l.LogIt('RTC010', 'I', 'command get trace: %d', (level))
#--------------------
elif args[0] == 'version':
msg = ver.getVersion()
if cfg.snasrv_version != '':
msg = msg + ' snasrv: ' + cfg.snasrv_version
self.senddata.append(msg)
#--------------------
else:
self.senddata.append('command: '+args[0]+': not implemented')
