def iter_images(filename, index=None, header=None): ''' Read a set of SPIDER images :Parameters: filename : str or file object Filename or open stream for a file index : int, optional Index of image to start, if None, start with the first image (Default: None) header : dict, optional Output dictionary to place header values :Returns: out : array Array with image information from the file ''' f = util.uopen(filename, 'rb') if index is None: index = 0 try: h = read_web_header(f) #if header is not None: util.update_header(header, h, web2ara, 'web') count = count_images(h) offset, ar_args = array_from_header(h) f.seek(int(offset)) if not hasattr(index, '__iter__'): index = xrange(index, count) else: index = index.astype(numpy.int) for i in index: yield util.read_image(f, *ar_args) finally: util.close(filename, f)
def valid_image(filename): ''' Test if the image is valid :Parameters: filename : str Input filename to test :Returns: flag : bool True if image is valid ''' f = util.uopen(filename, 'rb') try: h = read_spider_header(f) h_len = int(h['labbyt']) d_len = int(h['nx']) * int(h['ny']) * int(h['nz']) i_len = d_len * 4 count = count_images(h) if count > 1 or h['istack'] == 2: return file_size(f) == (h_len + count * (h_len+i_len)) else: return file_size(f) == (h_len + count * i_len) finally: util.close(filename, f)
def valid_image(filename): ''' Test if the image is valid :Parameters: filename : str Input filename to test :Returns: flag : bool True if image is valid ''' f = util.uopen(filename, 'rb') try: h = read_spider_header(f) h_len = int(h['labbyt']) d_len = int(h['nx']) * int(h['ny']) * int(h['nz']) i_len = d_len * 4 count = count_images(h) if count > 1 or h['istack'] == 2: return file_size(f) == (h_len + count * (h_len + i_len)) else: return file_size(f) == (h_len + count * i_len) finally: util.close(filename, f)
def write_image(filename, img, index=None, header=None, inplace=False): ''' Write an image array to a file in the MRC format :Parameters: filename : str Name of the output file img : array Image array index : int, optional Index to write image in the stack header : dict, optional Dictionary of header values inplace : bool Write new image to stack without removing the stack ''' if header is None and hasattr(img, 'header'): header=img.header mode = 'rb+' if index is not None and (index > 0 or inplace and index > -1) else 'wb+' f = util.uopen(filename, mode) if header is None or not is_format_header(header): header = create_header(img.shape, img.dtype, img.order, header) try: if inplace: f.seek(int(header.itemsize+int(header['extended'])+index*img.ravel().shape[0]*img.dtype.itemsize)) elif f != filename: f.seek(0) header.tofile(f) if index > 0: f.seek(int(header.itemsize+int(header['extended'])+index*img.ravel().shape[0]*img.dtype.itemsize)) img.tofile(f) finally: util.close(filename, f)
def valid_image(filename, no_strict_mrc=False): ''' Test if the image is valid :Parameters: filename : str Input filename to test no_strict_mrc : bool Perform strict MRC header checking (recommended) - Only EPU MRC files and Yifan's frame alignment require this to be off. :Returns: flag : bool True if image is valid ''' f = util.uopen(filename, 'rb') try: h = read_mrc_header(f, no_strict_mrc) total = file_size(f) dtype = numpy.dtype(mrc2numpy[h['mode'][0]]) return total == (1024 + int(h['nsymbt']) + int(h['nx'][0]) * int(h['ny'][0]) * int(h['nz'][0]) * dtype.itemsize) finally: util.close(filename, f)
def read_image(filename, index=None, header=None, cache=None): ''' Read an image from the specified file in the WEB format :Parameters: filename : str or file object Filename or open stream for a file index : int, optional Index of image to get, if None, first image (Default: None) header : dict, optional Output dictionary to place header values :Returns: out : array Array with image information from the file ''' idx = 0 if index is None else index f = util.uopen(filename, 'rb') try: h = read_web_header(f) #if header is not None: util.update_header(header, h, web2ara, 'web') if idx >= count_images(h): raise IOError, "Index exceeds number of images in stack: %d < %d"%(idx, count_images(h)) offset, ar_args = array_from_header(h) f.seek(offset + idx * ar_args[1] * ar_args[0].itemsize) out = util.read_image(f, *ar_args) finally: util.close(filename, f) return out
def read_web_header(filename, index=None): ''' Read the WEB header :Parameters: filename : str or file object Filename or open stream for a file index : int, ignored Index of image to get the header, if None, the stack header (Default: None) :Returns: out : array Array with header information in the file ''' f = util.uopen(filename, 'rb') m=None try: #curr = f.tell() h = numpy.fromfile(f, dtype=header_dtype, count=1) if not is_readable(h): h = h.byteswap().newbyteorder() if not is_readable(h): raise IOError, "Not an WEB file" if h['extended_ident'] == 'WEBMETADATA': count = h['extended'][0]/metadata_dtype.itemsize if (count*metadata_dtype.itemsize) != h['extended'][0]: _logger.warn("Unable to read metadata - size mismatch: %d *%d = %d != %d"%(count, metadata_dtype.itemsize, (count*metadata_dtype.itemsize), h['extended'][0])) else: m = numpy.fromfile(f, dtype=metadata_dtype, count=count) finally: util.close(filename, f) return h, m
def req_json(method, url, params='', headers={}, expects=[200]): parsed = urlparse(url) host = parsed.netloc uri = parsed.path if (parsed.query != ''): uri = uri + '?' + parsed.query conn = None if (parsed.scheme == 'https'): conn = httplib.HTTPSConnection(parsed.hostname, parsed.port or 443) else: conn = httplib.HTTPConnection(parsed.hostname, parsed.port or 80) if parsed.username != None: token = parsed.username + ':' + (parsed.password or '') token = 'Basic ' + base64.b64encode(token) headers['Authorization'] = token success = False try: conn.request(method, uri, params, headers) response = conn.getresponse() if response.status not in expects: raise Error('Status: %d %s %sri' % (response.status, response.reason, response.read())) string = response.read() if not string: return None ret = json.loads(string) success = True return ret finally: util.close(conn)
def read_mrc_header(filename, index=None, no_strict_mrc=False): ''' Read the MRC header :Parameters: filename : str or file object Filename or open stream for a file index : int, ignored Index of image to get the header, if None, the stack header (Default: None) no_strict_mrc : bool Perform strict MRC header checking (recommended) - Only EPU MRC files and Yifan's frame alignment require this to be off. :Returns: out : array Array with header information in the file ''' f = util.uopen(filename, 'rb') try: #curr = f.tell() h = util.fromfile(f, dtype=header_image_dtype, count=1) if not is_readable(h, no_strict_mrc): h = h.newbyteorder() if not is_readable(h, no_strict_mrc): raise IOError, "Not MRC header" finally: util.close(filename, f) return h
def numTest(): n = Num(520).nums([ 4, 10, 15, 38, 54, 57, 62, 83, 100, 100, 174, 190, 215, 225, 233, 250, 260, 270, 299, 300, 306, 333, 350, 375, 443, 475, 525, 583, 780, 1000 ]) print(n.mu) assert (util.close(n.mu, 270.3)) assert (util.close(n.sd, 231.946))
def location_handler(event): """ This is the handler function for the Location intent. :param event: the input event (data) received from AWS Lex :return: an ElicitIntent reponse """ # Extract the inputTranscript from the input event trans = event['inputTranscript'] # Also extract the slot values, in case we have to check for something in them place = event['currentIntent']['slots'].get('place', '') place_two = event['currentIntent']['slots'].get('place_two', '') # Also, to be sure, we are checking in case they were assigned 'null' in the JSON file if not place: place = '' if not place_two: place_two = '' # And send it to the checker to find the right function for handling func = _checker(trans, place, place_two) # Save the response from the function ans = func(event) # Default answer if all failed if not ans: # pragma: no cover # We are not convering this in the tests as this would require unproper JSON files to begin with. # Currently we have everything configured as we want, so simulating this would be a little hard. ans = "Unfortunately I can't seem to find the location" return util.close({}, 'Fulfilled', ans)
def weather_handler(): weather = Weather() data = weather.read_data() # card_title = "Weather in otaniemi" message = "The temperature is around " + str(data[0]) return util.close({}, 'Fulfilled', message)
def login(): oauth_base.delete_file() config = __config() server = HTTPServer(('localhost', 0), OAuthHandler) _, port = server.socket.getsockname() url = 'https://accounts.google.com/o/oauth2/auth?' + urllib.urlencode({ 'client_id': config['client_id'], 'redirect_uri': 'http://localhost:%s/redirect_uri' % (port), 'response_type': 'code', 'scope': ' '.join(config['scopes']), 'approval_prompt': 'force', 'access_type': 'offline' }) try: webbrowser.open(url, new=1, autoraise=True) server.handle_request() finally: util.close(server.socket)
def read_spider_header(filename, index=None): ''' Read the SPIDER header :Parameters: filename : str or file object Filename or open stream for a file index : int, optional Index of image to get the header, if None, the stack header (Default: None) :Returns: out : array Array with header information in the file ''' f = util.uopen(filename, 'rb') try: #curr = f.tell() h = numpy.fromfile(f, dtype=header_dtype, count=1) if not is_readable(h): h = h.newbyteorder() if not is_readable(h): raise IOError, "Not a SPIDER file" if index is not None: h_len = int(h['labbyt']) i_len = int(h['nx']) * int(h['ny']) * int(h['nz']) * 4 count = max(int(h['istack']), 1) if index >= count: raise IOError, "Index exceeds number of images in stack: %d < %d" % ( index, count) #offset = index * (h_len+i_len) offset = h_len + index * (h_len + i_len) if int( h['istack']) > 0 else 0 try: f.seek(offset) except: _logger.error("Offset: %s" % str(offset)) raise h = numpy.fromfile(f, dtype=h.dtype, count=1) finally: util.close(filename, f) return h
def hello_handler(): """ This intent simply responds to user's greeting Libby. We decided to have this as users usually test voice assitants with simply 'Hello' """ # Let's get a random chance for the reponse prob = random.uniform(0.0, 1.0) * 100 # If the response is smaller that _perc_chance, we give out a 'random' response if prob < _perc_chance: return util.close({}, 'Fulfilled', _randoms[random.randint(0, len(_randoms) - 1)]) # Otherwise a normal 'proper' one return util.close({}, 'Fulfilled', _responses[random.randint(0, len(_responses) - 1)])
def weather_handler(): weather = Weather() data = weather.read_data() # card_title = "Weather in otaniemi" message = { 'contentType': 'PlaintText', 'content': "The temperature is around " + str(data[0]) } return util.close({}, 'Fulfilled', message)
def read_spider_header(filename, index=None): ''' Read the SPIDER header :Parameters: filename : str or file object Filename or open stream for a file index : int, optional Index of image to get the header, if None, the stack header (Default: None) :Returns: out : array Array with header information in the file ''' f = util.uopen(filename, 'rb') try: #curr = f.tell() h = numpy.fromfile(f, dtype=header_dtype, count=1) if not is_readable(h): h = h.newbyteorder() if not is_readable(h): raise IOError, "Not a SPIDER file" if index is not None: h_len = int(h['labbyt']) i_len = int(h['nx']) * int(h['ny']) * int(h['nz']) * 4 count = max(int(h['istack']), 1) if index >= count: raise IOError, "Index exceeds number of images in stack: %d < %d"%(index, count) #offset = index * (h_len+i_len) offset = h_len + index * (h_len+i_len) if int(h['istack']) > 0 else 0 try: f.seek(offset) except: _logger.error("Offset: %s"%str(offset)) raise h = numpy.fromfile(f, dtype=h.dtype, count=1) finally: util.close(filename, f) return h
def valid_image(filename): ''' Test if the image is valid :Parameters: filename : str Input filename to test :Returns: flag : bool True if image is valid ''' f = util.uopen(filename, 'rb') try: h = read_web_header(f) offset, ar_args = array_from_header(h) return file_size(f) == (offset + h['count'] * ar_args[1] * ar_args[0].itemsize) finally: util.close(filename, f)
def find_info(book_id, field='buildings'): """ Finds info from some book defined by book_id and constructs an output message. Default information searched is the building(and only information which can be searched atm). :param book_id: Id of the book :param field: Field which teh user is looking for :return: Response to AWS server in JSON format """ request = record(book_id, field=['id', 'shortTitle', field])['json'] if request['status'] == 'OK': message = locate_book(request['records'][0]['buildings']) title = request['records'][0]['shortTitle'] message = "".join([title, message]) return util.close({ 'book_id': book_id, 'author': None }, 'Fulfilled', message) else: return util.close({'author': None}, 'Fulfilled', "Something went wrong")
def predict_single(self, instance, cands_by_action_index, follower_weight, verbose=False, train_utterances=None, comparison_stats=None): states = instance.states[:self.num_actions_to_take + 1] actions = instance.actions[:self.num_actions_to_take] ref_utterances = instance.utterances[:self.num_actions_to_take] if verbose: def ref_speaker_scores(speaker): dy.renew_cg() return [ -1 * l.npvalue() for l in speaker.loss(states, actions, ref_utterances, testing=True, loss_per_action=True) ] # average reference-wise scores across speakers ref_speaker_scores = np.array([ ref_speaker_scores(speaker) for speaker in self.speakers ]).mean(axis=0) pred_utterances = [] dy.renew_cg() inf_states, observe_fns, act_fns = zip(*[ follower._inf_state_observe_and_act(states[0], testing=True) for follower in self.followers ]) score_fn = make_score_fn(follower_weight) # def follower_score(follower, all_utterances): # dy.renew_cg() # return -1 * follower.loss(states, actions, all_utterances, testing=True, last_utterance_loss=True).npvalue() for action_ix in range(self.num_actions_to_take): rescored_candidates = [] for candidate in cands_by_action_index[action_ix]: assert candidate.instance == instance # all_utterances = pred_utterances + [candidate.utterance] marginal_ret = [ follower._marginal_loss(observe, act, inf_state, actions[action_ix], candidate.utterance) for follower, inf_state, observe, act in zip( self.followers, inf_states, observe_fns, act_fns) ] follower_scores = [ -1 * l.npvalue() for (l, inf_state) in marginal_ret ] new_inf_states = [inf_state for (l, inf_state) in marginal_ret] rescored_candidates.append( candidate._replace(follower_score=np.mean(follower_scores), inference_states=new_inf_states)) best_cand = max(rescored_candidates, key=score_fn) if verbose: ref_speaker_score = ref_speaker_scores[action_ix] ref_follower_score = np.mean([ -1 * follower._marginal_loss( observe, act, inf_state, actions[action_ix], ref_utterances[action_ix])[0].npvalue() for follower, inf_state, observe, act in zip( self.followers, inf_states, observe_fns, act_fns) ]) ref_combined_score = follower_weight * ref_follower_score + ( 1 - follower_weight) * ref_speaker_score for cand in rescored_candidates: if ref_utterances[action_ix] == cand.utterance: assert util.close(ref_speaker_score, cand.speaker_score) assert util.close(ref_follower_score, cand.follower_score) print(instance) print("\t".join(" ".join(utt) for utt in pred_utterances)) combined_speaker_follower_scores = [ score_fn(cand) for cand in rescored_candidates ] segment_utts = [cand.utterance for cand in rescored_candidates] compare_pragmatic_segment( instance, action_ix, segment_utts, combined_speaker_follower_scores, train_utterances, ref_speaker_score=ref_speaker_score, ref_follower_score=ref_follower_score, ref_combined_score=ref_combined_score) if comparison_stats is not None: if ref_speaker_score > best_cand.speaker_score: comparison_stats['ref_speaker_higher'] += 1 if ref_follower_score > best_cand.follower_score: comparison_stats['ref_follower_higher'] += 1 comparison_stats['compared'] += 1 pred_utterances.append(best_cand.utterance) inf_states = best_cand.inference_states return pred_utterances
def close(self): remainder = self.zip.flush() if write(self.fd, remainder) != len(remainder): raise self.exc return close(self.fd)
def close(self): close(self._fd)
def write_image(filename, img, index=None, header=None, inplace=False): ''' Write an image array to a file in the MRC format :Parameters: filename : str Name of the output file img : array Image array index : int, optional Index to write image in the stack header : dict, optional Dictionary of header values inplace : bool Write new image to stack without removing the stack ''' #float64 #complex64 if header is None and hasattr(img, 'header'): header=img.header dtype = numpy.complex64 if numpy.iscomplexobj(img) else numpy.float32 try: img = img.astype(dtype) except: raise TypeError, "Unsupported type for SPIDER writing: %s"%str(img.dtype) mode = 'rb+' if index is not None and (index > 0 or inplace and index > -1) else 'wb+' try: f = util.uopen(filename, mode) except: _logger.error("Mode: %s - Index: %s"%(str(mode), str(index))) raise try: if header is None or not hasattr(header, 'dtype') or not is_format_header(header): h = numpy.zeros(1, header_dtype) even = header['fourier_even'] if header is not None and 'fourier_even' in header else None util.update_header(h, spi_defaults, ara2spi) header=util.update_header(h, header, ara2spi, 'spi') # Image size in header header['nx'] = img.T.shape[0] header['ny'] = img.T.shape[1] if img.ndim > 1 else 1 header['nz'] = img.T.shape[2] if img.ndim > 2 else 1 header['lenbyt'] = img.shape[0]*4 header['labrec'] = 1024 / int(header['lenbyt']) if 1024%int(header['lenbyt']) != 0: header['labrec'] = int(header['labrec'])+1 header['labbyt'] = int(header['labrec'] ) * int(header['lenbyt']) imgsize = img.ravel().shape[0]*4 headsize = int(header['labbyt']) header['irec'] = header['labrec']+header['nx'] # #header['irec'] if numpy.iscomplexobj(img): header['iform'] = 3 if img.ndim == 3 else 1 # determine even or odd Fourier - assumes other dim are padded appropriately if even is None: v = int(round(float(img.shape[1])/img.shape[0])) v = img.shape[1]/v even = (v%2)==0 if even: header['iform'] = -22 if img.ndim == 3 else -12 else: header['iform'] = -21 if img.ndim == 3 else -11 else: header['iform'] = 3 if img.ndim == 3 else 1 fheader = numpy.zeros(int(header['labbyt'])/4, dtype=numpy.float32) for name, idx in _header_map.iteritems(): fheader[idx-1]=float(header[name]) if inplace: f.seek(index * (imgsize + headsize)+headsize+headsize) else: if index is not None: fheader[_header_map['maxim']-1] = index+1 fheader[_header_map['imgnum']-1] = index+1 fheader[_header_map['istack']-1] = 2 f.seek(0) fheader.tofile(f) fheader[_header_map['istack']-1] = 0 f.seek(index * (imgsize + headsize)+headsize) fheader[_header_map['maxim']-1] = 0 fheader.tofile(f) img.tofile(f) finally: util.close(filename, f)
def read_image(filename, index=None, header=None): ''' Read an image from the specified file in the SPIDER format :Parameters: filename : str or file object Filename or open stream for a file index : int, optional Index of image to get, if None, first image (Default: None) header : dict, optional Dictionary to hold header values :Returns: out : array Array with image information from the file ''' f = util.uopen(filename, 'rb') h = None try: if index is None: index = 0 h = read_spider_header(f) dtype = numpy.dtype(spi2numpy[float(h['iform'])]) #if header_dtype.newbyteorder()==h.dtype: dtype = dtype.newbyteorder() # - changed #if header is not None: util.update_header(header, h, spi2ara, 'spi') tmp = read_header(h) if header is not None: header.update(tmp) h_len = int(h['labbyt']) d_len = int(h['nx']) * int(h['ny']) * int(h['nz']) i_len = d_len * 4 count = count_images(h) if index >= count: raise IOError, "Index exceeds number of images in stack: %d < %d" % ( index, count) if count > 1 and int(h['istack']) == 0: raise ValueError, "Improperly formatted SPIDER header - not stack but contains mutliple images" offset = h_len * 2 + index * (h_len + i_len) if int( h['istack']) > 0 else h_len if count > 1 or h['istack'] == 2: if file_size(f) != (h_len + count * (h_len + i_len)): raise ValueError, "file size != header: %d != %d - count: %d -- nx:%d,ny:%d,nz:%d" % ( file_size(f), (h_len + count * (h_len + i_len)), count, int(h['nx']), int(h['ny']), int(h['nz'])) else: if file_size(f) != (h_len + count * i_len): f.seek(h_len + index * (h_len + i_len)) h2 = read_spider_header(f) raise ValueError, "file size != header: %d != %d - %d + %d * %d -- %d,%d == %d,%d -- count: " % ( file_size(f), (h_len + count * i_len), h_len, count, i_len, int(h['istack']), int(h['imgnum']), int( h2['istack']), int(h2['imgnum']), int(h['maxim'])) try: f.seek(offset) except: _logger.error("Offset: %s" % str(offset)) raise out = numpy.fromfile(f, dtype=dtype, count=d_len) if header_dtype.newbyteorder()[0] == h.dtype[0]: out = out.byteswap() #assert(out.ravel().shape[0]==d_len) if int(h['nz']) > 1: out = out.reshape(int(h['nz']), int(h['ny']), int(h['nx'])) elif int(h['ny']) > 1: try: out = out.reshape(int(h['ny']), int(h['nx'])) except: _logger.error("%d != %d*%d = %d" % (out.ravel().shape[0], int(h['nx']), int( h['ny']), int(h['nx']) * int(h['ny']))) raise finally: util.close(filename, f) #if header_image_dtype.newbyteorder()==h.dtype:out = out.byteswap() return out
def parse_subject(request, subject, session_attributes={}): """ :param request: JSON data from the Finna API :param subject: Subject or search term of the current session :param session_attributes: session attributes for current session if user has given some :return: Response to AWS server in JSON format """ author = session_attributes.get('author') if subject is "": if author: return parse_author(request, {'author': author}) return util.elicit_intent( session_attributes, "I'm sorry. I was not " "able to catch what book" " you wanted to find. " "Could you please repeat.") if request['status'] == 'OK': result_count = request['resultCount'] # no books was found with the subject if result_count == 0: message = "Oh I'm so sorry, no books was found with search term: "\ + subject if author: message += " written by " + str(author) return util.close({ 'subject': subject, 'author': author }, 'Fulfilled', message) elif result_count == 1: return find_info(request['records'][0]['id']) # if less than five books was found, will be checked all different # locations in books way that there is not same locations twice. elif result_count < 5: real_count = 0 find_locations = [] while real_count < result_count: buildings = request['records'][real_count]['buildings'] for layer in buildings: if re.compile("1/AALTO/([a-z])*/").match(layer['value']): if layer['translated'] not in find_locations: find_locations.append(layer['translated']) real_count += 1 # if author was also found. if author: message = subject + " books by " + str(author) \ + " can be found in " + util.make_string_list(find_locations) else: message = subject + " books can be found in " + \ util.make_string_list(find_locations) return util.close({ 'subject': subject, 'author': author }, 'Fulfilled', message) # there is more than five results, this ask more information from user else: if not author: message = "I found " + str(result_count) + " books with term " \ + subject + ". Please specify an author or a year, " \ " so I can narrow down the search." else: message = "I found " + str(result_count) + " books with " \ + subject + " by author " + author + ". Can you " \ "give the publication date for example to narrow " \ "down the search." return util.elicit_intent({ 'subject': subject, 'author': author }, message) else: return util.close({'author': None}, 'Fulfilled', "Something went wrong")
def iter_images(filename, index=None, header=None): ''' Read a set of SPIDER images :Parameters: filename : str or file object Filename or open stream for a file index : int, optional Index of image to start, if None, start with the first image (Default: None) header : dict, optional Dictionary to hold header values :Returns: out : array Array with image information from the file ''' f = util.uopen(filename, 'rb') if index is None: index = 0 try: h = read_spider_header(f) dtype = numpy.dtype(spi2numpy[float(h['iform'])]) #if header_dtype.newbyteorder()==h.dtype: dtype = dtype.newbyteorder() #if header is not None: util.update_header(header, h, spi2ara, 'spi') tmp=read_header(h) if header is not None: header.update(tmp) h_len = int(h['labbyt']) d_len = int(h['nx']) * int(h['ny']) * int(h['nz']) i_len = d_len * 4 count = count_images(h) if numpy.any(index >= count): raise IOError, "Index exceeds number of images in stack: %s < %d"%(str(index), count) #offset = h_len + 0 * (h_len+i_len) size = ( h_len + count * (h_len+i_len) ) if int(h['istack']) > 0 else (h_len + i_len) if file_size(f) != size: raise ValueError, "file size != header: %d != %d - %d -- %d,%d,%d"%(file_size(f), (h_len + count * (h_len+i_len)), count, int(h['nx']), int(h['ny']), int(h['nz'])) try: f.seek(h_len) except: _logger.error("Offset: %s"%str(h_len)) raise if int(h['istack']) == 0: # This file contains a single image! out = numpy.fromfile(f, dtype=dtype, count=d_len) if header_dtype.newbyteorder()[0]==h.dtype[0]: out = out.byteswap() if int(h['nz']) > 1: out = out.reshape(int(h['nz']), int(h['ny']), int(h['nx'])) elif int(h['ny']) > 1: out = out.reshape(int(h['ny']), int(h['nx'])) yield out return if not hasattr(index, '__iter__'): index = xrange(index, count) else: index = index.astype(numpy.int) last=-1 if count > 1 and int(h['istack']) == 0: raise ValueError, "Improperly formatted SPIDER header - not stack but contains mutliple images" for i in index: if i < 0: raise ValueError, "Cannot have a negative index" if i != (last+1): offset = h_len*2 + i * (h_len+i_len) try: f.seek(int(offset)) except: _logger.error("Offset: %s"%str(offset)) _logger.error("i: %s"%str(i)) raise else: f.seek(h_len, 1) last=i out = numpy.fromfile(f, dtype=dtype, count=d_len) if header_dtype.newbyteorder()[0]==h.dtype[0]: out = out.byteswap() if int(h['nz']) > 1: try: out = out.reshape(int(h['nz']), int(h['ny']), int(h['nx'])) except: _logger.error("%d, %d, %d == %d == %d", int(h['nz']), int(h['ny']), int(h['nx']), numpy.prod((int(h['nz']), int(h['ny']), int(h['nx']))), out.shape[0]) raise elif int(h['ny']) > 1: try: out = out.reshape(int(h['ny']), int(h['nx'])) except: _logger.error("(%d < %d) -- %d, %d == %d == %d", i, count, int(h['ny']), int(h['nx']), numpy.prod((int(h['ny']), int(h['nx']))), out.shape[0]) raise yield out finally: util.close(filename, f)
def write_image(filename, img, index=None, header=None, inplace=False): ''' Write an image array to a file in the MRC format :Parameters: filename : str Name of the output file img : array Image array index : int, optional Index to write image in the stack header : dict, optional Dictionary of header values inplace : bool Write new image to stack without removing the stack ''' #float64 #complex64 if header is None and hasattr(img, 'header'): header = img.header dtype = numpy.complex64 if numpy.iscomplexobj(img) else numpy.float32 try: img = img.astype(dtype) except: raise TypeError, "Unsupported type for SPIDER writing: %s" % str( img.dtype) mode = 'rb+' if index is not None and ( index > 0 or inplace and index > -1) else 'wb+' try: f = util.uopen(filename, mode) except: _logger.error("Mode: %s - Index: %s" % (str(mode), str(index))) raise try: if header is None or not hasattr( header, 'dtype') or not is_format_header(header): h = numpy.zeros(1, header_dtype) even = header[ 'fourier_even'] if header is not None and 'fourier_even' in header else None util.update_header(h, spi_defaults, ara2spi) header = util.update_header(h, header, ara2spi, 'spi') # Image size in header header['nx'] = img.T.shape[0] header['ny'] = img.T.shape[1] if img.ndim > 1 else 1 header['nz'] = img.T.shape[2] if img.ndim > 2 else 1 header['lenbyt'] = img.shape[0] * 4 header['labrec'] = 1024 / int(header['lenbyt']) if 1024 % int(header['lenbyt']) != 0: header['labrec'] = int(header['labrec']) + 1 header['labbyt'] = int(header['labrec']) * int(header['lenbyt']) imgsize = img.ravel().shape[0] * 4 headsize = int(header['labbyt']) header['irec'] = header['labrec'] + header['nx'] # #header['irec'] if numpy.iscomplexobj(img): header['iform'] = 3 if img.ndim == 3 else 1 # determine even or odd Fourier - assumes other dim are padded appropriately if even is None: v = int(round(float(img.shape[1]) / img.shape[0])) v = img.shape[1] / v even = (v % 2) == 0 if even: header['iform'] = -22 if img.ndim == 3 else -12 else: header['iform'] = -21 if img.ndim == 3 else -11 else: header['iform'] = 3 if img.ndim == 3 else 1 fheader = numpy.zeros(int(header['labbyt']) / 4, dtype=numpy.float32) for name, idx in _header_map.iteritems(): fheader[idx - 1] = float(header[name]) if inplace: f.seek(index * (imgsize + headsize) + headsize + headsize) else: if index is not None: fheader[_header_map['maxim'] - 1] = index + 1 fheader[_header_map['imgnum'] - 1] = index + 1 fheader[_header_map['istack'] - 1] = 2 f.seek(0) fheader.tofile(f) fheader[_header_map['istack'] - 1] = 0 f.seek(index * (imgsize + headsize) + headsize) fheader[_header_map['maxim'] - 1] = 0 fheader.tofile(f) img.tofile(f) finally: util.close(filename, f)
def iter_images(filename, index=None, header=None): ''' Read a set of SPIDER images :Parameters: filename : str or file object Filename or open stream for a file index : int, optional Index of image to start, if None, start with the first image (Default: None) header : dict, optional Dictionary to hold header values :Returns: out : array Array with image information from the file ''' f = util.uopen(filename, 'rb') if index is None: index = 0 try: h = read_spider_header(f) dtype = numpy.dtype(spi2numpy[float(h['iform'])]) #if header_dtype.newbyteorder()==h.dtype: dtype = dtype.newbyteorder() #if header is not None: util.update_header(header, h, spi2ara, 'spi') tmp = read_header(h) if header is not None: header.update(tmp) h_len = int(h['labbyt']) d_len = int(h['nx']) * int(h['ny']) * int(h['nz']) i_len = d_len * 4 count = count_images(h) if numpy.any(index >= count): raise IOError, "Index exceeds number of images in stack: %s < %d" % ( str(index), count) #offset = h_len + 0 * (h_len+i_len) size = (h_len + count * (h_len + i_len)) if int(h['istack']) > 0 else (h_len + i_len) if file_size(f) != size: raise ValueError, "file size != header: %d != %d - %d -- %d,%d,%d" % ( file_size(f), (h_len + count * (h_len + i_len)), count, int(h['nx']), int(h['ny']), int(h['nz'])) try: f.seek(h_len) except: _logger.error("Offset: %s" % str(h_len)) raise if int(h['istack']) == 0: # This file contains a single image! out = numpy.fromfile(f, dtype=dtype, count=d_len) if header_dtype.newbyteorder()[0] == h.dtype[0]: out = out.byteswap() if int(h['nz']) > 1: out = out.reshape(int(h['nz']), int(h['ny']), int(h['nx'])) elif int(h['ny']) > 1: out = out.reshape(int(h['ny']), int(h['nx'])) yield out return if not hasattr(index, '__iter__'): index = xrange(index, count) else: index = index.astype(numpy.int) last = -1 if count > 1 and int(h['istack']) == 0: raise ValueError, "Improperly formatted SPIDER header - not stack but contains mutliple images" for i in index: if i < 0: raise ValueError, "Cannot have a negative index" if i != (last + 1): offset = h_len * 2 + i * (h_len + i_len) try: f.seek(int(offset)) except: _logger.error("Offset: %s" % str(offset)) _logger.error("i: %s" % str(i)) raise else: f.seek(h_len, 1) last = i out = numpy.fromfile(f, dtype=dtype, count=d_len) if header_dtype.newbyteorder()[0] == h.dtype[0]: out = out.byteswap() if int(h['nz']) > 1: try: out = out.reshape(int(h['nz']), int(h['ny']), int(h['nx'])) except: _logger.error( "%d, %d, %d == %d == %d", int(h['nz']), int(h['ny']), int(h['nx']), numpy.prod((int(h['nz']), int(h['ny']), int(h['nx']))), out.shape[0]) raise elif int(h['ny']) > 1: try: out = out.reshape(int(h['ny']), int(h['nx'])) except: _logger.error("(%d < %d) -- %d, %d == %d == %d", i, count, int(h['ny']), int(h['nx']), numpy.prod((int(h['ny']), int(h['nx']))), out.shape[0]) raise yield out finally: util.close(filename, f)
def iter_images(filename, index=None, header=None, no_strict_mrc=False): ''' Read a set of SPIDER images :Parameters: filename : str or file object Filename or open stream for a file index : int, optional Index of image to start, if None, start with the first image (Default: None) header : dict, optional Output dictionary to place header values no_strict_mrc : bool Perform strict MRC header checking (recommended) - Only EPU MRC files and Yifan's frame alignment require this to be off. :Returns: out : array Array with image information from the file ''' f = util.uopen(filename, 'rb') if index is None: index = 0 try: h = read_mrc_header(f, no_strict_mrc) count = count_images(h) #if header is not None: util.update_header(header, h, mrc2ara, 'mrc') tmp = read_header(h) if header is not None: header.update(tmp) d_len = h['nx'][0] * h['ny'][0] dtype = numpy.dtype(mrc2numpy[h['mode'][0]]) offset = 1024 + int(h['nsymbt']) + 0 * d_len * dtype.itemsize try: f.seek(int(offset)) except: _logger.error("%s -- %s" % (str(offset), str(offset.__class__.__name__))) raise if not hasattr(index, '__iter__'): index = xrange(index, count) else: index = index.astype(numpy.int) last = 0 total = file_size(f) if total != (1024 + int(h['nsymbt']) + int(h['nx'][0]) * int(h['ny'][0]) * int(h['nz'][0]) * dtype.itemsize): raise util.InvalidHeaderException, "file size != header: %d != %d -- %d" % ( total, (1024 + int(h['nsymbt']) + int(h['nx'][0]) * int(h['ny'][0]) * int(h['nz'][0]) * dtype.itemsize), int(h['nsymbt'])) for i in index: if i != (last + 1): f.seek( int(1024 + int(h['nsymbt']) + i * d_len * dtype.itemsize)) out = util.fromfile(f, dtype=dtype, count=d_len) out = reshape_data(out, h, index, count) if header_image_dtype.newbyteorder()[0] == h.dtype[0]: out = out.byteswap() yield out finally: util.close(filename, f)
def read_image(filename, index=None, header=None, cache=None, no_strict_mrc=False, force_volume=False): ''' Read an image from the specified file in the MRC format :Parameters: filename : str or file object Filename or open stream for a file index : int, optional Index of image to get, if None, first image (Default: None) header : dict, optional Output dictionary to place header values no_strict_mrc : bool Perform strict MRC header checking (recommended) - Only EPU MRC files and Yifan's frame alignment require this to be off. force_volume : bool For image to be read as a volume :Returns: out : array Array with image information from the file ''' idx = 0 if index is None else index f = util.uopen(filename, 'rb') try: h = read_mrc_header(f, no_strict_mrc) #if header is not None: util.update_header(header, h, mrc2ara, 'mrc') tmp = read_header(h, force_volume=force_volume) if header is not None: header.update(tmp) count = count_images(h) if idx >= count: raise IOError, "Index exceeds number of images in stack: %d < %d" % ( idx, count) if index is None and (count == h['nx'][0] or force_volume): d_len = h['nx'][0] * h['ny'][0] * h['nz'][0] else: d_len = h['nx'][0] * h['ny'][0] dtype = numpy.dtype(mrc2numpy[h['mode'][0]]) offset = 1024 + int(h['nsymbt']) + idx * d_len * dtype.itemsize total = file_size(f) if total != (1024 + int(h['nsymbt']) + int(h['nx'][0]) * int(h['ny'][0]) * int(h['nz'][0]) * dtype.itemsize): raise util.InvalidHeaderException, "file size != header: %d != %d -- %s, %d" % ( total, (1024 + int(h['nsymbt']) + int(h['nx'][0]) * int(h['ny'][0]) * int(h['nz'][0]) * dtype.itemsize), str(idx), int(h['nsymbt'])) f.seek(int(offset)) out = util.fromfile(f, dtype=dtype, count=d_len) out = reshape_data(out, h, index, count, force_volume) if header_image_dtype.newbyteorder()[0] == h.dtype[0]: out = out.byteswap() finally: util.close(filename, f) #assert(numpy.alltrue(numpy.logical_not(numpy.isnan(out)))) #if header_image_dtype.newbyteorder()==h.dtype:out = out.byteswap() return out
def exit(self, widget, data=None): self.stop() close()
def symTest(): s = Sym().syms( ['y', 'y', 'y', 'y', 'y', 'y', 'y', 'y', 'y', 'n', 'n', 'n', 'n', 'n']) assert (util.close(s.symEnt(), 0.9403)) print(s.symEnt())
def read_image(filename, index=None, header=None): ''' Read an image from the specified file in the SPIDER format :Parameters: filename : str or file object Filename or open stream for a file index : int, optional Index of image to get, if None, first image (Default: None) header : dict, optional Dictionary to hold header values :Returns: out : array Array with image information from the file ''' f = util.uopen(filename, 'rb') h = None try: if index is None: index = 0 h = read_spider_header(f) dtype = numpy.dtype(spi2numpy[float(h['iform'])]) #if header_dtype.newbyteorder()==h.dtype: dtype = dtype.newbyteorder() # - changed #if header is not None: util.update_header(header, h, spi2ara, 'spi') tmp=read_header(h) if header is not None: header.update(tmp) h_len = int(h['labbyt']) d_len = int(h['nx']) * int(h['ny']) * int(h['nz']) i_len = d_len * 4 count = count_images(h) if index >= count: raise IOError, "Index exceeds number of images in stack: %d < %d"%(index, count) if count > 1 and int(h['istack']) == 0: raise ValueError, "Improperly formatted SPIDER header - not stack but contains mutliple images" offset = h_len*2 + index * (h_len+i_len) if int(h['istack']) > 0 else h_len if count > 1 or h['istack'] == 2: if file_size(f) != (h_len + count * (h_len+i_len)): raise ValueError, "file size != header: %d != %d - count: %d -- nx:%d,ny:%d,nz:%d"%(file_size(f), (h_len + count * (h_len+i_len)), count, int(h['nx']), int(h['ny']), int(h['nz'])) else: if file_size(f) != (h_len + count * i_len): f.seek(h_len + index * (h_len+i_len)) h2 = read_spider_header(f) raise ValueError, "file size != header: %d != %d - %d + %d * %d -- %d,%d == %d,%d -- count: "%(file_size(f), (h_len + count * i_len), h_len, count, i_len, int(h['istack']), int(h['imgnum']), int(h2['istack']), int(h2['imgnum']), int(h['maxim']) ) try: f.seek(offset) except: _logger.error("Offset: %s"%str(offset)) raise out = numpy.fromfile(f, dtype=dtype, count=d_len) if header_dtype.newbyteorder()[0]==h.dtype[0]: out = out.byteswap() #assert(out.ravel().shape[0]==d_len) if int(h['nz']) > 1: out = out.reshape(int(h['nz']), int(h['ny']), int(h['nx'])) elif int(h['ny']) > 1: try: out = out.reshape(int(h['ny']), int(h['nx'])) except: _logger.error("%d != %d*%d = %d"%(out.ravel().shape[0], int(h['nx']), int(h['ny']), int(h['nx'])*int(h['ny']))) raise finally: util.close(filename, f) #if header_image_dtype.newbyteorder()==h.dtype:out = out.byteswap() return out
def write_image(filename, img, index=None, header=None, inplace=False): ''' Write an image array to a file in the MRC format :Parameters: filename : str Name of the output file img : array Image array index : int, optional Index to write image in the stack header : dict, optional Dictionary of header values inplace : bool Write new image to stack without removing the stack ''' if header is None and hasattr(img, 'header'): header = img.header try: img = img.astype(mrc2numpy[numpy2mrc[img.dtype.type]]) except: raise TypeError, "Unsupported type for MRC writing: %s" % str( img.dtype) mode = 'rb+' if index is not None and ( index > 0 or inplace and index > -1) else 'wb+' f = util.uopen(filename, mode) if header is None or not hasattr(header, 'dtype') or not is_format_header(header): h = numpy.zeros(1, header_image_dtype) util.update_header(h, mrc_defaults, ara2mrc) pix = header.get('apix', 1.0) if header is not None else 1.0 header = util.update_header(h, header, ara2mrc, 'mrc') header['nx'] = img.T.shape[0] header['ny'] = img.T.shape[1] if img.ndim > 1 else 1 if header['nz'] == 0: header['nz'] = img.shape[2] if img.ndim > 2 else 1 header['mode'] = numpy2mrc[img.dtype.type] header['mx'] = header['nx'] header['my'] = header['ny'] header['mz'] = header['nz'] header['xlen'] = header['nx'] * pix header['ylen'] = header['ny'] * pix header['zlen'] = header['nz'] * pix header['alpha'] = 90 header['beta'] = 90 header['gamma'] = 90 header['mapc'] = 1 header['mapr'] = 2 header['maps'] = 3 header['amin'] = numpy.min(img) header['amax'] = numpy.max(img) header['amean'] = numpy.mean(img) header['map'] = 'MAP' header['byteorder'] = byteorderint2[sys.byteorder] #'DA\x00\x00' header['nlabels'] = 1 header['label0'] = 'Created by Arachnid' #header['byteorder'] = numpy.fromstring('\x44\x41\x00\x00', dtype=header['byteorder'].dtype) #header['rms'] = numpy.std(img) if img.ndim == 3: header['nxstart'] = header['nx'] / -2 header['nystart'] = header['ny'] / -2 header['nzstart'] = header['nz'] / -2 if index is not None: stack_count = index + 1 header['nz'] = stack_count header['mz'] = stack_count header['zlen'] = stack_count #header['zorigin'] = stack_count/2.0 try: if inplace: f.seek( int(1024 + int(h['nsymbt']) + index * img.ravel().shape[0] * img.dtype.itemsize)) elif f != filename: f.seek(0) header.tofile(f) if index > 0: f.seek( int(1024 + int(h['nsymbt']) + index * img.ravel().shape[0] * img.dtype.itemsize)) img.tofile(f) finally: util.close(filename, f)
def handle_read(self): data = self.recv(1024) if data == '': self.handle_close() return log.l.LogIt('RTC006', 'D', 'cmd i: %s', (data)) args = string.split(data) if len(args) == 0: return; self.mode = args[0] self.target = args[1] args = args[2:] log.l.LogIt('RTC007', '1', '%s', (str(args))) if len(args) == 0: return #-------------------- if args[0] == 'close': self.senddata.append(util.close(self.mode == 'router', self.target, args[1:])) #-------------------- elif args[0] == 'data': if self.mode == 'domain': self.senddata.append('command: '+args[0]+': address only to router') if self.mode == 'router': self.senddata.append(util.data(args[1:])) elif args[0] == 'dump': if self.mode == 'domain': self.senddata.append('command: '+args[0]+': address only to router') if self.mode == 'router': self.senddata.append(util.dump()) #-------------------- elif args[0] == 'event': if self.mode == 'domain': self.senddata.append('command: '+args[0]+': address only to router') if self.mode == 'router': self.senddata.append(util.event(args[1:])) #-------------------- elif args[0] == 'fb' or args[0] == 'fallback' or args[0] == 'secondary': if len(args) > 1 and args[1] == 'auto': auto = 1 else: auto = 0 if self.mode == 'domain': cfg.domain[self.target]['fallback'] = 1 self.senddata.append(util.switch_secondary(self.target, auto)) if self.mode == 'router': for i in cfg.domain.keys(): if cfg.domain[i]['apr'] == cfg.name: cfg.domain[i]['fallback'] = 1 self.senddata.append(i+' '+util.switch_secondary(i, auto)) #-------------------- elif args[0] == 'ff' or args[0] == 'fallforward' or args[0] == 'primary': if len(args) > 1 and args[1] == 'auto': auto = 1 else: auto = 0 if self.mode == 'domain': cfg.domain[self.target]['fallback'] = 0 self.senddata.append(util.switch_primary(self.target, auto)) if self.mode == 'router': for i in cfg.domain.keys(): if cfg.domain[i]['apr'] == cfg.name: cfg.domain[i]['fallback'] = 0 self.senddata.append(i+' '+util.switch_primary(i, auto)) #-------------------- elif args[0] == 'pvc': if self.mode == 'domain': self.senddata.append('command: '+args[0]+': address only to router') if self.mode == 'router': self.senddata.append(util.pvc(args[1:])) #-------------------- elif args[0] == 'refresh': evt_hdlr.refresh() self.senddata.append('status refreshed') #-------------------- elif args[0] == 'set': self.senddata = util.set(self.mode == 'router', self.target, args[1:]) #-------------------- elif args[0] == 'sna': if self.mode == 'domain': self.senddata.append('command: '+args[0]+': address only to router') if self.mode == 'router': self.senddata.append(util.sna(args[1:])) #-------------------- elif args[0] == 'status': if self.mode == 'domain': self.senddata.append(util.status(self.target)) if self.mode == 'router': for i in cfg.domain.keys(): if cfg.domain[i]['apr'] == cfg.name: self.senddata.append(i+' '+util.status(i)) if len(self.senddata) == 0: self.senddata.append('not active') #-------------------- elif args[0] == 'stop': if self.mode == 'domain': self.senddata.append('command: '+args[0]+': address only to router') if self.mode == 'router': log.l.LogIt('RTC008', 'I', 'command termination', ()) cfg.stopping = 1 msg = '%s terminating' % self.target self.senddata.append(msg) for i in cfg.domain.keys(): util.closeall(i) #-------------------- elif args[0] == 'trace': if self.mode == 'domain': self.senddata.append('command: '+args[0]+': address only to router') if self.mode == 'router': if len(args) > 1: log.l.SetTraceLevel(int(args[1])) self.senddata.append('trace level %s' % (args[1])) log.l.LogIt('RTC009', 'I', 'command trace %s', (args[1])) else: level = log.l.GetTraceLevel() self.senddata.append('trace level %d' % (level)) log.l.LogIt('RTC010', 'I', 'command get trace: %d', (level)) #-------------------- elif args[0] == 'version': msg = ver.getVersion() if cfg.snasrv_version != '': msg = msg + ' snasrv: ' + cfg.snasrv_version self.senddata.append(msg) #-------------------- else: self.senddata.append('command: '+args[0]+': not implemented')