def set_book_page(self, state, page): width, height = utility.dimensions(state) book_n = state['user']['current_book'] books = OrderedDict(state['user']['books']) book = books[book_n].set_page(page) bookmarks = tuple(bm for bm in book.bookmarks if bm != 'deleted') book = book._replace(bookmarks=bookmarks) books[book_n] = book return state.copy( user=state['user'].copy(books=FrozenOrderedDict(books)), location='book', home_menu_visible=False)
def insert_bookmark(self, state, _): number = state['user']['current_book'] books = OrderedDict(state['user']['books']) book = books[number] page = book.page_number if page not in book.bookmarks: bookmarks = sorted(book.bookmarks + tuple([page])) book = book._replace(bookmarks=tuple(bookmarks)) books[number] = book return state.copy(user=state['user'].copy( books=FrozenOrderedDict(books))) return state
def _create_layer_dict(self): layer_ids = self._compute_layer_ids() layer_dict = OrderedDict() last_layer_id = '' last_layer_type = None current_layers = [ ] # Layers that could not be wrapped in objects yet, because they were missing the activation function. for layer_id in layer_ids: keras_layer_obj = self._model.get_layer(layer_id) layer_type = keras_layer_obj.__class__.__name__ # Check whether the layer type is considered a real layer. if layer_type in self._layer_types_to_classes.keys(): # Check that if the layer is a neural layer it contains an activation function. # If so the layer can be added savely. # Otherwise take the next activation function and merge the layers if ((layer_type in self._neural_layer_types and keras_layer_obj.activation.__name__ != 'linear') or (layer_type in self._transformation_layer_types)): # Check that there is no unfinished layer with missing activation function. # If not add the layer. if not current_layers: layer_dict[layer_id] = self._layer_types_to_classes[ layer_type](self, [keras_layer_obj]) else: raise ParsingError('Missing activation function.') else: # Check that there is no other unfinished layer before that one. if not current_layers: current_layers.append(keras_layer_obj) last_layer_id = layer_id last_layer_type = layer_type else: raise ParsingError( 'Two consectutive layers with no activation function.' ) elif layer_type == 'Activation': # Check that there was a layer before without activation function and merge. if current_layers: current_layers.append(keras_layer_obj) layer_dict[last_layer_id] = self._layer_types_to_classes[ last_layer_type](self, current_layers) current_layers = [] else: raise ParsingError( 'Two activation layers after each other.') else: raise ParsingError( 'Not sure how to deal with that layer type at that position.' ) return FrozenOrderedDict(layer_dict)
def make_hashable(values): """ Make the elements of iterable `values` hashable. Or if `values` is a singleton, make it hashable. """ def _hashable_one(value): """...""" if isinstance(value, Hashable): return value try: return FrozenOrderedDict(value) except TypeError as frozen_ordered_dict_error: try: return tuple(value) except TypeError as tuple_error: pass raise TypeError( """ {} object is neither hashable, mapping, or iterable: Error when tried to create FrozenOrderedDict: {} Error when tried to create a tuple: {} """\ .format( value, frozen_ordered_dict_error, tuple_error)) def _ordered(dict_like): """...""" if isinstance(dict_like, (FrozenOrderedDict, OrderedDict)): return _ordered assert isinstance(dict_like, dict) return OrderedDict( sorted(dict_like.items(), key=lambda kv: kv[0])) if not isinstance(values, Iterable): return _hashable_one(values) if isinstance(values, six.string_types): return _hashable_one(values) if isinstance(values, Mapping): return\ FrozenOrderedDict( (make_hashable(key), make_hashable(value)) for key, value in _ordered(values).items()) generator = (make_hashable(value) for value in values) if isinstance(values, Generator): return generator if isinstance(values, pd.Series): return pd.Series(generator) return tuple(generator)
def close_menu(self, state, value): books = state['user']['books'] # fully delete deleted bookmarks changed_books = OrderedDict() for filename in books: book = books[filename] bookmarks = tuple(bm for bm in book.bookmarks if bm != 'deleted') book = book._replace(bookmarks=bookmarks) changed_books[filename] = book bookmarks_menu = state['bookmarks_menu'] return state.copy(location='book', bookmarks_menu=bookmarks_menu.copy(page=0), home_menu_visible=False, go_to_page_selection='', help_menu=frozendict({'visible': False, 'page': 0}), user=state['user'].copy(books=FrozenOrderedDict(changed_books)))
async def read_user_state(path): global prev global manual book_files = utility.find_files(path, ('brf', 'pef')) main_toml = os.path.join(path, 'sd-card', USER_STATE_FILE) current_book = manual_filename if os.path.exists(main_toml): main_state = toml.load(main_toml) if 'current_book' in main_state: current_book = main_state['current_book'] if not current_book == manual_filename: current_book = os.path.join(path, current_book) manual_toml = os.path.join(path, to_state_file(manual_filename)) if os.path.exists(manual_toml): t = toml.load(manual_toml) if 'current_page' in t: manual = manual._replace(page_number=t['current_page'] - 1) if 'bookmarks' in t: manual = manual._replace(bookmarks=tuple( sorted(manual.bookmarks + tuple(bm - 1 for bm in t['bookmarks'])))) books = OrderedDict({manual_filename: manual}) for book_file in book_files: toml_file = to_state_file(book_file) book = BookFile(filename=book_file, width=40, height=9) if os.path.exists(toml_file): t = toml.load(toml_file) if 'current_page' in t: book = book._replace(page_number=t['current_page'] - 1) if 'bookmarks' in t: book = book._replace(bookmarks=tuple( sorted(book.bookmarks + tuple(bm - 1 for bm in t['bookmarks'])))) try: books[book_file] = await init(book) except Exception as e: log.warning('could not open {}'.format(book_file)) log.warning(e) if current_book not in books: current_book = manual_filename user_state = frozendict(books=FrozenOrderedDict(books), current_book=current_book) prev = user_state return user_state
def go_to_page(self, state, page): width, height = utility.dimensions(state) if page < 0: page = 0 if state['help_menu']['visible']: location = 'help_menu' else: location = state['location'] if location == 'library': books = state['user']['books'] max_pages = (len(books) - 1) // (height - 1) if page > max_pages: page = max_pages elif page < 0: page = 0 library = frozendict({'page': page}) return state.copy(library=library) elif location == 'book': book_n = state['user']['current_book'] book = state['user']['books'][book_n] books = OrderedDict(state['user']['books']) book = book.set_page(page) books[book_n] = book return state.copy(user=state['user'].copy( books=FrozenOrderedDict(books))) elif location == 'bookmarks_menu': book_n = state['user']['current_book'] book = state['user']['books'][book_n] bookmarks_data = book.bookmarks max_pages = (len(bookmarks_data) - 1) // height if page > max_pages: page = max_pages bookmarks_menu = state['bookmarks_menu'].copy(page=page) return state.copy(bookmarks_menu=bookmarks_menu) elif location == 'help_menu': max_pages = 1 if page >= max_pages: page = max_pages - 1 return state.copy(help_menu=state['help_menu'].copy(page=page)) return state
def _hashable_one(value): """...""" if isinstance(value, Hashable): return value try: return FrozenOrderedDict(value) except TypeError as frozen_ordered_dict_error: try: return tuple(value) except TypeError as tuple_error: pass raise TypeError( """ {} object is neither hashable, mapping, or iterable: Error when tried to create FrozenOrderedDict: {} Error when tried to create a tuple: {} """\ .format( value, frozen_ordered_dict_error, tuple_error))
def delete_bookmark(self, state, n): width, height = state_helpers.dimensions(state) # adjust for title height -= 1 page = state['bookmarks_menu']['page'] book_n = state['user']['current_book'] books = OrderedDict(state['user']['books']) book = books[book_n] bookmarks = book.bookmarks line = page * height # don't delete go-to-start end and go-to-end bookmarks if (line + n) == 0 or (line + n) == (len(bookmarks) - 1): return state changed_bookmarks = list(bookmarks[line:line + height]) if n >= len(changed_bookmarks): return state changed_bookmarks[n] = 'deleted' bookmarks = bookmarks[0:line] + tuple(changed_bookmarks) \ + bookmarks[line + height:len(bookmarks)] books[book_n] = book._replace(bookmarks=bookmarks) return state.copy(user=state['user'].copy( books=FrozenOrderedDict(books)))
class DataObject(object): def __init__(self, **kwargs): for column in self.__class__.columns: if column.name in kwargs: try: converted_value = column.convert(kwargs[column.name]) except Exception as e: traceback.print_exc(e) try: error_string = 'Unable to convert value %s for field %s' % ( kwargs[column.name], column.name) except: error_string = 'Unable to convert value for field %s' % ( column.name) raise ValueError(error_string) setattr(self, column.name, converted_value) del kwargs[column.name] else: setattr(self, column.name, None) if kwargs: raise TypeError('Unexpected argument(s) initializing %s: %s' % (self.__class__.__name__, str(kwargs))) for k, v in self.__class__.relationships.items(): if v[1]: setattr(self, k, []) else: setattr(self, k, None) @classmethod def to_sqa_table(cls, metadata, name, **kwargs): ''' Create a sqalchemy Table object corresponding to this class. Use kwargs to override any desired columns. ''' bad_kwargs = set(kwargs.keys()) - set( map(lambda col: col.name, cls.columns)) if bad_kwargs: raise ValueError( 'Optional keyword argument names must correspond to field names. The following names are not compliant: %s' % str(sorted(bad_kwargs))) cols = [ col.to_sqa() if col.name not in kwargs else kwargs[col.name] for col in cls.columns ] return Table(name, metadata, *cols) def __getstate__(self): state = {} for col in self.columns: if hasattr(self, col.name): state[col.name] = getattr(self, col.name) for k in sorted(self.relationships.keys()): state[k] = getattr(self, k) return state def __setstate__(self, state): self.__dict__.update(state) def __eq__(self, other): # TODO: This could be made much faster by a custom implementation return self.__class__ is other.__class__ and self.__getstate__( ) == other.__getstate__() def __hash__(self): return hash((self.__class__, frozendict(valmap(freeze, self.__getstate__()).items()))) def to_row(self): return [ col.unconvert(getattr(self, col.name, None)) for col in self.columns ] @classmethod def header(cls): return [col.name for col in cls.columns] @classmethod def init_schema(cls): ''' Called after the schema has been added. Allows class to do any necessary initialization steps that require schema information. ''' pass @classmethod def init_relationships(cls): ''' Called after a relationship is added. Allows class to do any necessary initialization steps that require relationship information. ''' pass relationships = FrozenOrderedDict() @classmethod def subtypes(cls): stack = [cls] result = [] while stack: item = stack.pop() stack.extend(item.__subclasses__()) result.append(item) return result @classmethod def typerank(cls): if cls.concrete(): d = {cls: 0} else: d = {t: i for i, t in enumerate(cls.subtypes())} def _typerank(obj): return d[obj] return _typerank @classmethod def objrank(cls): tr = cls.typerank() def _objrank(obj): return tr(type(obj)) return _objrank @classmethod def relationship_sort_key(cls, relationship): klass = cls.relationships[relationship][0] typerank = klass.typerank() def sort_key(obj): return (typerank(type(obj)), obj.sort_key()) return sort_key @classproperty @classmethod @abstractmethod def partition_attribute(cls): print(cls) raise NotImplementedError def set_container_key(self, key): translation = self.translate_container_key(key) for k, v in translation.items(): setattr(self, k, v) @classmethod def translate_container_key(cls, key): result = {} for i, (k, v) in enumerate(cls.container_key_): result[v] = key[i] return result def set_identity_key(self, key): translation = self.translate_identity_key(key) for k, v in translation.items(): setattr(self, k, v) @classmethod def translate_identity_key(cls, key): result = {} for i, (k, v) in enumerate(cls.identity_key_): result[v] = key[i] return result def container_key(self): return tuple(getattr(self, v) for k, v in self.container_key_) def identity_key(self): return tuple(getattr(self, v) for k, v in self.identity_key_) def sort_key(self): return tuple(getattr(self, key) for key in self.sort_key_) @classproperty @classmethod def sort_column_numbers(cls): return [cls.header().index(key) for key in cls.sort_key_] @classproperty @classmethod def sort_column_names(cls): return cls.sort_key_ # def __richcmp__(DataObject self, DataObject other, int op): # if op == 0:# < # return self.sort_key() < other.sort_key() # elif op == 1:# == # return self.__class__ is other.__class__ and self.__getstate__() == other.__getstate__() # elif op == 2:# > # return self.sort_key() > other.sort_key() # elif op == 3:# <= # return self.sort_key() <= other.sort_key() # elif op == 4:# != # return not (self.__class__ is other.__class__ and self.__getstate__() == other.__getstate__()) # elif op == 5:# >= # return self.sort_key() >= other.sort_key() def __lt__(self, other): if not isinstance(other, DataObject): return NotImplemented try: return self.sort_key() < other.sort_key() except AttributeError: return NotImplemented def __le__(self, other): if not isinstance(other, DataObject): return NotImplemented try: return self.sort_key() <= other.sort_key() except AttributeError: return NotImplemented def __gt__(self, other): if not isinstance(other, DataObject): return NotImplemented try: return self.sort_key() > other.sort_key() except AttributeError: return NotImplemented def __ge__(self, other): if not isinstance(other, DataObject): return NotImplemented try: return self.sort_key() >= other.sort_key() except AttributeError: return NotImplemented @classmethod def concrete(cls): return not cls.__subclasses__() @classmethod def reader_class(cls, config): if cls.__subclasses__(): assert cls not in config return PolymorphicReader elif cls.relationships: if cls in config: return CompoundReader else: return ImplicitReader else: assert cls in config assert isinstance(config[cls], SimpleReaderConfig ), 'No config found for class %s' % cls.__name__ return SimpleReader @classmethod def writer_class(cls, config): if cls.__subclasses__(): assert cls not in config return PolymorphicWriter elif cls.relationships: if cls in config: return CompoundWriter else: return ImplicitWriter else: assert cls in config assert isinstance(config[cls], SimpleWriterConfig) return SimpleWriter @classmethod def reader(cls, config): return cls.reader_class(config)(cls, config) @classmethod def writer(cls, config): return cls.writer_class(config)(cls, config)
def go_to_page(self, state, page): width, height = state_helpers.dimensions(state) if page < 0: page = 0 if state['help_menu']['visible']: location = 'help_menu' else: location = state['location'] if location == 'library': books = state['user']['books'] max_pages = (len(books) - 1) // (height - 1) if page > max_pages: page = max_pages elif page < 0: page = 0 library = frozendict({'page': page}) return state.copy(library=library) elif location == 'book': book_n = state['user']['current_book'] book = state['user']['books'][book_n] books = OrderedDict(state['user']['books']) book = book.set_page(page) books[book_n] = book return state.copy(user=state['user'].copy(books=FrozenOrderedDict(books))) elif location == 'bookmarks_menu': book_n = state['user']['current_book'] book = state['user']['books'][book_n] bookmarks_data = book.bookmarks max_pages = (len(bookmarks_data) - 1) // height if page > max_pages: page = max_pages bookmarks_menu = state['bookmarks_menu'].copy(page=page) return state.copy(bookmarks_menu=bookmarks_menu) elif location == 'language': lang_n = state['user'].get('current_language', 'en_GB:en') lang = list(state['languages']['available'].keys())[lang_n] language_menu = state['user'].copy(current_language=lang) return state.copy(language=language_menu) elif location == 'help_menu': # To calculate help page bounds and ensure we stay within # them, do a dummy render of the help and count the pages. mapping = { 'book': render_book_help, 'library': render_library_help, 'system_menu': render_system_help, 'language': render_language_help, 'bookmarks_menu': render_bookmarks_help, 'go_to_page': render_gtp_help, } if state['home_menu_visible']: mapping['book'] = render_home_menu_help help_getter = mapping.get(state['location']) num_pages = 1 if help_getter: num_pages = len(help_getter(width, height)) // height if page >= num_pages: page = num_pages - 1 return state.copy(help_menu=state['help_menu'].copy(page=page)) return state
def remove_book(self, state, book): books = OrderedDict(state['user']['books']) if book.filename in books: del books[book.filename] books = FrozenOrderedDict(books) return state.copy(user=state['user'].copy(books=books))
def sort_books(books): return FrozenOrderedDict( sorted(books.items(), key=lambda x: x[1].title.lower()))
def produce( self, *, inputs: container.DataFrame, timeout: float = None, iterations: int = None) -> base.CallResult[container.DataFrame]: cols = ["idx", "name", "rank"] # Make sure the target column is of a valid type and return no ranked features if it isn't. target_idx = self.hyperparams["target_col_index"] if not self._can_use_column(inputs.metadata, target_idx): return base.CallResult(container.DataFrame(data={}, columns=cols)) # check if target is discrete or continuous semantic_types = inputs.metadata.query_column( target_idx)["semantic_types"] discrete = len(set(semantic_types).intersection( self._discrete_types)) > 0 # make a copy of the inputs and clean out any missing data feature_df = inputs.copy() if self.hyperparams["sub_sample"]: sub_sample_size = (self.hyperparams["sub_sample_size"] if self.hyperparams["sub_sample_size"] < inputs.shape[0] else inputs.shape[0]) rows = random.sample_without_replacement(inputs.shape[0], sub_sample_size) feature_df = feature_df.iloc[rows, :] # makes sure that if an entire column is NA, we remove that column, so as to not remove ALL rows cols_to_drop = feature_df.columns[feature_df.isna().sum() == feature_df.shape[0]] feature_df.drop(columns=cols_to_drop, inplace=True) feature_df.dropna(inplace=True) # split out the target feature target_df = feature_df.iloc[:, feature_df.columns. get_loc(inputs.columns[target_idx])] # drop features that are not compatible with ranking feature_indices = set( inputs.metadata.list_columns_with_semantic_types( self._semantic_types)) role_indices = set( inputs.metadata.list_columns_with_semantic_types(self._roles)) feature_indices = feature_indices.intersection(role_indices) feature_indices.remove(target_idx) for categ_ind in inputs.metadata.list_columns_with_semantic_types( ("https://metadata.datadrivendiscovery.org/types/CategoricalData", )): if categ_ind in feature_indices: if (np.unique(inputs[inputs.columns[categ_ind]]).shape[0] == inputs.shape[0]): feature_indices.remove(categ_ind) elif (inputs.metadata.query( (metadata_base.ALL_ELEMENTS, categ_ind))["structural_type"] == str): feature_df[inputs.columns[categ_ind]] = pd.to_numeric( feature_df[inputs.columns[categ_ind]]) text_indices = inputs.metadata.list_columns_with_semantic_types( self._text_semantic) tfv = TfidfVectorizer(max_features=20) column_to_text_features = {} text_feature_indices = [] for text_index in text_indices: if (text_index not in feature_indices and text_index in role_indices and text_index != target_idx): word_features = tfv.fit_transform( feature_df[inputs.columns[text_index]]) if issparse(word_features): column_to_text_features[inputs.columns[ text_index]] = pd.DataFrame.sparse.from_spmatrix( word_features) else: column_to_text_features[ inputs.columns[text_index]] = word_features text_feature_indices.append(text_index) text_feature_indices = set(text_feature_indices) # return an empty result if all features were incompatible numeric_features = len(feature_indices) > 0 if not numeric_features and len(column_to_text_features) == 0: return base.CallResult(container.DataFrame(data={}, columns=cols)) all_indices = set(range(0, inputs.shape[1])) skipped_indices = all_indices.difference( feature_indices.union(text_feature_indices)) # remove columns that were dropped feature_indices = feature_indices - set( [inputs.columns.get_loc(c) for c in cols_to_drop]) for i, v in enumerate(skipped_indices): feature_df.drop(inputs.columns[v], axis=1, inplace=True) # figure out the discrete and continuous feature indices and create an array # that flags them feature_columns = inputs.columns[list(feature_indices)] numeric_data = feature_df[feature_columns] discrete_indices = inputs.metadata.list_columns_with_semantic_types( self._discrete_types) discrete_flags = [False] * numeric_data.shape[1] for v in discrete_indices: col_name = inputs.columns[v] if col_name in numeric_data: # only mark columns with a least 1 duplicate value as discrete when predicting # a continuous target - there's a check in the bowels of MI code that will throw # an exception otherwise if numeric_data[col_name].duplicated().any() and not discrete: col_idx = numeric_data.columns.get_loc(col_name) discrete_flags[col_idx] = True target_np = target_df.values # compute mutual information for discrete or continuous target ranked_features_np = np.empty([0]) text_ranked_features_np = np.empty((len(column_to_text_features), )) if discrete: if numeric_features: ranked_features_np = mutual_info_classif( numeric_data.values, target_np, discrete_features=discrete_flags, n_neighbors=self.hyperparams["k"], random_state=self._random_seed, ) for i, column in enumerate(column_to_text_features): text_rankings = mutual_info_classif( column_to_text_features[column], target_np, discrete_features=[False] * column_to_text_features[column].shape[1], n_neighbors=self.hyperparams["k"], random_state=self._random_seed, ) sum_text_rank = np.sum(text_rankings) text_ranked_features_np[i] = sum_text_rank else: if numeric_features: ranked_features_np = mutual_info_regression( numeric_data.values, target_np, discrete_features=discrete_flags, n_neighbors=self.hyperparams["k"], random_state=self._random_seed, ) for i, column in enumerate(column_to_text_features): text_rankings = mutual_info_regression( column_to_text_features[column], target_np, discrete_features=[False] * column_to_text_features[column].shape[1], n_neighbors=self.hyperparams["k"], random_state=self._random_seed, ) sum_text_rank = np.sum(text_rankings) text_ranked_features_np[i] = sum_text_rank ranked_features_np, target_entropy = self._normalize( ranked_features_np, feature_df[feature_columns], target_np, discrete, discrete_flags, ) text_ranked_features_np = self._normalize_text( text_ranked_features_np, column_to_text_features, target_entropy) if self.hyperparams["return_as_metadata"]: ranked_features_np = np.append(ranked_features_np, text_ranked_features_np) for i, f in enumerate(feature_indices.union(text_feature_indices)): column_metadata = inputs.metadata.query( (metadata_base.ALL_ELEMENTS, f)) rank_dict = dict(column_metadata) rank_dict["rank"] = ranked_features_np[i] inputs.metadata = inputs.metadata.update( (metadata_base.ALL_ELEMENTS, f), FrozenOrderedDict(rank_dict.items()), ) return base.CallResult(inputs) # merge back into a single list of col idx / rank value tuples data: typing.List[typing.Tuple[int, str, float]] = [] data = self._append_rank_info(inputs, data, ranked_features_np, feature_df[feature_columns]) data = self._append_rank_info( inputs, data, text_ranked_features_np, feature_df[inputs.columns[list(text_feature_indices)]], ) # wrap as a D3M container - metadata should be auto generated results = container.DataFrame(data=data, columns=cols, generate_metadata=True) results = results.sort_values(by=["rank"], ascending=False).reset_index(drop=True) return base.CallResult(results)
def _create_layer_dict(self): layer_dict = OrderedDict() num_flatten_layers = 0 # Make sure that the first layer after input is a net input layer. if (self.protonet.layer[0].type in self._LAYER_TYPES['input'] and self.protonet.layer[1].type in self._LAYER_TYPES['net_input_layers']): last_caffe_layer = self._caffenet.layers[1] last_proto_layer = self.protonet.layer[1] else: raise ParsingError('First layer is not net input layer.') for caffe_layer, proto_layer in zip(self._caffenet.layers, self.protonet.layer): # Check whether the layer is considered a layer or just input. if caffe_layer.type in self._LAYER_TYPES_TO_IGNORE: continue # Check that if we have a Pooling layer, we are really dealing with a max pooling layer. # Allow no other types of pooling right now. if caffe_layer.type == 'Pooling': if not proto_layer.pooling_param.MAX == 0: raise ParsingError( 'Only max pooling is allowed, but was not used in layer {}' .format(str(proto_layer))) # Check whether the layer has a separate activation function, as to whether net input can # be computed or not. if caffe_layer.type in self._TRANSFORMATION_LAYER_TYPES: # Transformation layers are just by themselves, they can be added right away. layer_dict[proto_layer.name] = self._LAYER_TYPES_TO_CLASSES[ caffe_layer.type](self, caffe_layer, proto_layer) elif caffe_layer.type in self._LAYER_TYPES['net_input_layers']: # Check whether there occurs an implicit flatten operation between two layers and # and artificial flatten layer has to be added. # That is the case if we now have a dense layer and the previous layer still had NCHW dimensions. if (caffe_layer.type == 'InnerProduct' and len( self._caffenet.blobs[last_proto_layer.name].data.shape) > 2): # Keep a count of the number of added Flatten layers, to name them properly. num_flatten_layers += 1 layer_dict[ 'flatten_' + str(num_flatten_layers )] = self._LAYER_TYPES_TO_CLASSES['Flatten']( self, last_caffe_layer, last_proto_layer, caffe_layer, proto_layer) # Conv and Dense layers should wait for there activation function. # If however the last layer was a net input layer as well it needs to be added now. # The activation in this case would be the layer itself. if last_caffe_layer.type in self._LAYER_TYPES[ 'net_input_layers']: layer_dict[ last_proto_layer.name] = self._LAYER_TYPES_TO_CLASSES[ last_caffe_layer.type](self, last_caffe_layer, last_proto_layer, last_caffe_layer, last_proto_layer) elif caffe_layer.type in self._LAYER_TYPES['activation_functions']: # If we are dealing with an activation function, last layer before needs to be a net input layer. if last_caffe_layer.type in self._LAYER_TYPES[ 'net_input_layers']: layer_dict[ last_proto_layer.name] = self._LAYER_TYPES_TO_CLASSES[ last_caffe_layer.type]( self, last_caffe_layer, last_proto_layer, caffe_layer, proto_layer, ) else: raise ParsingError( 'Activation function not after conv or dense layer.') # Save the values of the current iteration to compare with the next. last_caffe_layer = caffe_layer last_proto_layer = proto_layer return FrozenOrderedDict(layer_dict)
def test_copy_with_keys(self): frozen = FrozenOrderedDict(a=0, b=1) copied = frozen.copy(b=2, c=3) self.assertIs(type(copied), type(frozen)) self.assertMappingEqual(copied, dict(a=0, b=2, c=3))
def __init__(self, **kwargs): self._data_format = kwargs['data_format'] self.layer_dict = FrozenOrderedDict(input_layer=MockLayer( input_shape=kwargs['input_shape']))
async def read_user_state(path): global prev global manual current_book = manual_filename current_language = None book_files = utility.find_files(path, ('brf', 'pef')) config = config_loader.load() state_sources = ['sd_card_dir'] if config.has_option('files', 'additional_lib_1'): state_sources.append('additional_lib_1') if config.has_option('files', 'additional_lib_2'): state_sources.append('additional_lib_2') for state_source in state_sources: _dir = config.get('files', state_source) main_toml = os.path.join(path, _dir, USER_STATE_FILE) if os.path.exists(main_toml): main_state = toml.load(main_toml) if 'current_book' in main_state: current_book = main_state['current_book'] if not current_book == manual_filename: current_book = os.path.join(path, current_book) if 'current_language' in main_state: current_language = main_state['current_language'] break if not current_language or current_language == OLD_DEFAULT_LOCALE: current_language = DEFAULT_LOCALE.code install(current_language) manual = Manual.create() manual_toml = os.path.join(path, to_state_file(manual_filename)) if os.path.exists(manual_toml): t = toml.load(manual_toml) if 'current_page' in t: manual = manual._replace(page_number=t['current_page'] - 1) if 'bookmarks' in t: manual = manual._replace(bookmarks=tuple( sorted(manual.bookmarks + tuple(bm - 1 for bm in t['bookmarks'])))) books = OrderedDict({manual_filename: manual}) for book_file in book_files: toml_file = to_state_file(book_file) book = BookFile(filename=book_file, width=40, height=9) if os.path.exists(toml_file): t = toml.load(toml_file) if 'current_page' in t: book = book._replace(page_number=t['current_page'] - 1) if 'bookmarks' in t: book = book._replace(bookmarks=tuple( sorted(book.bookmarks + tuple(bm - 1 for bm in t['bookmarks'])))) books[book_file] = book books[cleaning_filename] = CleaningAndTesting.create() if current_book not in books: current_book = manual_filename user_state = frozendict(books=FrozenOrderedDict(books), current_book=current_book, current_language=current_language) prev = user_state return user_state.copy(books=user_state['books'])
def test_copy(self): frozen = FrozenOrderedDict(a=0, b=1) copied = frozen.copy() self.assertIs(type(copied), type(frozen)) self.assertMappingEqual(copied, frozen)
def download_files(primitive_metadata: frozendict.FrozenOrderedDict, output: str, redownload: bool) -> None: for installation_entry in primitive_metadata.get('installation', []): if installation_entry['type'] not in ['FILE', 'TGZ']: continue # We store into files based on digest. In this way we deduplicate same # files used by multiple primitives. output_path = os.path.join(output, installation_entry['file_digest']) if installation_entry['type'] == 'FILE': if os.path.isfile(output_path) and not redownload: print( "File for volume {type}/{key} for primitive {python_path} ({primitive_id}) already exists, skipping: {file_uri}" .format( python_path=primitive_metadata['python_path'], primitive_id=primitive_metadata['id'], type=installation_entry['type'], key=installation_entry['key'], file_uri=installation_entry['file_uri'], ), flush=True) continue elif installation_entry['type'] == 'TGZ': if os.path.isdir(output_path) and not redownload: print( "Directory for volume {type}/{key} for primitive {python_path} ({primitive_id}) already exists, skipping: {file_uri}" .format( python_path=primitive_metadata['python_path'], primitive_id=primitive_metadata['id'], type=installation_entry['type'], key=installation_entry['key'], file_uri=installation_entry['file_uri'], ), flush=True) continue # Cleanup. if os.path.isdir(output_path): shutil.rmtree(output_path) elif os.path.exists(output_path): os.remove(output_path) print( "Downloading file for volume {type}/{key} for primitive {python_path} ({primitive_id}): {file_uri}" .format( python_path=primitive_metadata['python_path'], primitive_id=primitive_metadata['id'], type=installation_entry['type'], key=installation_entry['key'], file_uri=installation_entry['file_uri'], ), flush=True) output_file_obj: typing.Optional[typing.BinaryIO] = None output_tar_process = None try: if installation_entry['type'] == 'FILE': output_file_obj = open(output_path, 'wb') elif installation_entry['type'] == 'TGZ': os.makedirs(output_path, mode=0o755, exist_ok=True) output_tar_process = subprocess.Popen( ['tar', '-xz', '-C', output_path], stdin=subprocess.PIPE) output_file_obj = typing.cast(typing.BinaryIO, output_tar_process.stdin) hash = hashlib.sha256() downloaded = 0 start = time.time() last_progress_report = None while True: try: headers = {} if downloaded: headers['Range'] = 'bytes={downloaded}-'.format( downloaded=downloaded) with requests.get( installation_entry['file_uri'], stream=True, headers=headers, allow_redirects=True, timeout=30, ) as response: response.raise_for_status() # We require the server to support partial requests. In the case that it returns code 200 when we # have send "Range" header (when downloaded is not zero) we raise an exception. if (downloaded and response.status_code != 206) or ( not downloaded and response.status_code != 200): raise requests.HTTPError( "Unexpected status: {status_code}".format( status_code=response.status_code), response=response) download_total = response.headers.get('Content-Length') for data in response.iter_content(chunk_size=1024 * hash.block_size): hash.update(data) downloaded += len(data) output_file_obj.write(data) # type: ignore # Output at most once every 10 seconds. now = time.time() if last_progress_report is None or now > last_progress_report + 10: last_progress_report = now if download_total: print( "Downloaded {downloaded}/{download_total} B" .format(downloaded=downloaded, download_total=download_total), flush=True, ) else: print( "Downloaded {downloaded} B".format( downloaded=downloaded), flush=True, ) break except requests.Timeout: # If timeout, retry/resume. print("Timeout. Retrying.", flush=True) end = time.time() print("Downloaded {downloaded} B in {seconds} second(s).".format( downloaded=downloaded, seconds=end - start, ), flush=True) if output_tar_process is not None: # Close the input to the process to signal that we are done. output_file_obj.close() # type: ignore output_file_obj = None # Wait for 60 seconds to finish writing everything out. if output_tar_process.wait(60) != 0: raise subprocess.CalledProcessError( output_tar_process.returncode, output_tar_process.args) output_tar_process = None if installation_entry['file_digest'] != hash.hexdigest(): raise ValueError( "Digest for downloaded file does not match one from metadata. Metadata digest: {metadata_digest}. Computed digest: {computed_digest}." .format( metadata_digest=installation_entry['file_digest'], computed_digest=hash.hexdigest(), )) except Exception: # Cleanup. if output_tar_process is not None: try: output_tar_process.kill() output_tar_process.wait() output_file_obj = None except Exception: # We ignore errors cleaning up. pass if os.path.isdir(output_path): shutil.rmtree(output_path) elif os.path.exists(output_path): os.remove(output_path) raise finally: if output_file_obj is not None: output_file_obj.close()