def __add_network_layer(self, layer):
"""网络模型，上一层输入可能有多个模型"""
model_rdd_list = self.model_rdd
if not isinstance(model_rdd_list, list):
model_rdd_list = [model_rdd_list]
if isinstance(layer, InputLayer):
output_model = Model(inputs=layer.input, outputs=layer.output)
self.layer_num += 1
else:
if self.model_rdd is None:
raise ValueError(
"neural network model first layer must be InputLayer!")
inputs = []
outputs = []
for model_rdd in model_rdd_list:
model_config = json.loads(model_rdd.first().model_config)
self.layer_num += len(model_config.get('layers', []))
input_model = Model.from_config(model_config)
inputs.extend(input_model.inputs)
outputs.extend(input_model.outputs)
outputs = outputs[0] if len(outputs) == 1 else outputs
output_model = Model(inputs=inputs, outputs=layer(outputs))
self.layer_num += 1
return self.model2df(output_model)
def convert2model(model_config):
if 'name' not in model_config:
raise ValueError("config is not a model config!")
name = model_config['name'] or ''
if name.startswith('model'):
model = Model.from_config(model_config)
elif name.startswith('sequential'):
model = Sequential.from_config(model_config)
else:
raise ValueError("model config incorrect!")
return model
def build_model(self):
if self.task_index is None:
raise ValueError("task_index cannot None!!!")
with tf.device(tf.train.replica_device_setter(
worker_device="/job:worker/task:{}".format(self.task_index), cluster=self.cluster)):
h, w = self.image_size
num_anchors = len(self.anchors)
y_true = [Input(shape=(h // {0: 32, 1: 16, 2: 8}[l], w // {0: 32, 1: 16, 2: 8}[l],
num_anchors // 3, self.num_classes + 5), name='tmp_{}'.format(l)) for l in range(3)]
model = Model.from_config(self.model_config)
model_loss = Lambda(yolo_loss, output_shape=(1,), name='yolo_loss',
arguments={'anchors': self.anchors,
'num_classes': self.num_classes,
'ignore_thresh': 0.5})(model.output + y_true)
self.model = Model([model.input] + y_true, model_loss)
self.model.compile(optimizer=Adam(lr=1e-4),
loss={'yolo_loss': lambda y_true, y_pred: y_pred})
def reset_pingpong(self):
""" Reset the models for pingpong training """
logger.debug("Resetting models")
# Clear models and graph
self.predictors = dict()
K.clear_session()
# Load Models for current training run
for model in self.networks.values():
model.network = Model.from_config(model.config)
model.network.set_weights(model.weights)
inputs = self.get_inputs()
self.build_autoencoders(inputs)
self.compile_predictors(initialize=False)
logger.debug("Reset models")
def build_model(self):
if self.task_index is None:
raise ValueError("task_index cannot None!!!")
with tf.device(
tf.train.replica_device_setter(
worker_device="/job:worker/task:{}".format(
self.task_index),
cluster=self.cluster)):
model_type = gmt(self.model_config)
if model_type == ModelType.SEQUENCE:
model = Sequential.from_config(self.model_config)
elif model_type == ModelType.NETWORK:
model = Model.from_config(self.model_config)
else:
raise ValueError(
"{}, unknown model type!!!".format(model_type))
self.parse_optimizer()
model.compile(**self.compile_config)
self.model = model
def run(self):
params = self.params
name = params.get('name')
model_rdd = inputRDD(name)
if not model_rdd:
raise ValueError("In Summary model_rdd cannot be empty！")
model_config = json.loads(model_rdd.first().model_config)
# model_name = model_config.get('name')
if get_mode_type(model_rdd) == ModelType.SEQUENCE:
model = Sequential.from_config(model_config)
elif get_mode_type(model_rdd) == ModelType.NETWORK:
model = Model.from_config(model_config)
else:
raise ValueError("model type incorrect!!!")
model.summary()
outputRDD('<#zzjzRddName#>_Summary', model_rdd)
def add(self, start_rdd, repeats=0):
if MODEL_CONFIG not in self.model_rdd.first():
raise ValueError('repeat units end node not exists model_config!')
model_config = json.loads(getattr(self.model_rdd.first(),
MODEL_CONFIG))
start_config = json.loads(getattr(start_rdd.first(), MODEL_CONFIG))
marker_layer = start_config['layers'][-1]
for index, layer in enumerate(model_config['layers']):
if marker_layer == layer:
layers = model_config['layers'][index + 1:]
if 'inbound_nodes' in layer:
self.model = Model.from_config(model_config)
self.repeat_networks(layers, repeats)
elif 'name' in layer['config']:
self.model = Sequential.from_config(model_config)
self.repeat_sequence(layers, repeats)
else:
raise ValueError(
"In RepeatBlock node, model type incorrect!")
return self.model2df(self.model)
