def testSPS():
from util.util import interval
from module.Module import Constant
config = SPSConfig.SPSConfig()
moduleFactory = ModuleFactory(config)
modelfactory = ModelFactory(moduleFactory)
model = modelfactory.get_built()
ltl = ['U[1, {0}]'.format(730), 'T', 'failure'] # 一年之内系统失效
checker = Checker(model, ltl, duration=730, fb=False)
wrapper = ExperimentWrapper(checker, samples_per_param=100)
trainx = interval(1, 10, 0.5)
testx = interval(1, 10, 0.1)
thickness_params = [Constant('SCREEN_THICKNESS', st) for st in trainx]
wrapper.setconstants([thickness_params])
result = wrapper.do_expe()
cases = map(lambda tuple: tuple[0], result)
labels = map(lambda tuple: tuple[1], result)
regressor = BPNeuralNetwork()
regressor.setup(1, 5, 1)
regressor.train(cases, labels)
test_cases = map(lambda c: Constant('SCREEN_THICKNESS', c), testx)
test_labels = [regressor.predict(test_case) for test_case in testx]
# 对多组参数进行模型验证
# logger.info("mc begin")
#
wrapper.setconstants([test_cases])
# mcresult = wrapper.modelcheck()
# mc_labels = map(lambda tuple: tuple[1], mcresult)
plt.plot(testx, [i[0] for i in test_labels], label='predict')
# plt.plot(map(lambda const: const.get_value(), test_cases), mc_labels, label='mc')
plt.show()
def get_products():
factory = ModelFactory()
products = factory.read(Product)
return jsonify(status=200,
message='OK',
body={
'status': 200,
'message': 'OK',
'payload': products
})
def get_users(user_id):
factory = ModelFactory()
user_info = factory.get(User, id=user_id)
return jsonify(status=200,
message='OK',
body={
'status': 200,
'message': 'OK',
'payload': user_info
})
def t2(model=None):
'''测试built模型运行checker的结果与PRISM中运行的一致'''
prism_result_x, prism_result_y = get_prism_checking_result() # (1, 5, 1)
checker = get_checker(ModelFactory.get_built())
samplesize = checker.get_sample_size()
thickness = range(1, 6)
probs = []
# logger = get_logger()
for t in thickness:
ModelFactory.setParam("SCREEN_THICKNESS", t)
checker.model.prepare()
probs.append(checker.run_checker())
# logger.info("samples={},c={},d={}".format(samplesize, c, d))
# logger.info(probs)
for v1, v2 in zip(probs, prism_result_y[-(len(probs)):]):
pass
def test():
model = ModelFactory.get_built()
model.duration = 730
model.prepare()
thrd = StepGenThd(model.gen_next_step(), model.steps_queue)
thrd.start()
thrd.join()
print model.steps_queue.qsize()
def do_expr_parsed():
model = ModelFactory.get_parsed()
checker = Checker(model=model, ltl=ltl, duration=TIME_LIMIT_IN_DAYS*2)
# wrapper = ExperimentWrapper(checker, samples_per_param=SAMPLES_PER_PARAM)
wrapper = ExperimentWrapper(checker)
wrapper.setconstants([thickness_cnsts,])
logger.info("Parsed model expr began.")
result = wrapper.do_expe()
logger.info("Parsed model expr end.")
return result
def test():
built = ModelFactory.get_built()
day_vals = range(1, 311, 50)
logger.info("day_vals = {}".format(day_vals))
sb_module = built.getModuleByName("SB")
sb_fail_cmd = sb_module.getCommand("sb_fail_cmd")
s3r_module = built.getModuleByName("S3R")
s3r_fail_cmd = s3r_module.getCommand("s3r_fail_cmd")
for day_val in day_vals:
built.setVariable("day", day_val)
sb_fail_prob = sb_fail_cmd.prob()/4.0
s3r_fail_prob = s3r_fail_cmd.prob()/4.0
logger.info("day={}".format(day_val) + ", sb_fail_prob=" + format(sb_fail_prob, '.15e'))
logger.info("day={}".format(day_val) + ", s3r_fail_prob=" + format(s3r_fail_prob, '.15e'))
def test_parsed():
parsed = ModelFactory.get_parsed()
parsed.prepareCommands()
pr = cProfile.Profile()
pr.enable()
result, path = parsed.gen_random_path(duration=duration)
pr.disable()
print "len of path:{}".format(len(path))
s = StringIO.StringIO()
sortby = "cumulative"
ps = pstats.Stats(pr, stream=io.FileIO("./parsed",
mode='w')).sort_stats(sortby)
ps.print_stats()
print s.getvalue()
def test_built():
built = ModelFactory.get_built()
built.prepare()
pr = cProfile.Profile()
pr.enable()
path = built.get_random_path_V2()
pr.disable()
# id1 = id(path[0].ap_set)
# id2 = id(path[1].ap_set)
print "len of path:{}".format(len(path))
s = StringIO.StringIO()
sortby = "cumulative"
ps = pstats.Stats(pr, stream=io.FileIO("./built",
mode='w')).sort_stats(sortby)
ps.print_stats()
print s.getvalue()
def create_product():
data = request.json
factory = ModelFactory()
product_id = factory.create(Product,
seller=data['seller'],
name=data['name'])
for color in data['option']['color']:
for size in data['option']['size']:
factory.create(ProductOption,
product_id=product_id,
color=color,
size=size)
for seq, image in enumerate(data['images']):
factory.create(ProductImage,
product_id=product_id,
seq=int(seq),
path=image['path'],
url=image['url'])
factory.create(ProductPrice, product_id=product_id, price=data['price'])
return jsonify(status=201, message='Created')
def main():
args = doParsing()
print(args)
# Read training configuration (config file is in common for dataset creation and training hyperparameters)
configParams = ConfigParams(args.configFile)
# Select train device
trainDevice = selectTrainDevice(args.useGpu)
# Load DataProvider
dataProvider = DatasetTFReader(
datasetDir=args.datasetDir,
datasetMetadata=DatasetMetadata().initFromJson(os.path.join(args.datasetDir, "metadata.json")),
configParams=configParams)
# Load base model graph (fine tuning training)
baseTFModel = None
try:
baseTFModel = TensorflowModel(os.path.join(args.baseModelDir, "graph.pb"))
except Exception as e:
print("Warning: no base model provided or impossible to read,"
" this works only for custom model created from scratch")
baseTFModel = TensorflowModel()
# Append classifier for fine tuning training
trainingModel = ModelFactory.create(config=configParams, tfmodel=baseTFModel,
dataProvider=dataProvider, trainDevice=trainDevice)
# Run training
trainProcess = TrainProcess(config=configParams, trainingModel=trainingModel,
dataProvider=dataProvider, outputDir=args.checkpointOutputDir,
tensorboardDir=args.tensorboardDir)
trainProcess.runTrain()
# Freeze graph (graphdef plus parameters),
# this includes in the graph only the layers needed to provide the output_node_names
freeze_graph(input_graph=args.checkpointOutputDir + "/model_graph.pb", input_saver="", input_binary=True,
input_checkpoint=args.checkpointOutputDir + "/model", output_node_names=configParams.outputName,
restore_op_name="save/restore_all", filename_tensor_name="save/Const:0",
output_graph=args.modelOutputDir + "/graph.pb", clear_devices=True, initializer_nodes="")
def test_parsed(parsed=None):
if not parsed:
parsed = ModelFactory.get_parsed()
parsed.duration = duration
parsed.prepare()
# thd = StepGenThd(model=parsed)
# thd.setDaemon(True)
# thd.start()
# time.sleep(2) # 模拟用户输入
pr = cProfile.Profile()
pr.enable()
path = parsed.get_random_path_V2()
pr.disable()
print "len of path:{}".format(len(path))
s = StringIO.StringIO()
sortby = "cumulative"
ps = pstats.Stats(pr, stream=io.FileIO("./parsed",
mode='w')).sort_stats(sortby)
ps.print_stats()
print s.getvalue()
def test_insert_document_and_check_conflict():
"""
Run the start.py file and insert a record into in_progress that has a conflict with the prod
collection so main method should return the conflict id. Also check if the newly inserted
document is in prod collection and with the correct manually_changed. Lastly fetch the
manual document instead of the document in prod collection.
"""
id = "295cc564fe771fbb92b3278a6eee2d5cbcae2606-3"
correct_conflicts = [{
"conflict_id": id,
"title": " Velkommen til Trondheim kommune"
}]
f = open("model/test/test_data/test_data_in_progress.json")
serialized_data = json.load(f)
# Just to make sure the new document has changed we add a number number here.
random_text = "Inserted_document_website_change: " + str(
random.randint(1, 100000))
serialized_data[0]["content"]["texts"][0] = random_text
# Check if the document was a conflict.
conflict_ids = insert_documents(serialized_data)
assert correct_conflicts[0]["conflict_id"] == conflict_ids[0][
"conflict_id"]
assert correct_conflicts[0]["title"] == conflict_ids[0]["title"]
# Fetch the document from the prod collection.
factory = ModelFactory.get_instance()
util.set_db(factory)
document = next(
factory.get_database().get_collection("prod").find({"id": id}), None)
assert document["content"]["texts"][0] == random_text
# Get the manually changed document.
manually_changed_doc = util.check_manually_changed(factory, document)
# Check if we actually got the manually changed document.
assert manually_changed_doc["content"]["texts"][0] == "El manual changos"
# -*- coding: utf-8 -*-
from model.ModelFactory import ModelFactory
from util.AnnotationHelper import profileit
model = ModelFactory.get_parsed(duration=1000000)
model.prepare()
# 测试将队列填满需要多少时间
@profileit("stepgentime")
def test():
step_gen = model.gen_next_step(passed_time=0.0)
for _ in range(73000):
step = next(step_gen)
model.steps_queue.append(step)
if __name__ == "__main__":
test()
def insert_documents(data):
"""
:param data: Is a list of serialized documents that should be inserted.
:return: a list of conflict document ids.
"""
factory = ModelFactory.get_instance()
util.set_db(factory)
"""
How we use MongoDB:
We have 3 different collections:
One for manual entries called "manual"
One for production called "prod"
One for the in_progress collection called "in_progress"
After we have scraped we add all the scraped data into the collection "in_progress" and then
we go through every entry in the "manual" collection and use that entry's ID to query both
prod and in_progress collection. We compare the two contents in prod and in_progress to see
if something changed from last time this was run and now. If they do not have the same
content then we need to alert someone that the manual entry needs to be updated.
When this is done in_progress will become our new prod.
"""
factory.get_database().drop_collection("in_progress")
print('Starting insertion of {} documents'.format(len(data)))
pbar = ProgressBar()
for i, doc in enumerate(pbar(data)):
factory.post_document(doc, "in_progress")
print('Successfully inserted {} documents'.format(i + 1))
manual_documents = factory.get_collection("manual").find()
# These are the IDs of the documents that are changed in manual and have been changed since
# last time.
conflict_ids = []
for manual_document in manual_documents:
if "id" in manual_document:
id = manual_document["id"]
else:
continue
factory.get_database().get_collection("in_progress").update(
{"id": id}, {"$set": {
"manually_changed": True
}})
prod_match = factory.get_collection("prod").find({"id": id})
in_progress_match = factory.get_collection("in_progress").find(
{"id": id})
prod_match_doc = next(prod_match, None)
in_progress_doc = next(in_progress_match, None)
if prod_match_doc and in_progress_doc:
if prod_match_doc['content'] != in_progress_doc['content']:
conflict_ids.append({
"conflict_id":
id,
"title":
in_progress_doc["content"]["title"]
})
print("Conflict IDs are", conflict_ids)
# Set ID to be unique.
factory.get_collection("conflict_ids").create_index([("conflict_id", 1)],
unique=True)
# Insert all the conflict ids into our collection.
for conflict in conflict_ids:
try:
factory.post_document(conflict, "conflict_ids")
except pymongo.errors.DuplicateKeyError:
# Then we already know this is a conflict ID and should not be added again to the list.
pass
# Delete the backup prod and rename prod to prod2 and then rename in_progress to prod.
factory.get_database().drop_collection("prod2")
try:
factory.get_database().get_collection("prod").rename("prod2")
except pymongo.errors.OperationFailure:
pass
factory.get_database().get_collection("in_progress").rename("prod")
util.set_index("in_progress", factory)
util.set_index("prod", factory)
util.set_index("manual", factory)
# Set query_text to be unique.
factory.get_collection("unknown_queries").create_index([("query_text", 1)],
unique=True)
return conflict_ids
def getBuiltModel():
return ModelFactory.get_built()
def getParsedModel():
return ModelFactory.get_parsed()
def get_main_page():
factory = ModelFactory()
products = factory.read(Product)
return render_template('products.html', products=products)
def get_product_detail(product_id):
factory = ModelFactory()
product = factory.get(Product, id=product_id)
return render_template('product_detail.html', product=product)
def get_built_model():
return ModelFactory().get_built()
def setUp(self):
self.parsed = ModelFactory.get_parsed()
self.ltl = ["U[1, {}]".format(DURATION), "T", "failure"]
self.checker = Checker(self.parsed, ltl=self.ltl, duration=DURATION)
import numpy as np
import random
import tensorflow as tf
from data.dataset import Dataset
from model.ModelFactory import ModelFactory
from evaluator import FoldOutEvaluator
np.random.seed(2018)
random.seed(2018)
tf.random.set_random_seed(2018)
if __name__ == "__main__":
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
model_factory = ModelFactory()
Model, config = model_factory.get_model()
dataset = Dataset(config)
evaluator = FoldOutEvaluator(dataset.train_matrix, dataset.test_matrix)
model = Model(sess, config, dataset, evaluator)
model.train_model()
import model.db_util as db_util
import api.flask.flask_util as flask_util
from model.ModelFactory import ModelFactory
from model.keyword_gen import lemmatize_content_keywords
import json
from flask import request, Blueprint
web_api = Blueprint('Website API', __name__, template_folder='templates')
factory = ModelFactory.get_instance()
db_util.set_db(factory)
@web_api.route("/v1/web/conflict_ids", methods=["GET"])
def get_all_conflict_ids():
"""
:return: a list of {"title" "...", "id": "..."}
"""
conflict_ids_docs = factory.get_collection("conflict_ids").find()
conflict_ids = []
for conflict_id_doc in conflict_ids_docs:
conflict_ids.append({"id": conflict_id_doc["conflict_id"],
"title": conflict_id_doc["title"]})
return json.dumps(conflict_ids)
@web_api.route("/v1/web/content/", methods=["GET"])
def get_content():
"""
:return: the content of the prod document and manual document (if we have it)
import numpy as np
import random
import tensorflow as tf
from model.ModelFactory import ModelFactory
np.random.seed(2018)
random.seed(2018)
tf.set_random_seed(2018)
if __name__ == "__main__":
model_factory = ModelFactory()
Model = model_factory.get_model()
model = Model()
model.train_model()
