def testSPS(): from util.util import interval from module.Module import Constant config = SPSConfig.SPSConfig() moduleFactory = ModuleFactory(config) modelfactory = ModelFactory(moduleFactory) model = modelfactory.get_built() ltl = ['U[1, {0}]'.format(730), 'T', 'failure'] # 一年之内系统失效 checker = Checker(model, ltl, duration=730, fb=False) wrapper = ExperimentWrapper(checker, samples_per_param=100) trainx = interval(1, 10, 0.5) testx = interval(1, 10, 0.1) thickness_params = [Constant('SCREEN_THICKNESS', st) for st in trainx] wrapper.setconstants([thickness_params]) result = wrapper.do_expe() cases = map(lambda tuple: tuple[0], result) labels = map(lambda tuple: tuple[1], result) regressor = BPNeuralNetwork() regressor.setup(1, 5, 1) regressor.train(cases, labels) test_cases = map(lambda c: Constant('SCREEN_THICKNESS', c), testx) test_labels = [regressor.predict(test_case) for test_case in testx] # 对多组参数进行模型验证 # logger.info("mc begin") # wrapper.setconstants([test_cases]) # mcresult = wrapper.modelcheck() # mc_labels = map(lambda tuple: tuple[1], mcresult) plt.plot(testx, [i[0] for i in test_labels], label='predict') # plt.plot(map(lambda const: const.get_value(), test_cases), mc_labels, label='mc') plt.show()
def get_products(): factory = ModelFactory() products = factory.read(Product) return jsonify(status=200, message='OK', body={ 'status': 200, 'message': 'OK', 'payload': products })
def get_users(user_id): factory = ModelFactory() user_info = factory.get(User, id=user_id) return jsonify(status=200, message='OK', body={ 'status': 200, 'message': 'OK', 'payload': user_info })
def t2(model=None): '''测试built模型运行checker的结果与PRISM中运行的一致''' prism_result_x, prism_result_y = get_prism_checking_result() # (1, 5, 1) checker = get_checker(ModelFactory.get_built()) samplesize = checker.get_sample_size() thickness = range(1, 6) probs = [] # logger = get_logger() for t in thickness: ModelFactory.setParam("SCREEN_THICKNESS", t) checker.model.prepare() probs.append(checker.run_checker()) # logger.info("samples={},c={},d={}".format(samplesize, c, d)) # logger.info(probs) for v1, v2 in zip(probs, prism_result_y[-(len(probs)):]): pass
def test(): model = ModelFactory.get_built() model.duration = 730 model.prepare() thrd = StepGenThd(model.gen_next_step(), model.steps_queue) thrd.start() thrd.join() print model.steps_queue.qsize()
def do_expr_parsed(): model = ModelFactory.get_parsed() checker = Checker(model=model, ltl=ltl, duration=TIME_LIMIT_IN_DAYS*2) # wrapper = ExperimentWrapper(checker, samples_per_param=SAMPLES_PER_PARAM) wrapper = ExperimentWrapper(checker) wrapper.setconstants([thickness_cnsts,]) logger.info("Parsed model expr began.") result = wrapper.do_expe() logger.info("Parsed model expr end.") return result
def test(): built = ModelFactory.get_built() day_vals = range(1, 311, 50) logger.info("day_vals = {}".format(day_vals)) sb_module = built.getModuleByName("SB") sb_fail_cmd = sb_module.getCommand("sb_fail_cmd") s3r_module = built.getModuleByName("S3R") s3r_fail_cmd = s3r_module.getCommand("s3r_fail_cmd") for day_val in day_vals: built.setVariable("day", day_val) sb_fail_prob = sb_fail_cmd.prob()/4.0 s3r_fail_prob = s3r_fail_cmd.prob()/4.0 logger.info("day={}".format(day_val) + ", sb_fail_prob=" + format(sb_fail_prob, '.15e')) logger.info("day={}".format(day_val) + ", s3r_fail_prob=" + format(s3r_fail_prob, '.15e'))
def test_parsed(): parsed = ModelFactory.get_parsed() parsed.prepareCommands() pr = cProfile.Profile() pr.enable() result, path = parsed.gen_random_path(duration=duration) pr.disable() print "len of path:{}".format(len(path)) s = StringIO.StringIO() sortby = "cumulative" ps = pstats.Stats(pr, stream=io.FileIO("./parsed", mode='w')).sort_stats(sortby) ps.print_stats() print s.getvalue()
def test_built(): built = ModelFactory.get_built() built.prepare() pr = cProfile.Profile() pr.enable() path = built.get_random_path_V2() pr.disable() # id1 = id(path[0].ap_set) # id2 = id(path[1].ap_set) print "len of path:{}".format(len(path)) s = StringIO.StringIO() sortby = "cumulative" ps = pstats.Stats(pr, stream=io.FileIO("./built", mode='w')).sort_stats(sortby) ps.print_stats() print s.getvalue()
def create_product(): data = request.json factory = ModelFactory() product_id = factory.create(Product, seller=data['seller'], name=data['name']) for color in data['option']['color']: for size in data['option']['size']: factory.create(ProductOption, product_id=product_id, color=color, size=size) for seq, image in enumerate(data['images']): factory.create(ProductImage, product_id=product_id, seq=int(seq), path=image['path'], url=image['url']) factory.create(ProductPrice, product_id=product_id, price=data['price']) return jsonify(status=201, message='Created')
def main(): args = doParsing() print(args) # Read training configuration (config file is in common for dataset creation and training hyperparameters) configParams = ConfigParams(args.configFile) # Select train device trainDevice = selectTrainDevice(args.useGpu) # Load DataProvider dataProvider = DatasetTFReader( datasetDir=args.datasetDir, datasetMetadata=DatasetMetadata().initFromJson(os.path.join(args.datasetDir, "metadata.json")), configParams=configParams) # Load base model graph (fine tuning training) baseTFModel = None try: baseTFModel = TensorflowModel(os.path.join(args.baseModelDir, "graph.pb")) except Exception as e: print("Warning: no base model provided or impossible to read," " this works only for custom model created from scratch") baseTFModel = TensorflowModel() # Append classifier for fine tuning training trainingModel = ModelFactory.create(config=configParams, tfmodel=baseTFModel, dataProvider=dataProvider, trainDevice=trainDevice) # Run training trainProcess = TrainProcess(config=configParams, trainingModel=trainingModel, dataProvider=dataProvider, outputDir=args.checkpointOutputDir, tensorboardDir=args.tensorboardDir) trainProcess.runTrain() # Freeze graph (graphdef plus parameters), # this includes in the graph only the layers needed to provide the output_node_names freeze_graph(input_graph=args.checkpointOutputDir + "/model_graph.pb", input_saver="", input_binary=True, input_checkpoint=args.checkpointOutputDir + "/model", output_node_names=configParams.outputName, restore_op_name="save/restore_all", filename_tensor_name="save/Const:0", output_graph=args.modelOutputDir + "/graph.pb", clear_devices=True, initializer_nodes="")
def test_parsed(parsed=None): if not parsed: parsed = ModelFactory.get_parsed() parsed.duration = duration parsed.prepare() # thd = StepGenThd(model=parsed) # thd.setDaemon(True) # thd.start() # time.sleep(2) # 模拟用户输入 pr = cProfile.Profile() pr.enable() path = parsed.get_random_path_V2() pr.disable() print "len of path:{}".format(len(path)) s = StringIO.StringIO() sortby = "cumulative" ps = pstats.Stats(pr, stream=io.FileIO("./parsed", mode='w')).sort_stats(sortby) ps.print_stats() print s.getvalue()
def test_insert_document_and_check_conflict(): """ Run the start.py file and insert a record into in_progress that has a conflict with the prod collection so main method should return the conflict id. Also check if the newly inserted document is in prod collection and with the correct manually_changed. Lastly fetch the manual document instead of the document in prod collection. """ id = "295cc564fe771fbb92b3278a6eee2d5cbcae2606-3" correct_conflicts = [{ "conflict_id": id, "title": " Velkommen til Trondheim kommune" }] f = open("model/test/test_data/test_data_in_progress.json") serialized_data = json.load(f) # Just to make sure the new document has changed we add a number number here. random_text = "Inserted_document_website_change: " + str( random.randint(1, 100000)) serialized_data[0]["content"]["texts"][0] = random_text # Check if the document was a conflict. conflict_ids = insert_documents(serialized_data) assert correct_conflicts[0]["conflict_id"] == conflict_ids[0][ "conflict_id"] assert correct_conflicts[0]["title"] == conflict_ids[0]["title"] # Fetch the document from the prod collection. factory = ModelFactory.get_instance() util.set_db(factory) document = next( factory.get_database().get_collection("prod").find({"id": id}), None) assert document["content"]["texts"][0] == random_text # Get the manually changed document. manually_changed_doc = util.check_manually_changed(factory, document) # Check if we actually got the manually changed document. assert manually_changed_doc["content"]["texts"][0] == "El manual changos"
# -*- coding: utf-8 -*- from model.ModelFactory import ModelFactory from util.AnnotationHelper import profileit model = ModelFactory.get_parsed(duration=1000000) model.prepare() # 测试将队列填满需要多少时间 @profileit("stepgentime") def test(): step_gen = model.gen_next_step(passed_time=0.0) for _ in range(73000): step = next(step_gen) model.steps_queue.append(step) if __name__ == "__main__": test()
def insert_documents(data): """ :param data: Is a list of serialized documents that should be inserted. :return: a list of conflict document ids. """ factory = ModelFactory.get_instance() util.set_db(factory) """ How we use MongoDB: We have 3 different collections: One for manual entries called "manual" One for production called "prod" One for the in_progress collection called "in_progress" After we have scraped we add all the scraped data into the collection "in_progress" and then we go through every entry in the "manual" collection and use that entry's ID to query both prod and in_progress collection. We compare the two contents in prod and in_progress to see if something changed from last time this was run and now. If they do not have the same content then we need to alert someone that the manual entry needs to be updated. When this is done in_progress will become our new prod. """ factory.get_database().drop_collection("in_progress") print('Starting insertion of {} documents'.format(len(data))) pbar = ProgressBar() for i, doc in enumerate(pbar(data)): factory.post_document(doc, "in_progress") print('Successfully inserted {} documents'.format(i + 1)) manual_documents = factory.get_collection("manual").find() # These are the IDs of the documents that are changed in manual and have been changed since # last time. conflict_ids = [] for manual_document in manual_documents: if "id" in manual_document: id = manual_document["id"] else: continue factory.get_database().get_collection("in_progress").update( {"id": id}, {"$set": { "manually_changed": True }}) prod_match = factory.get_collection("prod").find({"id": id}) in_progress_match = factory.get_collection("in_progress").find( {"id": id}) prod_match_doc = next(prod_match, None) in_progress_doc = next(in_progress_match, None) if prod_match_doc and in_progress_doc: if prod_match_doc['content'] != in_progress_doc['content']: conflict_ids.append({ "conflict_id": id, "title": in_progress_doc["content"]["title"] }) print("Conflict IDs are", conflict_ids) # Set ID to be unique. factory.get_collection("conflict_ids").create_index([("conflict_id", 1)], unique=True) # Insert all the conflict ids into our collection. for conflict in conflict_ids: try: factory.post_document(conflict, "conflict_ids") except pymongo.errors.DuplicateKeyError: # Then we already know this is a conflict ID and should not be added again to the list. pass # Delete the backup prod and rename prod to prod2 and then rename in_progress to prod. factory.get_database().drop_collection("prod2") try: factory.get_database().get_collection("prod").rename("prod2") except pymongo.errors.OperationFailure: pass factory.get_database().get_collection("in_progress").rename("prod") util.set_index("in_progress", factory) util.set_index("prod", factory) util.set_index("manual", factory) # Set query_text to be unique. factory.get_collection("unknown_queries").create_index([("query_text", 1)], unique=True) return conflict_ids
def getBuiltModel(): return ModelFactory.get_built()
def getParsedModel(): return ModelFactory.get_parsed()
def get_main_page(): factory = ModelFactory() products = factory.read(Product) return render_template('products.html', products=products)
def get_product_detail(product_id): factory = ModelFactory() product = factory.get(Product, id=product_id) return render_template('product_detail.html', product=product)
def get_built_model(): return ModelFactory().get_built()
def setUp(self): self.parsed = ModelFactory.get_parsed() self.ltl = ["U[1, {}]".format(DURATION), "T", "failure"] self.checker = Checker(self.parsed, ltl=self.ltl, duration=DURATION)
import numpy as np import random import tensorflow as tf from data.dataset import Dataset from model.ModelFactory import ModelFactory from evaluator import FoldOutEvaluator np.random.seed(2018) random.seed(2018) tf.random.set_random_seed(2018) if __name__ == "__main__": config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) model_factory = ModelFactory() Model, config = model_factory.get_model() dataset = Dataset(config) evaluator = FoldOutEvaluator(dataset.train_matrix, dataset.test_matrix) model = Model(sess, config, dataset, evaluator) model.train_model()
import model.db_util as db_util import api.flask.flask_util as flask_util from model.ModelFactory import ModelFactory from model.keyword_gen import lemmatize_content_keywords import json from flask import request, Blueprint web_api = Blueprint('Website API', __name__, template_folder='templates') factory = ModelFactory.get_instance() db_util.set_db(factory) @web_api.route("/v1/web/conflict_ids", methods=["GET"]) def get_all_conflict_ids(): """ :return: a list of {"title" "...", "id": "..."} """ conflict_ids_docs = factory.get_collection("conflict_ids").find() conflict_ids = [] for conflict_id_doc in conflict_ids_docs: conflict_ids.append({"id": conflict_id_doc["conflict_id"], "title": conflict_id_doc["title"]}) return json.dumps(conflict_ids) @web_api.route("/v1/web/content/", methods=["GET"]) def get_content(): """ :return: the content of the prod document and manual document (if we have it)
import numpy as np import random import tensorflow as tf from model.ModelFactory import ModelFactory np.random.seed(2018) random.seed(2018) tf.set_random_seed(2018) if __name__ == "__main__": model_factory = ModelFactory() Model = model_factory.get_model() model = Model() model.train_model()