def upload_file(): """ Lấy dữ liệu ảnh từ phía browser về server Thực hiện nhận dạng gương mặt :return: Trang web nhận dạng gương mặt """ if request.method == 'POST': utils.reload() # Kiểm tra xem có đối tượng files trong trang 'recognition.html' không if 'file' not in request.files: flash('No file part') return redirect(request.url) image = request.files['file'] # Trường hợp người dùng chưa tải ảnh lên # Website không thay đổi if image.filename == '': flash('No selected file') return redirect(request.url) # Trường hợp người dùng đã tải ảnh lên # Lấy dữ liệu ảnh về server và lưu trũ để xử lý if image and allowed_file(image.filename): filename = secure_filename(image.filename) image.save(os.path.join(app.config['UPLOAD_FOLDER'], filename)) # Thực hiện nhận dạng gương mặt có trong ảnh utils.face_recog('static/' + filename) #utils.recog_LBPH('static/' + filename) return render_template("recognition.html", image_upload=filename) return render_template("recognition.html")
def configure_haproxy(service_ports): ''' Configure HAProxy based on the current peers in the service cluster using the provided port map: "swift": [ 8080, 8070 ] HAproxy will also be reloaded/started if required service_ports: dict: dict of lists of [ frontend, backend ] ''' cluster_hosts = {} cluster_hosts[os.getenv('JUJU_UNIT_NAME').replace('/', '-')] = \ unit_get('private-address') for r_id in relation_ids('cluster'): for unit in relation_list(r_id): cluster_hosts[unit.replace('/', '-')] = \ relation_get(attribute='private-address', rid=r_id, unit=unit) context = {'units': cluster_hosts, 'service_ports': service_ports} with open(HAPROXY_CONF, 'w') as f: f.write(render_template(os.path.basename(HAPROXY_CONF), context)) with open(HAPROXY_DEFAULT, 'w') as f: f.write('ENABLED=1') reload('haproxy')
def _build_data(self, data_dir, data_dir_new, cfg, db): data_filename = data_dir + '/' + cfg.data_file with open(data_filename, 'r') as f: origin_data = json.load(f) for part in ['train','valid','test']: self.data[part] = [] self.goal[part] = {} valList = [] with open(data_dir + '/' + cfg.val_file) as f: for line in f: valList.append(line.split('.')[0]) testList = [] with open(data_dir + '/' + cfg.test_file) as f: for line in f: testList.append(line.split('.')[0]) for k_sess in origin_data: sess = origin_data[k_sess] if k_sess in valList: part = 'valid' elif k_sess in testList: part = 'test' else: part = 'train' turn_data, session_data = init_session(k_sess, cfg) belief_state = turn_data['belief_state'] goal_state = turn_data['goal_state'] init_goal(session_data, goal_state, sess['goal'], cfg) self.goal[part][k_sess] = deepcopy(session_data) current_domain = '' book_domain = '' turn_data['trg_user_action'] = {} turn_data['trg_sys_action'] = {} for i, turn in enumerate(sess['log']): turn_data['others']['turn'] = i turn_data['others']['terminal'] = i + 2 >= len(sess['log']) da_origin = turn['dialog_act'] expand_da(da_origin) turn_data['belief_state'] = deepcopy(belief_state) # from previous turn turn_data['goal_state'] = deepcopy(goal_state) if i % 2 == 0: # user turn_data['sys_action'] = deepcopy(turn_data['trg_sys_action']) del(turn_data['trg_sys_action']) turn_data['trg_user_action'] = dict() for domint in da_origin: domain_intent = da_origin[domint] _domint = domint.lower() _domain, _intent = _domint.split('-') if _domain in cfg.belief_domains: current_domain = _domain for slot, p, value in domain_intent: _slot = slot.lower() _value = value.strip() _da = '-'.join((_domint, _slot)) if _da in cfg.da_usr: turn_data['trg_user_action'][_da] = _value if _intent == 'inform': inform_da = _domain+'-'+_slot if inform_da in cfg.inform_da: belief_state[_domain][_slot] = _value if inform_da in cfg.inform_da_usr and _slot in session_data[_domain] \ and session_data[_domain][_slot] != '?': discard(goal_state[_domain], _slot) elif _intent == 'request': request_da = _domain+'-'+_slot if request_da in cfg.request_da: belief_state[_domain][_slot] = '?' else: # sys book_status = turn['metadata'] for domain in cfg.belief_domains: if book_status[domain]['book']['booked']: entity = book_status[domain]['book']['booked'][0] if 'booked' in belief_state[domain]: continue book_domain = domain if domain in ['taxi', 'hospital', 'police']: belief_state[domain]['booked'] = f'{domain}-booked' elif domain == 'train': found = db.query(domain, [('trainID', entity['trainID'])]) belief_state[domain]['booked'] = found[0]['ref'] else: found = db.query(domain, [('name', entity['name'])]) belief_state[domain]['booked'] = found[0]['ref'] turn_data['user_action'] = deepcopy(turn_data['trg_user_action']) del(turn_data['trg_user_action']) turn_data['others']['change'] = False turn_data['trg_sys_action'] = dict() for domint in da_origin: domain_intent = da_origin[domint] _domint = domint.lower() _domain, _intent = _domint.split('-') for slot, p, value in domain_intent: _slot = slot.lower() _value = value.strip() _da = '-'.join((_domint, _slot, p)) if _da in cfg.da and current_domain: if _slot == 'ref': turn_data['trg_sys_action'][_da] = belief_state[book_domain]['booked'] else: turn_data['trg_sys_action'][_da] = _value if _intent in ['inform', 'recommend', 'offerbook', 'offerbooked', 'book']: inform_da = current_domain+'-'+_slot if inform_da in cfg.request_da: discard(belief_state[current_domain], _slot, '?') if inform_da in cfg.request_da_usr and _slot in session_data[current_domain] \ and session_data[current_domain][_slot] == '?': goal_state[current_domain][_slot] = _value elif _intent in ['nooffer', 'nobook']: # TODO: better transition for da in turn_data['user_action']: __domain, __intent, __slot = da.split('-') if __intent == 'inform' and __domain == current_domain: discard(belief_state[current_domain], __slot) turn_data['others']['change'] = True reload(goal_state, session_data, current_domain) if i + 1 == len(sess['log']): turn_data['final_belief_state'] = belief_state turn_data['final_goal_state'] = goal_state self.data[part].append(deepcopy(turn_data)) add_domain_mask(self.data) def _set_default(obj): if isinstance(obj, set): return list(obj) raise TypeError os.makedirs(data_dir_new) for part in ['train','valid','test']: with open(data_dir_new + '/' + part + '.json', 'w') as f: self.data[part] = json.dumps(self.data[part], default=_set_default) f.write(self.data[part]) self.data[part] = json.loads(self.data[part]) with open(data_dir_new + '/' + part + '_goal.json', 'w') as f: self.goal[part] = json.dumps(self.goal[part], default=_set_default) f.write(self.goal[part]) self.goal[part] = json.loads(self.goal[part])
def debug(main_dict): import ipdb ipdb.set_trace() # breakpoint 69b4b68e // metric_name = main_dict["metric_name"] loss_name = main_dict["loss_name"] batch_size = main_dict["batch_size"] mu.print_welcome(main_dict) train_set, val_set = mu.load_trainval(main_dict) model, opt, _ = mu.init_model_and_opt(main_dict) print("Model from scratch...") batch = ut.get_batch(train_set, indices=[15]) probs = model.predict(batch, "probs") blobs = model.predict(batch, "blobs") tr.fitBatch(model, batch, loss_name=loss_name, opt=opt, epochs=100) val.valBatch(model, batch) import ipdb ipdb.set_trace() # breakpoint 5cd16f8f // ul.visSp_prob(model, batch) vis.images(batch["images"], aa, denorm=1) vis.visBlobs(model, batch) ul.vis_nei(model, batch, topk=1000, thresh=0.8, bg=True) ul.vis_nei(model, batch, topk=1000, bg=False) tr.fitQuick(model, train_set, batch_size=batch_size, loss_name=loss_name, metric_name=metric_name) val.validate(model, val_set, metric_name=main_dict["metric_name"], batch_size=main_dict["val_batchsize"]) tr.fitQuick(model, train_set, batch_size=batch_size, loss_name=loss_name, metric_name=metric_name) tr.fitBatch(model, batch, loss_name=loss_name, opt=opt, epochs=100) val.valBatch(model, batch_train, metric_name=metric_name) tr.fitBatch(model, batch, loss_function=losses.expand_loss, opt=opt, epochs=100) vis.visBlobs(model, batch) vis.visWater(model, batch) val.validate(model, val_set, metric_name="MUCov") import ipdb ipdb.set_trace() # breakpoint ddad840d // model, opt, _ = mu.init_model_and_opt(main_dict) tr.fitBatch(model, batch, loss_name="water_loss_B", opt=opt, epochs=100) tr.fitQuick(model, train_set, loss_name=loss_name, metric_name=metric_name) # vis.images(batch["images"], batch["labels"], denorm=1) # mu.init.LOSS_DICT["water_loss"](model, batch) import ipdb ipdb.set_trace() # breakpoint f304b83a // vis.images(batch["images"], model.predict(batch, "labels"), denorm=1) val.valBatch(model, batch, metric_name=main_dict["metric_name"]) vis.visBlobs(model, batch) import ipdb ipdb.set_trace() # breakpoint 074c3921 // tr.fitBatch(model, batch, loss_name=main_dict["loss_name"], opt=opt, epochs=100) for e in range(10): if e == 0: scoreList = [] scoreList += [ tr.fitIndices(model, train_set, loss_name=main_dict["loss_name"], batch_size=batch_size, metric_name=metric_name, opt=opt, epoch=e, num_workers=1, ind=np.random.randint(0, len(train_set), 32)) ] tr.fitData(model, train_set, opt=opt, epochs=10) ut.reload(sp) water = sp.watersplit(model, batch).astype(int) vis.images(batch["images"], water, denorm=1) vis.visBlobs(model, batch) vis.images(batch["images"], ul.split_crf(model, batch), denorm=1) losses.dense_crf(model, batch, alpha=61, beta=31, gamma=1) vis.visBlobs(model, batch) model.blob_mode = "superpixels" #---------------------- # Vis Blobs vis.visBlobs(model, batch) vis.images(batch["images"], model.predict(batch, "labels"), denorm=1) # Vis Blobs #vis.visBlobs(model, batch) vis.images(batch["images"], sp.watersplit_test(model, batch).astype(int), denorm=1) #=sp.watersplit(model, batch).astype(int); # Vis CRF vis.images(batch["images"], ul.dense_crf(model, batch, alpha=5, gamma=5, beta=5, smooth=False), denorm=1) vis.images(batch["images"], ul.dense_crf(model, batch), denorm=1) # Eval val.valBatch(model, batch, metric_name=main_dict["metric_name"]) import ipdb ipdb.set_trace() # breakpoint e9cd4eb0 // model = mu.load_best_model(main_dict) val.valBatch(model, batch, metric_name=main_dict["metric_name"]) tr.fitBatch(model, batch, loss_name=main_dict["loss_name"], opt=opt) vis.visBlobs(model, batch) import ipdb ipdb.set_trace() # breakpoint 2167961a // batch = ut.get_batch(train_set, indices=[5]) tr.fitBatch(model, batch, loss_name=main_dict["loss_name"], opt=opt) vis.images(batch["images"], model.predict(batch, "probs"), denorm=1) vis.visBlobs(model, batch) val.validate(model, val_set, metric_name=main_dict["metric_name"]) val.validate(model, val_set, metric_name="SBD")
def update_belief_sys(self, old_s, a): """ update belief/goal state with sys action """ s = deepcopy(old_s) a_index = torch.nonzero(a) # get multiple da indices self.time_step += 1 s['others']['turn'] = self.time_step # update sys/user dialog act s['sys_action'] = dict() # update belief part das = [self.cfg.idx2da[idx.item()] for idx in a_index] das = [da.split('-') for da in das] sorted(das, key=lambda x: x[0]) # sort by domain entities = [] if self.topic == '' else self.get_entities(s, self.topic) return_flag = False for domain, intent, slot, p in das: if domain in self.cfg.belief_domains and domain != self.topic: self.topic = domain entities = self.get_entities(s, domain) da = '-'.join((domain, intent, slot, p)) if intent == 'request': s['sys_action'][da] = '?' elif intent in ['nooffer', 'nobook'] and self.topic != '': return_flag = True if slot in s['belief_state'][self.topic] and s['belief_state'][ self.topic][slot] != '?': s['sys_action'][da] = s['belief_state'][self.topic][slot] else: s['sys_action'][da] = 'none' elif slot == 'choice': s['sys_action'][da] = str(len(entities)) elif slot == 'none': s['sys_action'][da] = 'none' else: num = int(p) - 1 if self.topic and len( entities) > num and slot in self.cfg.mapping[ self.topic]: typ = self.cfg.mapping[self.topic][slot] if typ in entities[num]: s['sys_action'][da] = entities[num][typ] else: s['sys_action'][da] = 'none' else: s['sys_action'][da] = 'none' if not self.topic: continue if intent in [ 'inform', 'recommend', 'offerbook', 'offerbooked', 'book' ]: discard(s['belief_state'][self.topic], slot, '?') if slot in s['user_goal'][self.topic] and s['user_goal'][ self.topic][slot] == '?': s['goal_state'][self.topic][slot] = s['sys_action'][da] # booked if intent == 'inform' and slot == 'car': # taxi if 'booked' not in s['belief_state']['taxi']: s['belief_state']['taxi']['booked'] = 'taxi-booked' elif intent in ['offerbooked', 'book' ] and slot == 'ref': # train if self.topic in ['taxi', 'hospital', 'police']: s['belief_state'][ self.topic]['booked'] = f'{self.topic}-booked' s['sys_action'][da] = f'{self.topic}-booked' elif entities: book_domain = entities[0]['ref'].split('-')[0] if 'booked' not in s['belief_state'][ book_domain] and entities: s['belief_state'][book_domain][ 'booked'] = entities[0]['ref'] s['sys_action'][da] = entities[0]['ref'] if return_flag: for da in s['user_action']: d_usr, i_usr, s_usr = da.split('-') if i_usr == 'inform' and d_usr == self.topic: discard(s['belief_state'][d_usr], s_usr) reload(s['goal_state'], s['user_goal'], self.topic) if not self.lock_evalutor: self.evaluator.add_sys_da(s['sys_action']) return s
from django.db import connection from utils import reload if "keyvalue_store" in connection.introspection.table_names(): reload()
""" Author: Pratyush Kumar, [email protected] Script to train neural network models using tensorflow's implementation of keras """ import utils utils.reload() utils.set_path() # Load some tensorflow utilities import tensorflow as tf from tensorflow.keras.models import Model from tensorflow.keras.layers import Dense, Activation, Dropout, Input, LSTM, Reshape, Lambda, RepeatVector from tensorflow.keras.optimizers import Adam # Load some more python modules import numpy as np import matplotlib.pyplot as plt import math """ One-hot encoding of ToD and DoW """ #sim_data = utils.load("output_data.pkl") qother = utils.load("input_data.pkl") #start_time = input_data[-2] #end_time = input_data[-1]
def _build_data(self, data_dir, data_dir_new, cfg, db): """ 按照train,dev,test构建session数据 belief_state, goal_state,others,sys_action,trg_sys_action,usr_action,trg_usr_action 以及附加的状态包括 final_belief_state, final_goal_state, next_avaliable_domain, invisible_domains """ data_filename = data_dir + '/' + cfg.data_file with open(data_filename, 'r') as f: origin_data = json.load(f) for part in ['train', 'valid', 'test']: self.data[part] = [] self.goal[part] = {} valList = [] with open(data_dir + '/' + cfg.val_file) as f: for line in f: valList.append(line.split('.')[0]) testList = [] with open(data_dir + '/' + cfg.test_file) as f: for line in f: testList.append(line.split('.')[0]) num_sess = 0 for k_sess in origin_data: sess = origin_data[k_sess] if k_sess in valList: part = 'valid' elif k_sess in testList: part = 'test' else: part = 'train' turn_data, session_data = init_session(k_sess, cfg) # belief_state belief_state = turn_data['belief_state'] # goal_state goal_state = turn_data['goal_state'] init_goal(session_data, goal_state, sess['goal'], cfg) # 直接跳过多domain场景 if "SNG" not in k_sess: continue # 判断如果数据中没有指定domain场景,则不使用该数据w contain_domain = False for domain in session_data: content = session_data[domain] if domain in cfg.belief_domains and len(content) > 0: contain_domain = True break if contain_domain is False: continue num_sess += 1 # goal # 完整的用户目标, goal 和 goal_state 的差异在于, goal为任务最终的目标, goal_state 表示用户目标在某一时刻的状态 self.goal[part][k_sess] = deepcopy(session_data) current_domain = '' book_domain = '' turn_data['trg_user_action'] = {} turn_data['trg_sys_action'] = {} for i, turn in enumerate(sess['log']): # 注意区分 turn_data为session数据,turn为原始数据 turn_data['others']['turn'] = i turn_data['others']['terminal'] = i + 2 >= len(sess['log']) # 记录本轮的动作 da_origin = turn['dialog_act'] expand_da(da_origin) turn_data['belief_state'] = deepcopy( belief_state) # from previous turn turn_data['goal_state'] = deepcopy(goal_state) if i % 2 == 0: # user # sys_action为上一轮系统动作,trg_sys_action为本轮系统动作, # 因此需要将trg_sys_action赋值给sys_action,之后删除trg_sys_action turn_data['sys_action'] = deepcopy( turn_data['trg_sys_action']) del (turn_data['trg_sys_action']) turn_data['trg_user_action'] = dict() for domint in da_origin: domain_intent = da_origin[domint] _domint = domint.lower() _domain, _intent = _domint.split('-') # current_domain 防止出现booking domain的情况 if _domain in cfg.belief_domains: current_domain = _domain for slot, p, value in domain_intent: _slot = slot.lower() _value = value.strip() _da = '-'.join((_domint, _slot)) # 符合要求的用户动作,如果需要限制用户动作的范围,可以通过修改cfg.da_user即可 if _da in cfg.da_usr: turn_data['trg_user_action'][_da] = _value if _intent == 'inform': inform_da = _domain + '-' + _slot # 根据动作,调整belief_state和goal_state if inform_da in cfg.inform_da: belief_state[_domain][_slot] = _value if inform_da in cfg.inform_da_usr and _slot in session_data[_domain] \ and session_data[_domain][_slot] != '?': discard(goal_state[_domain], _slot) elif _intent == 'request': request_da = _domain + '-' + _slot if request_da in cfg.request_da: belief_state[_domain][_slot] = '?' else: # sys # metadata 记录book的状态 book_status = turn['metadata'] for domain in cfg.belief_domains: if book_status[domain]['book']['booked']: entity = book_status[domain]['book']['booked'][0] # 表示已经完成booked,可以不需要再更新订购状体 if 'booked' in belief_state[domain]: continue book_domain = domain if domain in ['taxi', 'hospital', 'police']: belief_state[domain][ 'booked'] = f'{domain}-booked' elif domain == 'train': found = db.query( domain, [('trainID', entity['trainID'])]) belief_state[domain]['booked'] = found[0][ 'ref'] else: found = db.query(domain, [('name', entity['name'])]) belief_state[domain]['booked'] = found[0][ 'ref'] # 对用户动作做相同的处理 turn_data['user_action'] = deepcopy( turn_data['trg_user_action']) del (turn_data['trg_user_action']) # 保证只会影响到下一个用户动作 turn_data['others']['change'] = False turn_data['trg_sys_action'] = dict() for domint in da_origin: domain_intent = da_origin[domint] _domint = domint.lower() _domain, _intent = _domint.split('-') for slot, p, value in domain_intent: _slot = slot.lower() _value = value.strip() _da = '-'.join((_domint, _slot, p)) if _da in cfg.da and current_domain: if _slot == 'ref': turn_data['trg_sys_action'][ _da] = belief_state[book_domain][ 'booked'] else: turn_data['trg_sys_action'][_da] = _value if _intent in [ 'inform', 'recommend', 'offerbook', 'offerbooked', 'book' ]: inform_da = current_domain + '-' + _slot if inform_da in cfg.request_da: discard(belief_state[current_domain], _slot, '?') if inform_da in cfg.request_da_usr and _slot in session_data[current_domain] \ and session_data[current_domain][_slot] == '?': goal_state[current_domain][ _slot] = _value elif _intent in ['nooffer', 'nobook']: # TODO: better transition for da in turn_data['user_action']: __domain, __intent, __slot = da.split( '-') # 如果系统无法提供服务,则删除上一次用户针对本domain提供的inform if __intent == 'inform' and __domain == current_domain: discard( belief_state[current_domain], __slot) # 改变change标志位,表示用户目标发生转变 turn_data['others']['change'] = True reload(goal_state, session_data, current_domain) # 最后一轮,登入final状态 if i + 1 == len(sess['log']): turn_data['final_belief_state'] = belief_state turn_data['final_goal_state'] = goal_state self.data[part].append(deepcopy(turn_data)) print("session number: ", num_sess) add_domain_mask(self.data) def _set_default(obj): if isinstance(obj, set): return list(obj) raise TypeError # 实验一 # 按照单个domain划分 os.makedirs(data_dir_new) for part in ['train', 'valid', 'test']: with open(data_dir_new + '/' + part + '.json', 'w') as f: self.data[part] = json.dumps(self.data[part], default=_set_default) f.write(self.data[part]) self.data[part] = json.loads(self.data[part]) with open(data_dir_new + '/' + part + '_goal.json', 'w') as f: self.goal[part] = json.dumps(self.goal[part], default=_set_default) f.write(self.goal[part]) self.goal[part] = json.loads(self.goal[part])
import sys import pexpect missing_modules = [] try: from colour import Color except: missing_modules.append('colour') from mycroft.messagebus.message import Message try: reload(mycroftbss) except NameError: try: import mycroftbss # https://github.com/skeledrew/mcbss except Exception: missing_modules.append('mycroft brain skill services') import utils state = None class State(): def __init__(self):
def command(): text = request.form.get('text') if not text: return '', 400 args = text.split() # Check if we know about this command if args[0] == 'add': success, m = add_movie(' '.join(args[1:])) if success: m = m.get_details() m['text'] = "Added movie:" return Response(json.dumps(m), mimetype='application/json') elif args[0] == 'choose': if len(args) == 1: rand = Movie.select().order_by(fn.Random()) else: genre = ' '.join(args[1:]) rand = movies_in_genre(genre).order_by(fn.Random()) try: m = rand.get().get_details() m['text'] = random.choice(config.SAYINGS) return Response(json.dumps(m), mimetype='application/json') except Movie.DoesNotExist: return "No movies yet!" elif args[0] == 'watched': name = ' '.join(args[1:]) try: movie = Movie.get(fn.Lower(Movie.name) == name.lower()) except Movie.DoesNotExist: return "Sorry, I couldn't find that movie. You need to be exact." movie.watched = True movie.save() return "Marked _{}_ as watched".format(movie.name) elif args[0] == 'unwatch': # Undocumented commands name = ' '.join(args[1:]) try: movie = Movie.get(fn.Lower(Movie.name) == name.lower()) except Movie.DoesNotExist: return "Sorry, I couldn't find that movie. You need to be exact." movie.watched = False movie.save() return "Marked _{}_ as un-watched".format(movie.name) elif args[0] == 'list': data = {'text': "Must specify `movies` or `genres`."} if len(args) > 1: if args[1] == 'movies': if len(args) > 2: data = format_movies(' '.join(args[2:])) else: data = format_movies() elif args[1] == 'genres': data = format_genres() return Response(json.dumps(data), mimetype='application/json') # Management commands elif args[0] == 'refresh_genres': get_genres() return 'Done' elif args[0] == 'update': update() return '', 204 elif args[0] == 'reload': reload() return '', 204 return Response(json.dumps(help_text()), mimetype='application/json')