def transform_location(self): insert( self._geolocation.select("geolocation_zip_code_prefix", "geolocation_lat", "geolocation_lng", "geolocation_city", "geolocation_state").distinct(), "dim_location")
def _update_pairwise_similarity(self, topic_id, content, date): """ updates similarity data within the corpus """ bow = self.dictionary.doc2bow(content) for tid, data in self.data.items(): day_delta = (int(date) - int(data['date'])) / NUM_SECONDS_PER_DAY time_factor = math.pow(self.time_decay, day_delta) if tid == topic_id: continue bow1 = self.dictionary.doc2bow(data['body']) sim = matutils.cossim(bow, bow1) sim_1 = sim * min(1.0, 1 / time_factor) sim_2 = sim * min(1.0, time_factor) if self.irrelevant_thresh <= sim_1 <= self.duplicate_thresh: del_id = insert(data['sim_list'], topic_id, sim_1, self.max_recoms) if del_id is not None: self.data[topic_id]['appears_in'].append(tid) self.data[tid]['updated'] = True if del_id != '': remove(self.data[del_id]['appears_in'], tid) if self.irrelevant_thresh <= sim_2 <= self.duplicate_thresh: del_id = insert(self.data[topic_id]['sim_list'], tid, sim_2, self.max_recoms) if del_id is not None: self.data[tid]['appears_in'].append(topic_id) if del_id != '': remove(self.data[del_id]['appears_in'], topic_id)
def create_category(message): if message.text: category_2 = message.text categories[category_2] = [category] with open('categories', 'w') as configfile: json.dump(categories, configfile, indent=2) bot.send_message(message.chat.id, 'Создана новая категория!') spending['category_2'] = category_2 utils.insert(spending['date'], spending['price'], spending['category'], spending['category_2'], spending['category_3'], config.database_name)
def reg(): if request.method == "POST": data = {k: v[0] for k, v in dict(request.form).items()} field = ['username', 'password', 'role'] reg_user = utils.user_name(table, field, data['username']) if reg_user: msg = {'code': 1, 'errmsg': 'user is already'} return render_template('reg.html', msg=msg) else: utils.insert(table, field, data) return redirect('/login/') msg = {} return render_template('reg.html', msg=msg)
def reg(): if request.method=='POST': user_dict={k:v[0] for k,v in dict(request.form).items()} user=utils.get_name(User,user_list,user_dict['username']) if user: result={'code':1,'msg':'username is already'} return render_template("reg.html",result=result) else: utils.insert(User,user_dict) result={'code':1,'msg':' register success'} return redirect("/login/") result={} return render_template("reg.html",result=result)
def classify(self, x): aux = [] for i in self.data: dist_i_x = self.distanceFn(i[:-1], x) if len(aux) < self.k: aux = utils.insert(aux, (dist_i_x, i), acessorFn=lambda x: x[0]) else: aux = utils.insert(aux, (dist_i_x, i), acessorFn=lambda x: x[0]) aux.pop() return numpy.mean([x[1][-1] for x in aux])
def reg(): user = {} result = {} if request.method == "POST": user = dict((k, v[0]) for k, v in dict(request.form).items()) result = getone('user', field, user) if result['code'] == 0: return render_template("reg1.html", result='该用户已被注册,请重新输入') elif not user['username']: return render_template("reg1.html", result="用户名为空,无法注册") else: insert('user', user) return redirect('/login/') if request.method == "GET": return render_template("reg1.html")
def add(): if not session: return redirect("/login") if session['role'] == 0: if request.method == 'POST': user_info_dict = dict(request.form) user_data = {k: v[0] for k, v in user_info_dict.items()} if not utils.checkout_user_exist(user_data['name']): res = utils.insert('user', insert_filed, user_data) return json.dumps(res) else: return render_template('add.html', id=session['id'], user=session['username'], user_cn=session['name'], email=session['email'], phone=session['phone'], role=session['role']) else: return render_template('index.html', id=session['id'], user=session['username'], user_cn=session['name'], email=session['email'], phone=session['phone'], role=session['role'])
def find_most_similar(self, topic): """ Given a topic, compute its similarities with all topics in the corpus and return the top n most similar ones from the corpus """ sim_list = [] bow = self.dictionary.doc2bow(topic['body']) for tid, data in self.data.items(): bow1 = self.dictionary.doc2bow(data['body']) sim = matutils.cossim(bow, bow1) if self.irrelevant_thresh <= sim <= self.duplicate_thresh: insert(sim_list, tid, sim, self.max_recoms) return sim_list
def afterreg(): if request.method == 'POST': username=request.form.get('name') user_pass1=request.form.get('pwd') user_pass2=request.form.get('pwd1') user_sex=request.form.get('sex') user_age=request.form.get('age') user_phone=request.form.get('phone') user_email=request.form.get('email') user_role=request.form.get('role') if not utils.checkout_user_exist(username) and user_pass1 == user_pass2: utils.insert('user',insert_filed,username,user_pass1,user_sex,user_age,user_phone,user_email,user_role) return render_template('reg.html',ok='Congratulations on your successful registration') else: return render_template('reg.html',ok='regist failed,agine') else: return render_template('reg.html')
def add(): if not session: return redirect('/login/') if request.method == "POST": field = ['username', 'password', 'role'] user = {k: v[0] for k, v in dict(request.form).items()} data = utils.insert('user', field, user) return json.dumps(data) return render_template('add.html')
def reg(): if request.method == 'POST': user = {k: v[0] for k, v in dict(request.form).items()} result = insert('user1', field, user) if result['code'] == 0: return redirect('/login/') else: return render_template('reg.html', result=resutl) return render_template('reg.html')
def add_user(): if request.method == 'POST': user_dict = {k: v[0] for k, v in dict(request.form).items()} print user_dict field = ['username', 'password', 'role'] result = insert('taokey', field, user_dict) print result if result['code'] == 0: return json.dumps(result) return render_template('adduser.html')
def register(): if request.method == "POST": field = ['username', 'password', 'tel', 'email', 'role'] data = {k: v[0] for k, v in dict(request.form).items()} result = utils.insert('user1', field, data) if result["code"] == 0: return redirect('/login/') else: return render_template('register.html', result=result) return render_template('register.html')
def reg(): if request.method == 'POST': data = {k: v[0] for k, v in dict(request.form).items()} field = ['username', 'password', 'role'] insert_user = utils.insert('user', field, data) if insert_user['code'] == 0: return redirect('/login/') else: return render_template('reg.html', msg=insert_user) return render_template('reg.html', msg=" ")
def reg(): if request.method == 'POST': data = {k: v[0] for k, v in dict(request.form).items()} field = ['username', 'password', 'role'] result = insert('user1', field, data) if result['code'] == 0: return redirect('/login/') else: return render_template('register.html', result=result) return render_template('register.html')
def storeFeatureTotal(db): table = 'feature' setting = """PID INT NOT NULL, ROOM VARCHAR(255) NOT NULL, STAY_TIME FLOAT NOT NULL, ROOM4 BOOLEAN NOT NULL, ROOM5 BOOLEAN NOT NULL, ROOM6 BOOLEAN NOT NULL, ROMES_COUNT FLOAT NOT NULL, NO_REGISTER BOOLEAN NOT NULL """ if not tableExists(db, table): data = pd.read_csv('../frontend/static/feature.csv') data.drop(columns=['Unnamed: 0'],inplace=True) createTable(db, table, setting) for _, row in tqdm(data.iterrows()): sqlInsert = "INSERT INTO {}(PID, ROOM, STAY_TIME, ROOM4, ROOM5, ROOM6, ROMES_COUNT, NO_REGISTER) VALUES ({},'{}',{},{},{},{},{},{})".format( table, row['id'], row['Room'], row['Stay Time'], row['Been Room4'], row['Been Room5'], row['Been Room6'], row['Rooms Count'], row['No Register']) insert(db, sqlInsert)
def __init__(self): log("Initalizing Fact objects") self.customers = Customer().getcustomer self.geolocation = Geolocation().getgeolocation self.items = Items().getitems self.orders = Orders().getorders self.payments = Payments().getpayments self.products = Products().getproducts self.reviews = Reviews().getreviews self.sellers = Sellers().getsellers log("Initializing Data Transformation and Ingestion Process") self.df = self.prepair() self.middle = self.arrange() self.middle = self.transformation() self.fact = self.finall() insert(self.fact, "fact_orders")
def reg(): if request.method == "POST": user = {k: v[0] for k, v in dict(request.form).items()} field = ['username', 'password'] result = insert('user', field, user) if result['code'] == 0: return redirect('/login/') else: return render_template("reg.html", result=result) return render_template("reg.html")
def reg(): if request.method == 'POST': users = {k: v[0] for k, v in dict(request.form).items()} filed = ['username', 'password', 'role'] res = insert('user', filed, users) if res['code'] == 0: return render_template("reg.html", msg=res['msg']) else: return render_template("reg.html", mag=res['msg']) return render_template("reg.html", msg=" ")
def register(): if request.method == 'POST': user = {k:v[0] for k,v in dict(request.form).items()} print user field = ['username','password','role'] result = insert('taokey',field,user) if result['code'] == 0: return redirect('/userlist/') else: return render_template("register.html",result = result) return render_template("register.html")
def reg(): if request.method == "POST": data = {k: v[0] for k, v in dict(request.form).items()} print "模态窗添加用户的用户信息" print data util.WriteLog('[模态窗添加用户的用户信息]').info(data) field = ['username', 'password', 'email', 'phone', 'role'] result = insert('user', field, data) print "添加用户写完数据库返回的的用户信息" print result util.WriteLog('[管理员添加用户完成返回]').info(result) return json.dumps(result)
def afterreg(): if request.method == 'POST': user_info_dict=dict(request.form) user_data={ k:v[0] for k,v in user_info_dict.items() } if not utils.checkout_user_exist(user_data['u_name']) and user_data['password'] == user_data['password1']: res=utils.insert('user',insert_filed,user_data) if res['code'] == 0: return redirect("/login/") else: return render_template('reg.html',ok='regist failed,agine') else: return render_template('reg.html')
def jobadd(): if not session: return redirect("/") if request.method=='POST': job={k:v[0] for k,v in dict(request.form).items()} field=['apply_date','apply_type','apply_desc','status','apply_persion'] job['apply_date']=time.strftime('%Y-%m-%d %H:%M') job['status']=0 print 'job-->',job data=utils.insert(table,field,job) return json.dumps(data) return render_template("job_add.html",res=session)
def add(): if request.method == 'POST': data = dict(request.form) data = {k: v[0] for k, v in data.items()} field = ['username', 'password', 'role'] result = insert('user1', field, data) if result['code'] == 0: return json.dumps(data) else: return render_template("add.html", result=result) else: return render_template("add.html")
def addserver(): if request.method == "POST": data = { k:v[0] for k,v in dict(request.form).items()} print "模态窗添加server的信息" print data util.WriteLog('[模态窗添加server信息]').info(data) field = ['name','ip','idc_id','cabinet','os','core_num','mem','disk'] result = insert('server',field,data) print "添加server写完数据库返回的的用户信息" print result util.WriteLog('[管理员添加server完成返回]').info(result) return json.dumps(result)
def reg(): if request.method == "POST": field = ['id', 'username', 'password', 'role'] data = {k: v[0] for k, v in dict(request.form).items()} result = utils.getone('user', field, data) if result['code'] == 0: return render_template("reg.html", errmsg="用户名已被使用") else: field = ['username', 'password', 'role'] result = utils.insert('user', field, data) return redirect("/login/") return render_template("reg.html")
def add_to_category(message): keyboard_hider = types.ReplyKeyboardRemove() if message.text != 'create' or message.text != '/create' or message.text != '/exit': category_2 = message.text categories[category_2].append(category) with open('categories', 'w') as configfile: json.dump(categories, configfile, indent=2) bot.send_message(message.chat.id, 'Добавлено!', reply_markup=keyboard_hider) spending['category_2'] = category_2 utils.insert(spending['date'], spending['price'], spending['category'], spending['category_2'], spending['category_3'], config.database_name) elif message.text == 'create' or message.text == '/create': bot.register_next_step_handler(message, create_category) else: bot.send_message(message.chat.id, 'Попробуй еще раз!', reply_markup=keyboard_hider) bot.register_next_step_handler(message, select_category)
def storeRoute(db): setting = """PID INT NOT NULL, POSITION INT NOT NULL, TIME INT NOT NULL """ table1 = 'route1' table2 = 'route2' table3 = 'route3' route1Data = pd.read_csv('../frontend/static/route1.csv', header=None) route2Data = pd.read_csv('../frontend/static/route2.csv', header=None) route3Data = pd.read_csv('../frontend/static/route3.csv', header=None) if not tableExists(db, table1): createTable(db, table1, setting) for _, row in tqdm(route1Data.iterrows()): pid = row[0] for idx in range(1, row.shape[0]): sqlInsert = "INSERT INTO {}(PID, POSITION, TIME) VALUES ({},{},{})".format( table1, pid, row[idx], idx) insert(db, sqlInsert) if not tableExists(db, table2): createTable(db, table2, setting) for _, row in tqdm(route2Data.iterrows()): pid = row[0] for idx in range(1, row.shape[0]): sqlInsert = "INSERT INTO {}(PID, POSITION, TIME) VALUES ({},{},{})".format( table2, pid, row[idx], idx) insert(db, sqlInsert) if not tableExists(db, table3): createTable(db, table3, setting) for _, row in tqdm(route3Data.iterrows()): pid = row[0] for idx in range(1, row.shape[0]): sqlInsert = "INSERT INTO {}(PID, POSITION, TIME) VALUES ({},{},{})".format( table3, pid, row[idx], idx) insert(db, sqlInsert)
def instantiate (self, specs = None): if specs is not None: self.specs = specs; self.Kc = insert (1, self.specs, 'Kc'); self.Ti = insert (np.inf, self.specs, 'Ti'); self.Td = insert (0, self.specs, 'Td'); self.b = insert (1.0, self.specs, 'b'); self.c = insert (1.0, self.specs, 'c'); self.N = insert (9.0, self.specs, 'N'); num_p = self.Kc * self.b; den_p = 1; Cp = ctrl.tf (num_p, den_p); if (self.Ti != np.inf): num_i = self.Kc; den_i = [self.Ti, 0]; Ci = ctrl.tf (num_i, den_i); else: Ci = 0; if (self.Td != 0): num_d = [self.Kc * self.Td * self.c, 0]; den_d = [(self.Td / self.N), 0]; Cd = ctrl.tf (num_d, den_d); else: Cd = 0; self.tf = Cp + Ci + Cd;
def storeRaw(db): table1 = 'day1csv' table2 = 'day2csv' table3 = 'day3csv' setting = """PID INT NOT NULL, SID INT NOT NULL, TIME INT NOT NULL """ # PID person id, SID sensor id if not tableExists(db, table1): day1data = pd.read_csv('../frontend/static/day1.csv') createTable(db, table1, setting) for _, row in tqdm(day1data.iterrows()): sqlInsert = "INSERT INTO {}(PID, SID, TIME) VALUES ({},{},{})".format( table1, row['id'], row['sid'], row['time']) insert(db, sqlInsert) if not tableExists(db, table2): day2data = pd.read_csv('../frontend/static/day2.csv') createTable(db, table2, setting) for _, row in tqdm(day2data.iterrows()): sqlInsert = "INSERT INTO {}(PID, SID, TIME) VALUES ({},{},{})".format( table2, row['id'], row['sid'], row['time']) insert(db, sqlInsert) if not tableExists(db, table3): day3data = pd.read_csv('../frontend/static/day3.csv') createTable(db, table3, setting) for _, row in tqdm(day3data.iterrows()): sqlInsert = "INSERT INTO {}(PID, SID, TIME) VALUES ({},{},{})".format( table3, row['id'], row['sid'], row['time']) insert(db, sqlInsert)
def instantiate (self, specs = None): if specs is not None: self.specs = specs; self.Kc = insert (1, self.specs, 'Kc'); self.Ti = insert (np.inf, self.specs, 'Ti'); self.Td = insert (0, self.specs, 'Td'); self.b = insert (1.0, self.specs, 'b'); self.c = insert (1.0, self.specs, 'c'); self.N = insert (np.inf, self.specs, 'N'); num_p = self.Kc * self.b; den_p = 1; Cpff = ctrl.tf (num_p, den_p); Cpc = ctrl.tf (self.Kc, 1); if (self.Ti != np.inf): num_i = self.Kc; den_i = [self.Ti, 0]; Ci = ctrl.tf (num_i, den_i); else: Ci = 0; if (self.Td != 0): num_d = [self.Kc * self.Td * self.c, 0]; den_d = [(self.Td / self.N), 1]; Cdff = ctrl.tf (num_d, 1); Cdc = ctrl.tf ([self.Kc * self.Td], 1); else: Cdff = 0; Cdc = 0; self.Gff = Cpff + Ci + Cdff; self.Gc = Cpc + Ci + Cdc;
def allCrawler(recipe_id, n): #recipe_id = 10 #for n in range(1, 220): a = n * 17 #grab every 17th page, to provide a wide spread of results urln = 'http://allrecipes.com/recipes/ViewAll.aspx?Page=' + str(a) time.sleep(1) #Sleeping is done according to the Web Crawling Standards S = pq(url=urln) S = utils.getPage(urln) templst = [] for u in S('.rectitlediv'): templst.append(S(u)) urlist = [] for u in templst: urlist.append(u.find('a').attr('href')) print 'all list made ' + str(n) for nextUrl in urlist: time.sleep(1) #Sleeping is done according to the Web Crawling Standards p = utils.getPage(nextUrl) #Creation of temp variables RecpName = [] IngrName = [] IngrAmnt = [] Description = [] ImageURL = [] Cookpmin = [] Cookphour = [] Cookcmin = [] Cookchour = [] Cooktmin = [] Cookthour = [] Servings = [] #temp variables filled for this in p('#itemTitle'): RecpName.append(p(this).text()) print RecpName if utils.isDuplicate(RecpName[0]): print 'duplicate' recipe_id += 1 continue for this in p('#lblIngName'): if p(this).text() != '': if p(this).attr('class') == 'ingredient-name': IngrName.append(p(this).text()) for this in p('#lblIngAmount'): IngrAmnt.append(p(this).text()) for this in p('span.plaincharacterwrap.break'): Description.append(p(this).text()) for this in p('#imgPhoto'): ImageURL.append(p(this).attr('src')) for this in p('#prepMinsSpan'): Cookpmin.append(p(this).text()) for this in p('#cookMinsSpan'): Cookcmin.append(p(this).text()) for this in p('#totalMinsSpan'): Cooktmin.append(p(this).text()) for this in p('#prepHoursSpan'): Cookphour.append(p(this).text()) for this in p('#cookHoursSpan'): Cookchour.append(p(this).text()) for this in p('#totalHoursSpan'): Cookthour.append(p(this).text()) for this in p('#lblYield'): Servings.append(p(this).text()) #parse out the decimal times for cooking HpTime = utils.makeDecimalTime(Cookphour) MpTime = utils.makeDecimalTime(Cookpmin) #parse out the decimal times for preping HTime = utils.makeDecimalTime(Cookthour) MTime = utils.makeDecimalTime(Cooktmin) #convert all time into minutes Ctime = HTime*60 + MTime Ptime = HpTime*60 + MpTime #error checking if RecpName: Rname = RecpName[0] else: Rname = 'null' if ImageURL: ImgURL = ImageURL[0] else: ImgURL = 'null' if not Servings: Servings = ['null'] decript = utils.descriptionize(Description) k = 0 RecipeDict = { 'id': recipe_id, 'name':Rname, 'description' : decript, 'servings' : Servings[0], 'url' : nextUrl, 'img_url' : ImgURL, 'cook_time' : Ctime, 'prep_time' : Ptime} k = utils.insert('recipe', **RecipeDict) IngrDict = {} #if len(IngrName) > len(IngrAmnt): #print IngrName #print IngrAmnt if k != -1: for a in range(len(IngrName)): if len(IngrAmnt) < len(IngrName): IngrAmnt.append('') #in the case where "season to preference" is the 'ingredient' #print len(IngrAmnt) IngrDict = { 'recipe_id':recipe_id, 'ingredient_name':IngrName[a], 'quantity':IngrAmnt[a]} l = utils.insert('recipe_ingredient', **IngrDict) recipe_id += 1 return recipe_id
def get(self): left = self.get_argument('left') right = self.get_argument('right') ret = utils.insert(settings.PICDIR, left, right) self.write(json.dumps(ret))
def __init__ (self, specs = None): self.title = insert ("Standard Plot", specs, 'title'); self.xlab = insert ("Time (s)", specs, 'xlab'); self.ylab = insert ("Signal", specs, 'ylab'); self.grid = insert (False, specs, 'grid'); self.type = insert ("linear", specs, 'type');
def netCrawler(recipe_id, n): #recipe_id = 0 #for n in range (0, 100): a= 12*n urln = 'http://www.foodnetwork.com/search/delegate.do?Ntk=site_search&Nr=Record%20Type:Result&N=501&No=' + str(a) time.sleep(1)#--# S = utils.getPage(urln) templst = [] for u in S('.result-item.recipe'): templst.append(S(u)) urlist = [] for u in templst: urlist.append('http://www.foodnetwork.com' + u.find('a').attr('href')) print 'net list made ' + str(n) for nextUrl in urlist: time.sleep(1)#--# p = utils.getPage(nextUrl) RecpName = [] for this in p('.fn_name'): RecpName.append(p(this).text()) print RecpName if utils.isDuplicate(RecpName[0]): recipe_id += 1 print 'duplicate' continue ImgURL = '' b = p('img#recipe-player-th') for this in b: ImgURL = p(this).attr('src') #if ImgURL is '' then there is no image given. Happens often. unparsedCtime = '' unparsedPtime = '' c = p('dd.clrfix') for this in c('meta'): if p(this).attr('itemprop') == 'cookTime': unparsedCtime = p(this).attr('content') elif p(this).attr('itemprop') == 'prepTime': unparsedPtime = p(this).attr('content') difficulty = '' difficulty = p(c[-1]).text() Ctime = utils.FN_minutesParser(unparsedCtime) + 60*(utils.FN_hourParser(unparsedCtime)) Ptime = utils.FN_minutesParser(unparsedPtime) + 60*(utils.FN_hourParser(unparsedPtime)) servings = [] for this in c('span'): servings.append(p(this).text()) IngrName = [] d = p('.kv-ingred-list1') #d is now a list of all the ingred lists for this in d('li'): IngrName.append(p(this).text()) #the ingredients here are smashed together with their ammounts, which I can't parse out. # if we can parse things like "1 large tomato, diced in 1/2-inch pieces" into ammounts and # ingredient lists, then someone should teach me. directions = [] e = p('.fn_instructions') for this in e('p'): elf = p(this).text() if elf != '\n': if elf != '': directions.append(elf) if not servings: servings = [''] decript = utils.descriptionize(directions) k = 0 RecipeDict = {} RecipeDict = { 'id': recipe_id, 'name':RecpName[0], 'description' : decript, 'servings' : servings[0], 'url' : nextUrl, 'img_url' : ImgURL, 'difficulty' : difficulty, 'cook_time' : Ctime, 'prep_time' : Ptime} k = utils.insert('recipe', **RecipeDict) #print RecpName #for this in decript: # print this #print servings[0] #print nextUrl #print ImgURL #print Ctime #print Ptime IngrDict = {} IngrAmnt = [] if k != -1: for a in range(len(IngrName)): if len(IngrAmnt) < len(IngrName): IngrAmnt.append('') #in the case where "season to preference" is the 'ingredient' #print len(IngrAmnt) #print IngrName[a] #print IngrAmnt[a] IngrDict = { 'recipe_id':recipe_id, 'ingredient_name':IngrName[a], 'quantity':IngrAmnt[a]} l = utils.insert('recipe_ingredient', **IngrDict) recipe_id += 1 return recipe_id