def data_for_apartments_table(): sql = """INSERT INTO apartments(id, home_id, floor, number_of_rooms, area, kitchen_area, number_of_bedrooms, number_of_bathrooms, separated_bathrooms) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s) """ collection = connect_mongo() cursor = collection.find() apts = set() apt_ids = set() for doc in cursor: house = doc.get('house', None) if not house: continue if 'object_info' not in doc: continue apt_info = doc['object_info'] address = doc['address'] locality = address.get('locality', None) if not locality: continue floor = apt_info.get('floor', None) number_of_rooms = apt_info.get('rooms', None) area = apt_info.get('area', None) kitchen_area = apt_info.get('kitchen_area', None) number_of_bedrooms = apt_info.get('bedrooms', None) number_of_bathrooms = apt_info.get('bathrooms', None) separated_bathrooms = bool(apt_info.get('separated_bathrooms', None)) hash_house_id = md5_hash((address['name'], locality['name'])) hash_apt_id = md5_hash((address['name'], locality['name'], floor, area, number_of_rooms)) if hash_apt_id not in apt_ids: apts.add((hash_apt_id, hash_house_id, floor, number_of_rooms, area, kitchen_area, number_of_bedrooms, number_of_bathrooms, separated_bathrooms)) apt_ids.add(hash_apt_id) insert_data_list_in_tables(sql, apts)
def sign_document(dbo, username, mid, sigurl, signdate): """ Signs an HTML document. sigurl: An HTML5 data: URL containing an image of the signature """ SIG_PLACEHOLDER = "signature:placeholder" date, medianame, mimetype, content = get_media_file_data(dbo, mid) # Is this an HTML document? if content.find("<p") == -1 and content.find("<td") == -1: raise utils.ASMValidationError("Cannot sign a non-HTML document") # Has this document already been signed? if 0 != db.query_int(dbo, "SELECT COUNT(*) FROM media WHERE ID = %d AND SignatureHash Is Not Null AND SignatureHash <> ''" % mid): raise utils.ASMValidationError("Document is already signed") # Does the document have a signing placeholder image? If so, replace it if content.find(SIG_PLACEHOLDER) != -1: content = content.replace(SIG_PLACEHOLDER, sigurl) else: # Create the signature at the foot of the document sig = "<hr />\n" sig += '<p><img src="' + sigurl + '" /></p>\n' sig += "<p>%s</p>\n" % signdate content += sig # Create a hash of the contents and store it with the media record db.execute(dbo, "UPDATE media SET SignatureHash = '%s' WHERE ID = %d" % (utils.md5_hash(content), mid)) # Update the dbfs contents update_file_content(dbo, username, mid, content)
def _push_new_attachment(self, attachment): attachment_full_path = self.fs.path_for(attachment.filepath) meta = self.req.post_attachment( attachment, attachment_full_path)['article_attachment'] meta['md5_hash'] = utils.md5_hash(attachment_full_path) meta = self.fs.save_json(attachment.meta_filepath, meta) attachment.meta = meta
def sign_document(dbo, username, mid, sigurl, signdate): """ Signs an HTML document. sigurl: An HTML5 data: URL containing an image of the signature """ al.debug("signing document %s for %s" % (mid, username), "media.sign_document", dbo) SIG_PLACEHOLDER = "signature:placeholder" date, medianame, mimetype, content = get_media_file_data(dbo, mid) # Is this an HTML document? if content.find("<p") == -1 and content.find("<td") == -1: al.error("document %s is not HTML" % mid, "media.sign_document", dbo) raise utils.ASMValidationError("Cannot sign a non-HTML document") # Has this document already been signed? if 0 != dbo.query_int("SELECT COUNT(*) FROM media WHERE ID = ? AND SignatureHash Is Not Null AND SignatureHash <> ''", [mid]): al.error("document %s has already been signed" % mid, "media.sign_document", dbo) raise utils.ASMValidationError("Document is already signed") # Does the document have a signing placeholder image? If so, replace it if content.find(SIG_PLACEHOLDER) != -1: al.debug("document %s: found signature placeholder" % mid, "media.sign_document", dbo) content = content.replace(SIG_PLACEHOLDER, sigurl) else: # Create the signature at the foot of the document al.debug("document %s: no placeholder, appending" % mid, "media.sign_document", dbo) sig = "<hr />\n" sig += '<p><img src="' + sigurl + '" /></p>\n' sig += "<p>%s</p>\n" % signdate content += sig # Create a hash of the contents and store it with the media record dbo.update("media", mid, { "SignatureHash": utils.md5_hash(content) }) # Update the dbfs contents update_file_content(dbo, username, mid, content)
def data_for_companies_table(): sql = """INSERT INTO companies(id, company_name, phone) VALUES(%s,%s,%s)""" collection = connect_mongo() cursor = collection.find() companies = set() company_ids = set() for doc in cursor: seller = doc['seller'] house = doc.get('house', None) if not house: continue address = doc['address'] locality = address.get('locality', None) if not locality: continue company = seller.get('company', None) company_name = company.get('name', None) if not company_name: continue phone = company.get('phone', None) hash_company_id = md5_hash((company_name,)) if hash_company_id not in company_ids: companies.add((hash_company_id, company_name, phone)) company_ids.add(hash_company_id) insert_data_list_in_tables(sql, companies)
def data_for_agents_table(): sql = """INSERT INTO agents(id, company_id, agent_name, phone, is_agent) VALUES(%s,%s,%s,%s,%s)""" collection = connect_mongo() cursor = collection.find() agents = set() agent_ids = set() counter = 0 for doc in cursor: counter += 1 seller = doc['seller'] house = doc.get('house', None) if not house: continue address = doc['address'] locality = address.get('locality', None) if not locality: continue company = seller.get('company', None) company_name = company.get('name', None) hash_company_id = md5_hash((company_name,)) if not company_name: hash_company_id = None agent = seller.get('agent', None) phone = agent.get('phone', None) is_agent = agent.get('is_agent', None) agent_name = agent.get('full_name', None) if not agent_name: print(doc['_id']) continue hash_city_id = md5_hash((locality['name'],)) hash_agent_id = md5_hash((hash_city_id, agent_name, phone)) if hash_agent_id not in agent_ids: agents.add((hash_agent_id, hash_company_id, agent_name, phone, is_agent)) agent_ids.add(hash_agent_id) if len(agents) > 100: insert_data_list_in_tables(sql, agents) agents = set() print('agents inserted. Docs passed ', counter) insert_data_list_in_tables(sql, agents)
def flush(self, ): if self._client is not None: self._client.add_transaction(self._transaction.as_dict()) # rows are sharded by cliend_id shard_idx = md5_hash(self._client.client_id()) % self.n_shards data = self._client.as_dict() self.outs[shard_idx].write(json.dumps(data) + "\n") self._client = None self._transaction = None
def register(): data = request.form username = data["username"] password = data["password"] cpassword = data["cpassword"] #TODO: here should check username and password valid or not in backend #Lazy to change if got time, should do client.register(username, utils.md5_hash(password)) session["user"] = client.get_user_by_name(username) return redirect("/")
def login(): data = request.form username = data["username"].strip() password = data["password"] user = client.get_user_by_username_password(username, utils.md5_hash(password)) # join_date = client.get_user_by_join_date(username) if user is not None: session["user"] = user # session["join_date"] = join_date else: flash(u'Login failed. Wrong username or password.', 'danger') return redirect("/")
def data_for_cities_table(): sql = "INSERT INTO cities(id, city_name) VALUES(%s,%s)" collection = connect_mongo() cursor = collection.find() cities = set() for doc in cursor: address = doc['address'] if 'locality' not in address: continue locality = address['locality'] hash_city_id = md5_hash((locality['name'],)) cities.add((hash_city_id, locality['name'])) insert_data_list_in_tables(sql, cities)
def data_for_homes_table(): sql = """INSERT INTO homes(id, city_id, build_year, number_of_floors, address, parking, lifts_passenger, lifts_freight, lat, lon) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) """ collection = connect_mongo() cursor = collection.find() homes = set() homes_id = set() for doc in cursor: address = doc['address'] house = doc.get('house', None) if not house: continue locality = address.get('locality', None) if not locality: continue build_year = house.get('build_year', None) floors = house.get('floors', None) parking = house.get('parking', None) if parking: parking_str = ', '.join([inner_dict['display_name'] for inner_dict in parking]) else: parking_str = None lifts_freight = house.get('lifts_freight', None) lifts_passenger = house.get('lifts_passenger', None) position = address['position'] lat = position.get('lat', None) lon = position.get('lon', None) hash_house_id = md5_hash((address['name'], locality['name'])) hash_city_id = md5_hash((locality['name'],)) if hash_house_id not in homes_id: homes.add((hash_house_id, hash_city_id, build_year, floors, address['name'], parking_str, lifts_passenger, lifts_freight, lat, lon)) homes_id.add(hash_house_id) insert_data_list_in_tables(sql, homes)
def query_cache(dbo, sql, age = 60): """ Runs the query given and caches the result for age seconds. If there's already a valid cached entry for the query, returns the cached result instead. If CACHE_COMMON_QUERIES is set to false, just runs the query without doing any caching and is equivalent to db.query() """ if not CACHE_COMMON_QUERIES: return query(dbo, sql) cache_key = utils.md5_hash("%s:%s:%s" % (dbo.alias, dbo.database, sql.replace(" ", "_"))) results = cachemem.get(cache_key) if results is not None: return results results = query(dbo, sql) cachemem.put(cache_key, results, age) return results
def query_cache(self, sql, params=None, age=60, limit=0, distincton=""): """ Runs the query given and caches the result for age seconds. If there's already a valid cached entry for the query, returns the cached result instead. If CACHE_COMMON_QUERIES is set to false, just runs the query without doing any caching and is equivalent to Database.query() """ if not CACHE_COMMON_QUERIES: return self.query(sql, params=params, limit=limit) cache_key = utils.md5_hash("%s:%s:%s" % (self.database, sql, params)) results = cachemem.get(cache_key) if results is not None: return results results = self.query(sql, params=params, limit=limit, distincton=distincton) cachemem.put(cache_key, results, age) return results
def search_tag(): gender = age = zcdj = yf = yc = intr = '未知' mac = str(request.form.get('macAddress')) mac = mac.replace(':', '') md5 = md5_hash(mac) mac = normalize_mac(mac) ret = r.get(md5) if ret: ret = ret.decode() ret = json.loads(ret) if 'gender' in ret: gender = translate(ret['gender']) if 'age' in ret: age = translate(ret['age']) if 'zcdj' in ret: zcdj = translate(ret['zcdj']) if 'yf' in ret: yf = translate(ret['yf']) if 'yc' in ret: yc = translate(ret['yc']) if 'intr' in ret: intr = translate(ret['intr']) return mac, gender, age, zcdj, yf, yc, intr
def _has_attachment_changed(self, attachment): attachment_full_path = self.fs.path_for(attachment.filepath) attachment_md5_hash = utils.md5_hash(attachment_full_path) if attachment_md5_hash == attachment.meta.get('md5_hash', ''): return False return True
def _save_attachment(self, attachment): attachment_path = self.fs.path_for(attachment.filepath) self.zd.get_attachment(attachment.meta['relative_path'], attachment_path) attachment.meta['md5_hash'] = utils.md5_hash(attachment_path) self.fs.save_json(attachment.meta_filepath, attachment.meta)
def test_md5_hash(self): """ Tests the md5_hash function """ data = md5_hash("web2py rocks") self.assertEqual(data, '79509f3246a2824dee64635303e99204')
def test_md5_hash(self): """ Tests the md5_hash function """ data = md5_hash("web2py rocks") self.assertEqual(data, "79509f3246a2824dee64635303e99204")
def from_json(cls, json, model=None): if not json: return cls() if model is None: model = cls() if "id" in json: model.key = ndb.Key(cls, json["id"]) else: model.key = ndb.Key(cls, str(uuid.uuid4())) for key, prop in model._properties.iteritems(): if key not in json.keys(): continue if key == "created" or key == "updated": continue value = None if key == "password": if len(json[key]) < 1: raise HttpResponseException(PASSWORD) value = utils.md5_hash(json[key]) # TODO - have the client hash the password before sending it up? elif type(prop) in NDB_KEY_TYPES: if json[key] is None: if prop._repeated: value = [] else: value = None elif type(prop) == ndb.BlobKeyProperty: if prop._repeated: value = [BlobKey(k) for k in json[key]] else: value = BlobKey(json[key]) else: cls_kind = prop._kind if cls_kind is None: cls_kind = cls if prop._repeated: value = [ndb.Key(cls_kind, k) for k in json[key]] else: value = ndb.Key(cls_kind, json[key]) elif type(prop) in NDB_STRING_TYPES: if not prop._repeated: value = json[key] # Should allow strings to be null # json[key] is None or if json[key] is not None and not isinstance(json[key], basestring): value = "" else: repeated_list = getattr(model, key) if not repeated_list: repeated_list = [] if not isinstance(json[key], list): json[key] = [json[key]] for each in json[key]: if each not in repeated_list: repeated_list.append(each) value = repeated_list # the original code sorted this list before saving it # sorted(repeated_list, key=lambda list_item: list_item.lower()) elif type(prop) in NDB_DATE_TYPES: if type(json[key]) in DATE_TYPES: value = json[key] elif type(json[key]) == basestring or type(json[key]) == unicode: if type(prop) == ndb.DateProperty: value = utils.DateParser.date(json[key]) elif type(prop) == ndb.DateTimeProperty: value = utils.DateParser.datetime(json[key]) elif type(prop) == ndb.TimeProperty: value == utils.DateParser.time(json[key]) else: value = None else: value = None elif type(prop) == ndb.StructuredProperty: if prop._repeated: repeated_list = getattr(model, key) if not repeated_list: repeated_list = [] if not isinstance(json[key], list): json[key] = [json[key]] repeated_objs = {} if repeated_list: for st_model in repeated_list: repeated_objs[st_model.id] = st_model.to_dict() for each in json[key]: if not each: continue kes = each.keys() if "id" not in kes: each["id"] = str(uuid.uuid4()) st_model = prop._modelclass().from_json(each) repeated_objs[st_model.id] = st_model else: if each["id"] in repeated_objs: del repeated_objs[each["id"]] st_model = prop._modelclass().from_json(each) repeated_objs[st_model.id] = st_model if st_model not in repeated_list: repeated_list.append(st_model) value = repeated_objs.values() else: value = prop._modelclass().from_json(json[key]) elif type(json[key]) in SIMPLE_TYPES or type(prop) in NDB_PRIMITIVE_TYPES: if prop._repeated: if type(prop) == ndb.IntegerProperty: value = [int(item) for item in json[key]] elif type(prop) == ndb.FloatProperty: value = [float(item) for item in json[key]] elif type(prop) == ndb.BooleanProperty: value = [bool(item) for item in json[key]] else: value = json[key] else: if json[key] is None: value = None elif type(prop) == ndb.IntegerProperty: value = int(json[key]) elif type(prop) == ndb.FloatProperty: value = float(json[key]) elif type(prop) == ndb.BooleanProperty: value = bool(json[key]) else: value = json[key] else: logging.info("invalid json key:" % key) setattr(model, key, value) return model
def data_for_listings_table(): sql = """INSERT INTO listings(id, apt_id, agent_id, category, deal_type, published_date, offer_type, living_area, room_area, window_view, communal_payments, deposit, commission) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)""" collection = connect_mongo() cursor = collection.find() listings = set() listing_ids = set() counter = 0 for doc in cursor: counter += 1 seller = doc['seller'] house = doc.get('house', None) if not house: continue address = doc['address'] locality = address.get('locality', None) if not locality: continue price_info = doc['price_info'] object_info = doc['object_info'] floor = object_info.get('floor', None) number_of_rooms = object_info.get('rooms', None) area = object_info.get('area', None) agent = seller.get('agent', None) phone = agent.get('phone', None) agent_name = agent.get('full_name', None) if not agent_name: continue category = doc.get('category', None) deal_type = doc.get('deal_type') published_date = doc.get('published_date', None) offer_type = doc.get('offer_type', None) living_area = object_info.get('living_area', None) if not living_area: living_area = None room_area = object_info.get('room_area', None) if not room_area: room_area = None window_view = object_info.get('window_view', None) if window_view: window_view_str = ', '.join([inner_dict['display_name'] for inner_dict in window_view]) else: window_view_str = None communal_payments = price_info.get('communal_payments', None) if communal_payments: communal_payments_str = communal_payments.get('display_name', None) else: communal_payments_str = None deposit = price_info.get('deposit', None) if not deposit: deposit = None commission = price_info.get('commission', None) if not commission: commission = None id = doc['_id'] hash_apt_id = md5_hash((address['name'], locality['name'], floor, area, number_of_rooms)) hash_city_id = md5_hash((locality['name'],)) hash_agent_id = md5_hash((hash_city_id, agent_name, phone)) if id not in listing_ids: listings.add((id, hash_apt_id, hash_agent_id, category, deal_type, published_date, offer_type, living_area, room_area, window_view_str, communal_payments_str, deposit, commission)) listing_ids.add(id) if len(listings) > 100: insert_data_list_in_tables(sql, listings) listings = set() print('listings inserted. Docs passed ', counter) insert_data_list_in_tables(sql, listings)
def accepts( self, request_vars, session=None, formname='%(tablename)s_%(record_id)s', keepvalues=False, onvalidation=None, dbio=True, hideerror=False, detect_record_change=False, ): """ similar FORM.accepts but also does insert, update or delete in SQLDB. but if detect_record_change == True than: form.record_changed = False (record is properly validated/submitted) form.record_changed = True (record cannot be submitted because changed) elseif detect_record_change == False than: form.record_changed = None """ if request_vars.__class__.__name__ == 'Request': request_vars = request_vars.post_vars keyed = hasattr(self.table,'_primarykey') # implement logic to detect whether record exist but has been modified # server side self.record_changed = None if detect_record_change: if self.record: self.record_changed = False serialized = '|'.join(str(self.record[k]) for k in self.table.fields()) self.record_hash = md5_hash(serialized) # logic to deal with record_id for keyed tables if self.record: if keyed: formname_id = '.'.join(str(self.record[k]) for k in self.table._primarykey if hasattr(self.record,k)) record_id = dict((k,request_vars[k]) for k in self.table._primarykey) else: (formname_id, record_id) = (self.record.id, request_vars.get('id', None)) keepvalues = True else: if keyed: formname_id = 'create' record_id = dict([(k,None) for k in self.table._primarykey]) else: (formname_id, record_id) = ('create', None) if not keyed and isinstance(record_id, (list, tuple)): record_id = record_id[0] if formname: formname = formname % dict(tablename = self.table._tablename, record_id = formname_id) # ## THIS IS FOR UNIQUE RECORDS, read IS_NOT_IN_DB for fieldname in self.fields: field = self.table[fieldname] requires = field.requires or [] if not isinstance(requires, (list, tuple)): requires = [requires] [item.set_self_id(self.record_id) for item in requires if hasattr(item, 'set_self_id') and self.record_id] # ## END fields = {} for key in self.vars: fields[key] = self.vars[key] ret = FORM.accepts( self, request_vars, session, formname, keepvalues, onvalidation, hideerror=hideerror, ) if not ret and self.record and self.errors: ### if there are errors in update mode # and some errors refers to an already uploaded file # delete error if # - user not trying to upload a new file # - there is existing file and user is not trying to delete it # this is because removing the file may not pass validation for key in self.errors.keys(): if self.table[key].type == 'upload' \ and request_vars.get(key,None) in (None,'') \ and self.record[key] \ and not key+UploadWidget.ID_DELETE_SUFFIX in request_vars: del self.errors[key] if not self.errors: ret = True requested_delete = \ request_vars.get(self.FIELDNAME_REQUEST_DELETE, False) self.custom.end = TAG[''](self.hidden_fields(), self.custom.end) auch = record_id and self.errors and requested_delete # auch is true when user tries to delete a record # that does not pass validation, yet it should be deleted if not ret and not auch: for fieldname in self.fields: field = self.table[fieldname] ### this is a workaround! widgets should always have default not None! if not field.widget and field.type.startswith('list:') and \ not OptionsWidget.has_options(field): field.widget = self.widgets.list.widget if hasattr(field, 'widget') and field.widget and fieldname in request_vars: if fieldname in self.vars: value = self.vars[fieldname] elif self.record: value = self.record[fieldname] else: value = self.table[fieldname].default row_id = '%s_%s%s' % (self.table,fieldname,SQLFORM.ID_ROW_SUFFIX) widget = field.widget(field, value) self.field_parent[row_id].components = [ widget ] if not field.type.startswith('list:'): self.field_parent[row_id]._traverse(False,hideerror) self.custom.widget[ fieldname ] = widget return ret if record_id and str(record_id) != str(self.record_id): raise SyntaxError, 'user is tampering with form\'s record_id: ' \ '%s != %s' % (record_id, self.record_id) if requested_delete and self.custom.deletable: if dbio: if keyed: qry = reduce(lambda x,y: x & y, [self.table[k]==record_id[k] for k in self.table._primarykey]) if self.table._db(qry).delete(): self.vars.update(record_id) else: self.table._db(self.table.id == self.record.id).delete() self.vars.id = self.record.id self.errors.clear() for component in self.elements('input, select, textarea'): component['_disabled'] = True return True for fieldname in self.fields: if not fieldname in self.table: continue if not self.ignore_rw and not self.table[fieldname].writable: continue field = self.table[fieldname] if field.type == 'id': continue if field.type == 'boolean': if self.vars.get(fieldname, False): self.vars[fieldname] = fields[fieldname] = True else: self.vars[fieldname] = fields[fieldname] = False elif field.type == 'password' and self.record\ and request_vars.get(fieldname, None) == \ PasswordWidget.DEFAULT_PASSWORD_DISPLAY: continue # do not update if password was not changed elif field.type == 'upload': f = self.vars[fieldname] fd = fieldname + '__delete' if f == '' or f == None: if self.vars.get(fd, False) or not self.record: fields[fieldname] = '' else: fields[fieldname] = self.record[fieldname] self.vars[fieldname] = fields[fieldname] continue elif hasattr(f,'file'): (source_file, original_filename) = (f.file, f.filename) elif isinstance(f, (str, unicode)): ### do not know why this happens, it should not (source_file, original_filename) = \ (cStringIO.StringIO(f), 'file.txt') newfilename = field.store(source_file, original_filename) # this line is for backward compatibility only self.vars['%s_newfilename' % fieldname] = newfilename fields[fieldname] = newfilename if isinstance(field.uploadfield,str): fields[field.uploadfield] = source_file.read() # proposed by Hamdy (accept?) do we need fields at this point? self.vars[fieldname] = fields[fieldname] continue elif fieldname in self.vars: fields[fieldname] = self.vars[fieldname] elif field.default == None and field.type!='blob': self.errors[fieldname] = 'no data' return False value = fields.get(fieldname,None) if field.type == 'list:string': if not isinstance(value,(tuple,list)): fields[fieldname] = value and [value] or [] elif field.type.startswith('list:'): if not isinstance(value,list): fields[fieldname] = [safe_int(x) for x in (value and [value] or [])] elif field.type == 'integer': if value != None: fields[fieldname] = safe_int(value) elif field.type.startswith('reference'): if value != None and isinstance(self.table,Table) and not keyed: fields[fieldname] = safe_int(value) elif field.type == 'double': if value != None: fields[fieldname] = safe_float(value) for fieldname in self.vars: if fieldname != 'id' and fieldname in self.table.fields\ and not fieldname in fields and not fieldname\ in request_vars: fields[fieldname] = self.vars[fieldname] if dbio: if keyed: if reduce(lambda x,y: x and y, record_id.values()): # if record_id if fields: qry = reduce(lambda x,y: x & y, [self.table[k]==self.record[k] for k in self.table._primarykey]) self.table._db(qry).update(**fields) else: pk = self.table.insert(**fields) if pk: self.vars.update(pk) else: ret = False else: if record_id: self.vars.id = self.record.id if fields: self.table._db(self.table.id == self.record.id).update(**fields) else: self.vars.id = self.table.insert(**fields) return ret