Exemple #1
0
def data_for_apartments_table():
    sql = """INSERT INTO apartments(id, home_id, floor, number_of_rooms, area,
        kitchen_area, number_of_bedrooms, number_of_bathrooms, separated_bathrooms)
        VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s)
        """
    collection = connect_mongo()
    cursor = collection.find()
    apts = set()
    apt_ids = set()
    for doc in cursor:
        house = doc.get('house', None)
        if not house:
            continue
        if 'object_info' not in doc:
            continue
        apt_info = doc['object_info']
        address = doc['address']
        locality = address.get('locality', None)
        if not locality:
            continue
        floor = apt_info.get('floor', None)
        number_of_rooms = apt_info.get('rooms', None)
        area = apt_info.get('area', None)
        kitchen_area = apt_info.get('kitchen_area', None)
        number_of_bedrooms = apt_info.get('bedrooms', None)
        number_of_bathrooms = apt_info.get('bathrooms', None)
        separated_bathrooms = bool(apt_info.get('separated_bathrooms', None))

        hash_house_id = md5_hash((address['name'], locality['name']))
        hash_apt_id = md5_hash((address['name'], locality['name'], floor, area, number_of_rooms))
        if hash_apt_id not in apt_ids:
            apts.add((hash_apt_id, hash_house_id, floor, number_of_rooms, area, kitchen_area,
                       number_of_bedrooms, number_of_bathrooms, separated_bathrooms))
            apt_ids.add(hash_apt_id)
    insert_data_list_in_tables(sql, apts)
Exemple #2
0
def sign_document(dbo, username, mid, sigurl, signdate):
    """
    Signs an HTML document.
    sigurl: An HTML5 data: URL containing an image of the signature
    """
    SIG_PLACEHOLDER = "signature:placeholder"
    date, medianame, mimetype, content = get_media_file_data(dbo, mid)
    # Is this an HTML document?
    if content.find("<p") == -1 and content.find("<td") == -1:
        raise utils.ASMValidationError("Cannot sign a non-HTML document")
    # Has this document already been signed? 
    if 0 != db.query_int(dbo, "SELECT COUNT(*) FROM media WHERE ID = %d AND SignatureHash Is Not Null AND SignatureHash <> ''" % mid):
        raise utils.ASMValidationError("Document is already signed")
    # Does the document have a signing placeholder image? If so, replace it
    if content.find(SIG_PLACEHOLDER) != -1:
        content = content.replace(SIG_PLACEHOLDER, sigurl)
    else:
        # Create the signature at the foot of the document
        sig = "<hr />\n"
        sig += '<p><img src="' + sigurl + '" /></p>\n'
        sig += "<p>%s</p>\n" % signdate
        content += sig
    # Create a hash of the contents and store it with the media record
    db.execute(dbo, "UPDATE media SET SignatureHash = '%s' WHERE ID = %d" % (utils.md5_hash(content), mid))
    # Update the dbfs contents
    update_file_content(dbo, username, mid, content)
Exemple #3
0
 def _push_new_attachment(self, attachment):
     attachment_full_path = self.fs.path_for(attachment.filepath)
     meta = self.req.post_attachment(
         attachment, attachment_full_path)['article_attachment']
     meta['md5_hash'] = utils.md5_hash(attachment_full_path)
     meta = self.fs.save_json(attachment.meta_filepath, meta)
     attachment.meta = meta
Exemple #4
0
def sign_document(dbo, username, mid, sigurl, signdate):
    """
    Signs an HTML document.
    sigurl: An HTML5 data: URL containing an image of the signature
    """
    al.debug("signing document %s for %s" % (mid, username), "media.sign_document", dbo)
    SIG_PLACEHOLDER = "signature:placeholder"
    date, medianame, mimetype, content = get_media_file_data(dbo, mid)
    # Is this an HTML document?
    if content.find("<p") == -1 and content.find("<td") == -1:
        al.error("document %s is not HTML" % mid, "media.sign_document", dbo)
        raise utils.ASMValidationError("Cannot sign a non-HTML document")
    # Has this document already been signed? 
    if 0 != dbo.query_int("SELECT COUNT(*) FROM media WHERE ID = ? AND SignatureHash Is Not Null AND SignatureHash <> ''", [mid]):
        al.error("document %s has already been signed" % mid, "media.sign_document", dbo)
        raise utils.ASMValidationError("Document is already signed")
    # Does the document have a signing placeholder image? If so, replace it
    if content.find(SIG_PLACEHOLDER) != -1:
        al.debug("document %s: found signature placeholder" % mid, "media.sign_document", dbo)
        content = content.replace(SIG_PLACEHOLDER, sigurl)
    else:
        # Create the signature at the foot of the document
        al.debug("document %s: no placeholder, appending" % mid, "media.sign_document", dbo)
        sig = "<hr />\n"
        sig += '<p><img src="' + sigurl + '" /></p>\n'
        sig += "<p>%s</p>\n" % signdate
        content += sig
    # Create a hash of the contents and store it with the media record
    dbo.update("media", mid, { "SignatureHash": utils.md5_hash(content) })
    # Update the dbfs contents
    update_file_content(dbo, username, mid, content)
Exemple #5
0
def data_for_companies_table():
    sql = """INSERT INTO companies(id, company_name, phone) VALUES(%s,%s,%s)"""
    collection = connect_mongo()
    cursor = collection.find()
    companies = set()
    company_ids = set()
    for doc in cursor:
        seller = doc['seller']
        house = doc.get('house', None)
        if not house:
            continue
        address = doc['address']
        locality = address.get('locality', None)
        if not locality:
            continue
        company = seller.get('company', None)
        company_name = company.get('name', None)
        if not company_name:
            continue
        phone = company.get('phone', None)
        hash_company_id = md5_hash((company_name,))
        if hash_company_id not in company_ids:
            companies.add((hash_company_id, company_name, phone))
            company_ids.add(hash_company_id)


    insert_data_list_in_tables(sql, companies)
Exemple #6
0
def data_for_agents_table():
    sql = """INSERT INTO agents(id, company_id, agent_name, phone, is_agent) VALUES(%s,%s,%s,%s,%s)"""
    collection = connect_mongo()
    cursor = collection.find()
    agents = set()
    agent_ids = set()
    counter = 0

    for doc in cursor:
        counter += 1
        seller = doc['seller']
        house = doc.get('house', None)
        if not house:
            continue
        address = doc['address']
        locality = address.get('locality', None)
        if not locality:
            continue

        company = seller.get('company', None)
        company_name = company.get('name', None)
        hash_company_id = md5_hash((company_name,))
        if not company_name:
            hash_company_id = None

        agent = seller.get('agent', None)
        phone = agent.get('phone', None)
        is_agent = agent.get('is_agent', None)
        agent_name = agent.get('full_name', None)
        if not agent_name:
            print(doc['_id'])
            continue

        hash_city_id = md5_hash((locality['name'],))
        hash_agent_id = md5_hash((hash_city_id, agent_name, phone))
        if hash_agent_id not in agent_ids:
            agents.add((hash_agent_id, hash_company_id, agent_name, phone, is_agent))
            agent_ids.add(hash_agent_id)
        if len(agents) > 100:
            insert_data_list_in_tables(sql, agents)
            agents = set()
            print('agents inserted. Docs passed ', counter)



    insert_data_list_in_tables(sql, agents)
    def flush(self, ):
        if self._client is not None:
            self._client.add_transaction(self._transaction.as_dict())
            # rows are sharded by cliend_id
            shard_idx = md5_hash(self._client.client_id()) % self.n_shards
            data = self._client.as_dict()
            self.outs[shard_idx].write(json.dumps(data) + "\n")

            self._client = None
            self._transaction = None
Exemple #8
0
def register():
    data = request.form
    username = data["username"]
    password = data["password"]
    cpassword = data["cpassword"] 
    #TODO: here should check username and password valid or not in backend
    #Lazy to change if got time, should do
    client.register(username, utils.md5_hash(password))
    session["user"] = client.get_user_by_name(username)
    return redirect("/")
Exemple #9
0
def login():
    data = request.form
    username = data["username"].strip()
    password = data["password"]
    user = client.get_user_by_username_password(username, utils.md5_hash(password))
    # join_date = client.get_user_by_join_date(username)
    if user is not None:
        session["user"] = user
        # session["join_date"] = join_date
    else:
        flash(u'Login failed. Wrong username or password.', 'danger')
    return redirect("/")
Exemple #10
0
def data_for_cities_table():
    sql = "INSERT INTO cities(id, city_name) VALUES(%s,%s)"
    collection = connect_mongo()
    cursor = collection.find()
    cities = set()
    for doc in cursor:
        address = doc['address']
        if 'locality' not in address:
            continue
        locality = address['locality']
        hash_city_id = md5_hash((locality['name'],))
        cities.add((hash_city_id, locality['name']))

    insert_data_list_in_tables(sql, cities)
Exemple #11
0
def data_for_homes_table():
    sql = """INSERT INTO homes(id, city_id, build_year, number_of_floors, address,
        parking, lifts_passenger, lifts_freight, lat, lon)
        VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
        """
    collection = connect_mongo()
    cursor = collection.find()
    homes = set()
    homes_id = set()
    for doc in cursor:
        address = doc['address']
        house = doc.get('house', None)
        if not house:
            continue
        locality = address.get('locality', None)
        if not locality:
            continue
        build_year = house.get('build_year', None)
        floors = house.get('floors', None)
        parking = house.get('parking', None)
        if parking:
            parking_str = ', '.join([inner_dict['display_name'] for inner_dict in parking])
        else:
            parking_str = None
        lifts_freight = house.get('lifts_freight', None)
        lifts_passenger = house.get('lifts_passenger', None)
        position = address['position']
        lat = position.get('lat', None)
        lon = position.get('lon', None)
        hash_house_id = md5_hash((address['name'], locality['name']))
        hash_city_id = md5_hash((locality['name'],))
        if hash_house_id not in homes_id:
            homes.add((hash_house_id, hash_city_id, build_year, floors, address['name'], parking_str,
                       lifts_passenger, lifts_freight, lat, lon))
            homes_id.add(hash_house_id)
    insert_data_list_in_tables(sql, homes)
Exemple #12
0
Fichier : db.py Projet : magul/asm3
def query_cache(dbo, sql, age = 60):
    """
    Runs the query given and caches the result
    for age seconds. If there's already a valid cached
    entry for the query, returns the cached result
    instead.
    If CACHE_COMMON_QUERIES is set to false, just runs the query
    without doing any caching and is equivalent to db.query()
    """
    if not CACHE_COMMON_QUERIES: return query(dbo, sql)
    cache_key = utils.md5_hash("%s:%s:%s" % (dbo.alias, dbo.database, sql.replace(" ", "_")))
    results = cachemem.get(cache_key)
    if results is not None:
        return results
    results = query(dbo, sql)
    cachemem.put(cache_key, results, age)
    return results
Exemple #13
0
 def query_cache(self, sql, params=None, age=60, limit=0, distincton=""):
     """
     Runs the query given and caches the result
     for age seconds. If there's already a valid cached
     entry for the query, returns the cached result
     instead.
     If CACHE_COMMON_QUERIES is set to false, just runs the query
     without doing any caching and is equivalent to Database.query()
     """
     if not CACHE_COMMON_QUERIES: return self.query(sql, params=params, limit=limit)
     cache_key = utils.md5_hash("%s:%s:%s" % (self.database, sql, params))
     results = cachemem.get(cache_key)
     if results is not None:
         return results
     results = self.query(sql, params=params, limit=limit, distincton=distincton)
     cachemem.put(cache_key, results, age)
     return results
Exemple #14
0
def search_tag():
    gender = age = zcdj = yf = yc = intr = '未知'
    mac = str(request.form.get('macAddress'))
    mac = mac.replace(':', '')
    md5 = md5_hash(mac)
    mac = normalize_mac(mac)
    ret = r.get(md5)
    if ret:
        ret = ret.decode()
        ret = json.loads(ret)
        if 'gender' in ret:
            gender = translate(ret['gender'])
        if 'age' in ret:
            age = translate(ret['age'])
        if 'zcdj' in ret:
            zcdj = translate(ret['zcdj'])
        if 'yf' in ret:
            yf = translate(ret['yf'])
        if 'yc' in ret:
            yc = translate(ret['yc'])
        if 'intr' in ret:
            intr = translate(ret['intr'])

    return mac, gender, age, zcdj, yf, yc, intr
Exemple #15
0
 def _has_attachment_changed(self, attachment):
     attachment_full_path = self.fs.path_for(attachment.filepath)
     attachment_md5_hash = utils.md5_hash(attachment_full_path)
     if attachment_md5_hash == attachment.meta.get('md5_hash', ''):
         return False
     return True
 def _save_attachment(self, attachment):
     attachment_path = self.fs.path_for(attachment.filepath)
     self.zd.get_attachment(attachment.meta['relative_path'], attachment_path)
     attachment.meta['md5_hash'] = utils.md5_hash(attachment_path)
     self.fs.save_json(attachment.meta_filepath, attachment.meta)
    def test_md5_hash(self):
        """ Tests the md5_hash function """

        data = md5_hash("web2py rocks")
        self.assertEqual(data, '79509f3246a2824dee64635303e99204')
Exemple #18
0
    def test_md5_hash(self):
        """ Tests the md5_hash function """

        data = md5_hash("web2py rocks")
        self.assertEqual(data, "79509f3246a2824dee64635303e99204")
Exemple #19
0
    def from_json(cls, json, model=None):
        if not json:
            return cls()

        if model is None:
            model = cls()

        if "id" in json:
            model.key = ndb.Key(cls, json["id"])
        else:
            model.key = ndb.Key(cls, str(uuid.uuid4()))

        for key, prop in model._properties.iteritems():
            if key not in json.keys():
                continue

            if key == "created" or key == "updated":
                continue

            value = None

            if key == "password":
                if len(json[key]) < 1:
                    raise HttpResponseException(PASSWORD)
                value = utils.md5_hash(json[key])  # TODO - have the client hash the password before sending it up?

            elif type(prop) in NDB_KEY_TYPES:
                if json[key] is None:
                    if prop._repeated:
                        value = []
                    else:
                        value = None
                elif type(prop) == ndb.BlobKeyProperty:
                    if prop._repeated:
                        value = [BlobKey(k) for k in json[key]]
                    else:
                        value = BlobKey(json[key])
                else:
                    cls_kind = prop._kind
                    if cls_kind is None:
                        cls_kind = cls
                    if prop._repeated:
                        value = [ndb.Key(cls_kind, k) for k in json[key]]
                    else:
                        value = ndb.Key(cls_kind, json[key])

            elif type(prop) in NDB_STRING_TYPES:
                if not prop._repeated:
                    value = json[key]
                    # Should allow strings to be null
                    # json[key] is None or
                    if json[key] is not None and not isinstance(json[key], basestring):
                        value = ""

                else:
                    repeated_list = getattr(model, key)
                    if not repeated_list:
                        repeated_list = []

                    if not isinstance(json[key], list):
                        json[key] = [json[key]]

                    for each in json[key]:
                        if each not in repeated_list:
                            repeated_list.append(each)

                    value = repeated_list
                    # the original code sorted this list before saving it
                    # sorted(repeated_list, key=lambda list_item: list_item.lower())

            elif type(prop) in NDB_DATE_TYPES:
                if type(json[key]) in DATE_TYPES:
                    value = json[key]
                elif type(json[key]) == basestring or type(json[key]) == unicode:
                    if type(prop) == ndb.DateProperty:
                        value = utils.DateParser.date(json[key])
                    elif type(prop) == ndb.DateTimeProperty:
                        value = utils.DateParser.datetime(json[key])
                    elif type(prop) == ndb.TimeProperty:
                        value == utils.DateParser.time(json[key])
                    else:
                        value = None
                else:
                    value = None

            elif type(prop) == ndb.StructuredProperty:
                if prop._repeated:
                    repeated_list = getattr(model, key)
                    if not repeated_list:
                        repeated_list = []

                    if not isinstance(json[key], list):
                        json[key] = [json[key]]

                    repeated_objs = {}
                    if repeated_list:
                        for st_model in repeated_list:
                            repeated_objs[st_model.id] = st_model.to_dict()

                    for each in json[key]:
                        if not each:
                            continue

                        kes = each.keys()
                        if "id" not in kes:
                            each["id"] = str(uuid.uuid4())
                            st_model = prop._modelclass().from_json(each)
                            repeated_objs[st_model.id] = st_model

                        else:
                            if each["id"] in repeated_objs:
                                del repeated_objs[each["id"]]

                            st_model = prop._modelclass().from_json(each)
                            repeated_objs[st_model.id] = st_model

                        if st_model not in repeated_list:
                            repeated_list.append(st_model)

                    value = repeated_objs.values()

                else:
                    value = prop._modelclass().from_json(json[key])

            elif type(json[key]) in SIMPLE_TYPES or type(prop) in NDB_PRIMITIVE_TYPES:
                if prop._repeated:
                    if type(prop) == ndb.IntegerProperty:
                        value = [int(item) for item in json[key]]
                    elif type(prop) == ndb.FloatProperty:
                        value = [float(item) for item in json[key]]
                    elif type(prop) == ndb.BooleanProperty:
                        value = [bool(item) for item in json[key]]
                    else:
                        value = json[key]
                else:
                    if json[key] is None:
                        value = None
                    elif type(prop) == ndb.IntegerProperty:
                        value = int(json[key])

                    elif type(prop) == ndb.FloatProperty:
                        value = float(json[key])

                    elif type(prop) == ndb.BooleanProperty:
                        value = bool(json[key])

                    else:
                        value = json[key]

            else:
                logging.info("invalid json key:" % key)

            setattr(model, key, value)

        return model
Exemple #20
0
def data_for_listings_table():
    sql = """INSERT INTO listings(id, apt_id, agent_id, category, deal_type, published_date, offer_type,
        living_area, room_area, window_view, communal_payments, deposit, commission)
        VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"""
    collection = connect_mongo()
    cursor = collection.find()
    listings = set()
    listing_ids = set()
    counter = 0

    for doc in cursor:
        counter += 1
        seller = doc['seller']
        house = doc.get('house', None)
        if not house:
            continue
        address = doc['address']
        locality = address.get('locality', None)
        if not locality:
            continue
        price_info = doc['price_info']
        object_info = doc['object_info']
        floor = object_info.get('floor', None)
        number_of_rooms = object_info.get('rooms', None)
        area = object_info.get('area', None)

        agent = seller.get('agent', None)
        phone = agent.get('phone', None)
        agent_name = agent.get('full_name', None)
        if not agent_name:
            continue

        category = doc.get('category', None)
        deal_type = doc.get('deal_type')
        published_date = doc.get('published_date', None)
        offer_type = doc.get('offer_type', None)
        living_area = object_info.get('living_area', None)
        if not living_area:
            living_area = None
        room_area = object_info.get('room_area', None)
        if not room_area:
            room_area = None
        window_view = object_info.get('window_view', None)
        if window_view:
            window_view_str = ', '.join([inner_dict['display_name'] for inner_dict in window_view])
        else:
            window_view_str = None
        communal_payments = price_info.get('communal_payments', None)
        if communal_payments:
            communal_payments_str = communal_payments.get('display_name', None)
        else:
            communal_payments_str = None
        deposit = price_info.get('deposit', None)
        if not deposit:
            deposit = None
        commission = price_info.get('commission', None)
        if not commission:
            commission = None

        id = doc['_id']
        hash_apt_id = md5_hash((address['name'], locality['name'], floor, area, number_of_rooms))
        hash_city_id = md5_hash((locality['name'],))
        hash_agent_id = md5_hash((hash_city_id, agent_name, phone))
        if id not in listing_ids:
            listings.add((id, hash_apt_id, hash_agent_id, category, deal_type, published_date, offer_type,
                          living_area, room_area, window_view_str, communal_payments_str, deposit,
                          commission))
            listing_ids.add(id)
        if len(listings) > 100:
            insert_data_list_in_tables(sql, listings)
            listings = set()
            print('listings inserted. Docs passed ', counter)



    insert_data_list_in_tables(sql, listings)
Exemple #21
0
    def accepts(
        self,
        request_vars,
        session=None,
        formname='%(tablename)s_%(record_id)s',
        keepvalues=False,
        onvalidation=None,
        dbio=True,
        hideerror=False,
        detect_record_change=False,
        ):

        """
        similar FORM.accepts but also does insert, update or delete in SQLDB.
        but if detect_record_change == True than:
          form.record_changed = False (record is properly validated/submitted)
          form.record_changed = True (record cannot be submitted because changed)
        elseif detect_record_change == False than:
          form.record_changed = None
        """

        if request_vars.__class__.__name__ == 'Request':
            request_vars = request_vars.post_vars

        keyed = hasattr(self.table,'_primarykey')

        # implement logic to detect whether record exist but has been modified
        # server side
        self.record_changed = None            
        if detect_record_change:
            if self.record:
                self.record_changed = False
                serialized = '|'.join(str(self.record[k]) for k in self.table.fields())
                self.record_hash = md5_hash(serialized)

        # logic to deal with record_id for keyed tables
        if self.record:
            if keyed:
                formname_id = '.'.join(str(self.record[k]) 
                                       for k in self.table._primarykey
                                       if hasattr(self.record,k))
                record_id = dict((k,request_vars[k]) for k in self.table._primarykey)
            else:
                (formname_id, record_id) = (self.record.id, request_vars.get('id', None))
            keepvalues = True
        else:
            if keyed:
                formname_id = 'create'
                record_id = dict([(k,None) for k in self.table._primarykey])
            else:
                (formname_id, record_id) = ('create', None)

        if not keyed and isinstance(record_id, (list, tuple)):
            record_id = record_id[0]

        if formname:
            formname = formname % dict(tablename = self.table._tablename,
                                       record_id = formname_id)

        # ## THIS IS FOR UNIQUE RECORDS, read IS_NOT_IN_DB

        for fieldname in self.fields:
            field = self.table[fieldname]
            requires = field.requires or []
            if not isinstance(requires, (list, tuple)):
                requires = [requires]
            [item.set_self_id(self.record_id) for item in requires
            if hasattr(item, 'set_self_id') and self.record_id]

        # ## END

        fields = {}
        for key in self.vars:
            fields[key] = self.vars[key]

        ret = FORM.accepts(
            self,
            request_vars,
            session,
            formname,
            keepvalues,
            onvalidation,
            hideerror=hideerror,
            )

        if not ret and self.record and self.errors:
            ### if there are errors in update mode
            # and some errors refers to an already uploaded file
            # delete error if
            # - user not trying to upload a new file
            # - there is existing file and user is not trying to delete it
            # this is because removing the file may not pass validation
            for key in self.errors.keys():
                if self.table[key].type == 'upload' \
                        and request_vars.get(key,None) in (None,'') \
                        and self.record[key] \
                        and not key+UploadWidget.ID_DELETE_SUFFIX in request_vars:
                    del self.errors[key]
            if not self.errors:
                ret = True

        requested_delete = \
            request_vars.get(self.FIELDNAME_REQUEST_DELETE, False)

        self.custom.end = TAG[''](self.hidden_fields(), self.custom.end)

        auch = record_id and self.errors and requested_delete

        # auch is true when user tries to delete a record
        # that does not pass validation, yet it should be deleted

        if not ret and not auch:
            for fieldname in self.fields:
                field = self.table[fieldname]
                ### this is a workaround! widgets should always have default not None!
                if not field.widget and field.type.startswith('list:') and \
                        not OptionsWidget.has_options(field):
                    field.widget = self.widgets.list.widget
                if hasattr(field, 'widget') and field.widget and fieldname in request_vars:
                    if fieldname in self.vars:
                        value = self.vars[fieldname]
                    elif self.record:
                        value = self.record[fieldname]
                    else:
                        value = self.table[fieldname].default
                    row_id = '%s_%s%s' % (self.table,fieldname,SQLFORM.ID_ROW_SUFFIX)
                    widget = field.widget(field, value)
                    self.field_parent[row_id].components = [ widget ]
                    if not field.type.startswith('list:'):
                        self.field_parent[row_id]._traverse(False,hideerror)
                    self.custom.widget[ fieldname ] = widget
            return ret

        if record_id and str(record_id) != str(self.record_id):
            raise SyntaxError, 'user is tampering with form\'s record_id: ' \
                               '%s != %s' % (record_id, self.record_id)

        if requested_delete and self.custom.deletable:
            if dbio:
                if keyed:
                    qry = reduce(lambda x,y: x & y,
                                 [self.table[k]==record_id[k] for k in self.table._primarykey])
                    if self.table._db(qry).delete():
                        self.vars.update(record_id)
                else:
                    self.table._db(self.table.id == self.record.id).delete()            
                    self.vars.id = self.record.id
            self.errors.clear()
            for component in self.elements('input, select, textarea'):
                component['_disabled'] = True
            return True

        for fieldname in self.fields:
            if not fieldname in self.table:
                continue

            if not self.ignore_rw and not self.table[fieldname].writable:
                continue

            field = self.table[fieldname]
            if field.type == 'id':
                continue
            if field.type == 'boolean':
                if self.vars.get(fieldname, False):
                    self.vars[fieldname] = fields[fieldname] = True
                else:
                    self.vars[fieldname] = fields[fieldname] = False
            elif field.type == 'password' and self.record\
                and request_vars.get(fieldname, None) == \
                    PasswordWidget.DEFAULT_PASSWORD_DISPLAY:
                continue  # do not update if password was not changed
            elif field.type == 'upload':
                f = self.vars[fieldname]
                fd = fieldname + '__delete'
                if f == '' or f == None:
                    if self.vars.get(fd, False) or not self.record:
                        fields[fieldname] = ''
                    else:
                        fields[fieldname] = self.record[fieldname]
                    self.vars[fieldname] = fields[fieldname]
                    continue
                elif hasattr(f,'file'):
                    (source_file, original_filename) = (f.file, f.filename)
                elif isinstance(f, (str, unicode)):
                    ### do not know why this happens, it should not
                    (source_file, original_filename) = \
                        (cStringIO.StringIO(f), 'file.txt')
                newfilename = field.store(source_file, original_filename)
                # this line is for backward compatibility only
                self.vars['%s_newfilename' % fieldname] = newfilename
                fields[fieldname] = newfilename
                if isinstance(field.uploadfield,str):
                    fields[field.uploadfield] = source_file.read()
                # proposed by Hamdy (accept?) do we need fields at this point?
                self.vars[fieldname] = fields[fieldname]
                continue
            elif fieldname in self.vars:
                fields[fieldname] = self.vars[fieldname]
            elif field.default == None and field.type!='blob':
                self.errors[fieldname] = 'no data'
                return False
            value = fields.get(fieldname,None)
            if field.type == 'list:string':
                if not isinstance(value,(tuple,list)):
                    fields[fieldname] = value and [value] or []
            elif field.type.startswith('list:'):
                if not isinstance(value,list):
                    fields[fieldname] = [safe_int(x) for x in (value and [value] or [])]
            elif field.type == 'integer':
                if value != None:
                    fields[fieldname] = safe_int(value)
            elif field.type.startswith('reference'):
                if value != None and isinstance(self.table,Table) and not keyed:
                    fields[fieldname] = safe_int(value)
            elif field.type == 'double':
                if value != None:
                    fields[fieldname] = safe_float(value)

        for fieldname in self.vars:
            if fieldname != 'id' and fieldname in self.table.fields\
                 and not fieldname in fields and not fieldname\
                 in request_vars:
                fields[fieldname] = self.vars[fieldname]

        if dbio:
            if keyed:
                if reduce(lambda x,y: x and y, record_id.values()): # if record_id
                    if fields:
                        qry = reduce(lambda x,y: x & y, [self.table[k]==self.record[k] for k in self.table._primarykey])
                        self.table._db(qry).update(**fields)
                else:
                    pk = self.table.insert(**fields)
                    if pk:
                        self.vars.update(pk)
                    else:
                        ret = False
            else:
                if record_id:
                    self.vars.id = self.record.id
                    if fields:
                        self.table._db(self.table.id == self.record.id).update(**fields)
                else:
                    self.vars.id = self.table.insert(**fields)
        return ret