Beispiel #1
0
    with open(filename) as f:
        for line in f:
            yield line.strip()


db = cachedb.CacheDB()

if __name__ == "__main__":
    datasets = {"mini": "testmini", "all": "testbig"}
    datasetlengths = {"all": 99072112, "mini": 4692576}
    for dataset in ["all", "mini"]:
        length = datasetlengths[dataset]
        records = load_records("nfp/%s.txt" % dataset)
        sitename = datasets[dataset]
        print
        print "Inserting %i records into %s" % (length, sitename)
        i = 0
        for record in records:
            if i % 1000 == 0:
                print_status("Inserting record %i" % i, i, length)
            i += 1
            (user, movie, rating, date) = record.split(",")
            datestamp = "%s%s" % (date.replace("-", ""), "120000000")
            db.insert_without_timestamp(sitename, ("user", user), ("movie", movie), ("rating", rating, datestamp))
        print_status()
        print "Finished importing %s" % dataset
        time.sleep(5)

while True:
    time.sleep(1)
Beispiel #2
0
#print type(l)
#print l

if __name__ == '__main__':
    datasets = {'mini': 'testmini', 'all': 'testbig'} 
    datasetlengths = {'all': 99072112, 'mini': 4692576}
    for dataset in ['mini', 'all']:
        length = datasetlengths[dataset]
        records = load_records('nfp/%s.txt' % dataset)
        sitename = datasets[dataset]
        print
        print 'Testing %i records into %s' % (length, sitename)
        i = 0
        users = {}
        for record in records:
            if (i % 1000 == 0): print_status('Testing record %i' % i, i, length)
            i += 1
            (user, movie, rating, date) = record.split(',')
            if user not in users:
                users[user] = db.get_key(sitename, ('user', user))
            #record = db.get_subkey(sitename, ('user', user), ('movie', movie))
            record = users[user][('movie', movie)]
            datestamp = '%s%s' % (date.replace('-', ''), '120000000')
            expected = [('rating', rating, datestamp)]
            test = (record == expected)
            if not test:
                print 'fail: %s != %s' % (record, expected)
            #ns.insert(('user', user), ('movie', movie), ('rating', rating, date), add_timestamp=False)
        print_status()
        print 'Finished testing %s' % dataset
        time.sleep(5)