def test(): ''' basic testing method ''' print('build in memory check') cms = CountMinSketch(width=100000, depth=7) # add elements for i in range(100): tmp = 100 * (i + 1) cms.add(str(i), tmp) print(cms.check(str(0), 'min')) print(cms.check(str(0), 'mean')) print(cms.check(str(0), 'mean-min')) cms.export('./dist/py_test.cms') print('import from disk check') cmsf = CountMinSketch(filepath='./dist/py_test.cms') if cms.width != cmsf.width: print('width does not match!') if cms.depth != cmsf.depth: print('depth does not match!') print(cmsf.check(str(0), 'min')) print(cmsf.check(str(0), 'mean')) print(cmsf.check(str(0), 'mean-min')) try: print('\n\nTest invalid initialization') cms_ex = CountMinSketch() except SyntaxError as ex: print(ex)
next(stock_trade_file) #print('CrossRef CMS Create Time: ' + str(today.strftime("%X")) + ' ' + str(today.strftime("%f"))) #CrossRef CMS Create for stock_trade_line in stock_trade_lines: stock_symbol = stock_trade_line[0].strip() add1 = crossref_stock_trade_frq_cms.add(stock_symbol) stock_trade_file.close() #print('CrossRef CMS Membership Check Time: ' + str(today.strftime("%X")) + ' ' + str(today.strftime("%f"))) #CrossRef CMS Membership Check check_stock_symbol_file = open(check_stock_symbol_filename, "r") for stock_symbol_line in check_stock_symbol_file: stock_symbol = stock_symbol_line.strip() stock_frq = crossref_stock_trade_frq_cms.check(stock_symbol) if stock_frq == 0: crossref_not_present_count = crossref_not_present_count + 1 else: crossref_present_count = crossref_present_count + 1 check_stock_symbol_file.close() #CrossRef CMS Process Completed #print('CrossRef CMS Process Completed Time: ' + str(today.strftime("%X")) + ' ' + str(today.strftime("%f"))) #Actual CMS Process Starts stock_trade_file = open(stock_trade_filename,"r") stock_trade_lines = csv.reader(stock_trade_file, delimiter=',', quoting=csv.QUOTE_ALL, skipinitialspace=True) next(stock_trade_file) stock_trade_record_count = 0 cms_starttime = time.process_time() cms_starttime2 = time.asctime(time.localtime(time.time()))
if stock_symbol in stock_vol_dist.keys(): stock_symbol_freq = stock_freq_dist[stock_symbol] + 1 stock_symbol_vol = stock_vol_dist[stock_symbol] + stock_vol else: stock_symbol_freq = 1 stock_symbol_vol = stock_vol stock_freq_dist.update({stock_symbol: stock_symbol_freq}) stock_vol_dist.update({stock_symbol: stock_symbol_vol}) vol_sketch_qrytime = 0 # read sketch for stock_symbol in list(stock_vol_dist.keys()): vol_sketch_starttime = time.process_time() stock_symbol_vol_apds = stock_vol_apds.check(stock_symbol) vol_sketch_endtime = time.process_time() vol_sketch_qrytime = vol_sketch_qrytime + (vol_sketch_endtime - vol_sketch_starttime) freq_accuracy = 0 stock_symbol_freq = stock_freq_dist[stock_symbol] #freq_accuracy = 1 - abs(stock_symbol_freq-stock_freq_dist)/stock_symbol_freq if freq_accuracy < 0: freq_accuracy = 0 total_freq_accuracy = total_freq_accuracy + freq_accuracy stock_symbol_vol = stock_vol_dist[stock_symbol] vol_accuracy = 1 - abs(stock_symbol_vol - stock_symbol_vol_apds)/stock_symbol_vol if vol_accuracy < 0: vol_accuracy = 0 total_vol_accuracy = total_vol_accuracy + vol_accuracy stock_symbol_file.write(stock_symbol + "," + str(stock_trade_record_count))