def test_hits(self): # lru sa_cache = Cache.Cache(2, 8) data = [(x, x**2) for x in range(12)] for i in range(8): sa_cache.put(data[i][0], data[i][1]) for i in range(8): sa_cache.get(data[i][0]) self.assertEqual(i + 1, sa_cache.get_hits()) for i in range(8, 12): sa_cache.put(data[i][0], data[i][1]) for i in range(0, 4): sa_cache.get(data[i][0]) self.assertEqual(i + 1, sa_cache.get_misses()) # mru sa_cache = Cache.Cache(2, 8, "mru") for i in range(8): sa_cache.put(data[i][0], data[i][1]) for i in range(8): sa_cache.get(data[i][0]) self.assertEqual(i + 1, sa_cache.get_hits()) for i in range(8, 12): sa_cache.put(data[i][0], data[i][1]) for i in range(4, 8): sa_cache.get(data[i][0]) self.assertEqual(i - 3, sa_cache.get_misses())
def test_built_in_types(self): sa_cache = Cache.Cache(2, 8) # test put & get sa_cache.put("Username", "johanan_lai1997") self.assertEqual("johanan_lai1997", sa_cache.get("Username")) sa_cache.put("Save password", False) self.assertFalse(sa_cache.get("Save password")) sa_cache.put("Login attempts", 3) self.assertEqual(3, sa_cache.get("Login attempts")) sa_cache.put(200, 500) self.assertEqual(500, sa_cache.get(200)) # test remove sa_cache.remove("Login attempts") self.assertIsNone(sa_cache.get("Login attempts")) self.assertEqual(500, sa_cache.remove(200)) self.assertIsNone(sa_cache.get(200)) sa_cache.remove("Username") sa_cache.remove("Save password") self.assertIsNone(sa_cache.get("Username")) self.assertIsNone(sa_cache.get("Save password")) # test clear for entry in sa_cache._cache: self.assertFalse(entry) sa_cache.put("Username", "johanan_lai1997") sa_cache.put("Save password", False) sa_cache.put("Login attempts", 3) sa_cache.put(200, 500) sa_cache.clear() for entry in sa_cache._cache: self.assertFalse(entry) # test update sa_cache.put("Prompt on quit", True) self.assertTrue(sa_cache.get("Prompt on quit")) sa_cache.update("Prompt on quit", False) self.assertFalse(sa_cache.get("Prompt on quit")) # test replacement sa_cache2 = Cache.Cache(2, 2) sa_cache2.put("string1", 10923) self.assertEqual(10923, sa_cache2.get("string1")) sa_cache2.put("string2", False) self.assertEqual(10923, sa_cache2.get("string1")) self.assertFalse(sa_cache2.get("string2")) sa_cache2.get("string1") sa_cache2.put(1000, "hello world") self.assertEqual("hello world", sa_cache2.get(1000)) self.assertIsNone(sa_cache2.get("string2"))
def level_chosen(self, event): level = self.level_choice.GetStringSelection() # Get level chosen self.subject_choice.Clear() self.paper_checklist.Clear() self.year_choice.Clear() self.season_choice.Clear() self.num_choice.Clear() self.region_choice.Clear() self.pairs_info = {} self.files_info = {} if level == self.level_list[0]: # Not choosing a level return # Cache global cache_folder cache_subject = os.path.join(cache_folder, "GCE Guide %s" % level) if not os.path.exists(cache_subject): self.subject_dict = Crawler.visit_level( Crawler.levels_dict[level]) # Return subject list if self.subject_dict == -1: # Connection error wx.MessageBox( "Please check your Internet connection and retry.", "Connection Error") self.level_choice.SetSelection(0) return else: Cache.store(self.subject_dict, cache_subject) else: self.subject_dict = Cache.load(cache_subject) subject_list = ["----- Select subject -----" ] + [each for each in self.subject_dict] self.subject_choice.Set(subject_list) # Update subject list self.subject_choice.SetSelection(0)
def load( self, url = "", onLoad = None, onProgress = None, onError = None ): if self.path: url = self.path + url cached = Cache.get( url ) if cached: self.manager.itemStart( url ) if onLoad: onLoad( cached ) self.manager.itemEnd( url ) return cached try: self.manager.itemStart( url ) cached = pygame.image.load( url ) except IOError as error: if onErorr: onError( error ) self.manager.itemEnd( url ) self.manager.itemError( url ) if onProgress: onProgress() Cache.add( url, cached ) if onLoad: onLoad( cached ) self.manager.itemEnd( url ) return cached
def GraphStocks(site): cache = Cache.Fetch("json/cache.json", "all") keys = Cache.Fetch("json/prices.json")["keys"] metal = Cache.Fetch("json/prices.json")["metal"] plt.figure(1, figsize=(16, 6.67), dpi=75) plt.clf() history = [] prices = [] i = 0 for time in sorted(cache)[-10:]: print("Result found at " + time) history.append(time) prices.append(cache[time][site]) i = i + 1 plt.ylabel('Value in refined metal') plt.title(site + " ref \nTotal: " + str(prices[i - 1]) + " refined " + locale.format("%d", prices[i - 1] / keys, grouping=True) + " keys $" + locale.format("%d", prices[i - 1] * metal, grouping=True) + " USD") x = range(i) plt.xticks(x, history) plt.plot(x, prices, "g") plt.ylim(0) print("Saved as graphs/" + site + ".svg") plt.savefig("graphs/" + site + ".svg")
def on_radio_button(self, event): choice = event.GetEventObject() if choice.GetLabel() == "Use default path": self.current_setting["Default path mode"] = True else: self.current_setting["Default path mode"] = False Cache.store(self.current_setting, self.config_path)
def test_different_n(self): # lru sa_cache_2way = Cache.Cache(2, 16) sa_cache_4way = Cache.Cache(4, 16) data = [(x, x**2) for x in range(100)] for i in range(100): sa_cache_2way.put(data[i][0], data[i][1]) for i in range(100): sa_cache_4way.put(data[i][0], data[i][1]) self.assertIn( 2, [len(sa_cache_2way._cache[set_num]) for set_num in range(8)]) self.assertIn( 4, [len(sa_cache_4way._cache[set_num]) for set_num in range(4)]) # mru sa_cache_3way = Cache.Cache(3, 15, "mru") sa_cache_5way = Cache.Cache(5, 15, "mru") data = [(x, x**2) for x in range(100)] for i in range(100): sa_cache_3way.put(data[i][0], data[i][1]) for i in range(100): sa_cache_5way.put(data[i][0], data[i][1]) self.assertIn( 3, [len(sa_cache_3way._cache[set_num]) for set_num in range(5)]) self.assertIn( 5, [len(sa_cache_5way._cache[set_num]) for set_num in range(3)])
def setCache(self, S, E, B, m): self.mem.extend([0] * ((1 >> m) - self.size)) # self.cache.S = S # self.cache.E = E # self.cache.B = B # self.cache.m = m self.cache = Cache(S, E, B, m)
def store(key, value): key = getKeyPayload(key) Cache.update(key, value) if memory_dictionary[key] == None: raise Exception('Location in memory is not defined: {}'.format(key)) memory_dictionary[key] = value
def _getInfo(hashFileName, httpCall, *httpParas): result = Cache.getCache(hashFileName) if result == '': result = httpCall(*httpParas) Cache.writeCache(hashFileName, result) return result
def __init__(self): self.methods=['GET','POST','HEAD','PUT','DELETE'] self.imagesSupported=['jpg','png','gif'] self.mimeTypes = {'jpg': 'image/jpeg', 'png': 'image/png', 'gif': 'image/gif', 'html': 'text/html','mp3':'audio/mpeg','mp4':'video/mp4','application/x-www-form-urlencoded':'application/json'} self.status = {'404': 'HTTP/1.1 404 Not Found', '400': 'HTTP/1.1 400 Bad Request', '403': 'HTTP/1.1 403 Forbidden','200':'HTTP/1.1 200 OK'} self.cache=Cache() self.authenticator=Authenticator() self.timeout=10
def getRadar(self, code): self._radar = Cache.read(BOM.modcachekey + '-radar(' + code + ')') if self._radar is None: img_file = urllib2.urlopen(self._radarbaseurl + code + '.gif' ) self._radar = img_file.read() Cache.write(BOM.modcachekey + '-radar(' + code + ')', self._radar, 360) return Image.open(StringIO(self._radar))
def getFeed(self, feed_url ): if not self.feed_url and self.provider.isDuplicateFeed( feed_url ): self.dead = True self.changed() else: self.feed_url = feed_url # if we have a feed object, then I'm not interested in re-parsing a stale file. wantStale = not self.feed Cache.getContentOfUrlAndCallback( callback = self.gotFeed, url = feed_url, username = self.username(), password = self.password(), timeout = self.timeout(), wantStale = wantStale, failure = self.failed )
def __init__(self, blockno, wordsinblock, fileloc): mem = [] with open(fileloc) as f: lines = f.readlines() for line in lines: token = string.split(string.lower(line)) if token[0] != 'go': mem.append((int(token[1]), int(token[0], 0))) self.codeCache = Cache.Cache(mem, blockno, wordsinblock) self.dataCache = Cache.Cache(mem, blockno, wordsinblock)
def on_change_path(self, event): dlg = wx.DirDialog(self, "Choose the default folder for past paper") if dlg.ShowModal() == wx.ID_OK: folder_directory = dlg.GetPath() self.current_setting["Default path"] = folder_directory self.default_directory.SetLabel(folder_directory) Cache.store(self.current_setting, self.config_path) dlg.Destroy() else: dlg.Destroy() return
def __init__(self, parent): wx.Panel.__init__(self, parent) self.config_path = Cache.preference_directory() self.current_setting = Cache.load(self.config_path) level_txt = wx.StaticText(self, label="Default level:") self.level_choice = wx.Choice(self, choices=["--- Select level ---", "IGCSE", "AS & A-Level", "O-Level"]) self.level_choice.SetSelection(self.current_setting["Default level"]) self.Bind(wx.EVT_CHOICE, self.on_choose_level, self.level_choice) level_sizer = wx.BoxSizer(wx.HORIZONTAL) level_sizer.Add(level_txt, flag=wx.RIGHT, border=5) level_sizer.Add(self.level_choice) split_after_level = wx.StaticLine(self, style=wx.LI_HORIZONTAL) download_txt = wx.StaticText(self, label="Download path:") ask_radio_button = wx.RadioButton(self, label="Ask every time") path_radio_button = wx.RadioButton(self, label="Use default path") if self.current_setting["Default path mode"]: path_radio_button.SetValue(True) else: ask_radio_button.SetValue(True) self.Bind(wx.EVT_RADIOBUTTON, self.on_radio_button) change_button = wx.Button(self, label="change", size=(65, -1)) self.Bind(wx.EVT_BUTTON, self.on_change_path, change_button) set_path_sizer = wx.BoxSizer(wx.HORIZONTAL) set_path_sizer.Add(path_radio_button, flag=wx.ALIGN_CENTER_VERTICAL) set_path_sizer.Add(change_button, flag=wx.LEFT | wx.ALIGN_CENTER_VERTICAL, border=5) self.default_directory = wx.StaticText(self, label=self.current_setting["Default path"]) split_after_download = wx.StaticLine(self, style=wx.LI_HORIZONTAL) border = 10 general_sizer = wx.BoxSizer(wx.VERTICAL) general_sizer.Add(level_sizer, flag=wx.LEFT | wx.TOP, border=border) general_sizer.AddSpacer(6) general_sizer.Add(split_after_level, flag=wx.EXPAND | wx.RIGHT | wx.LEFT, border=border) general_sizer.AddSpacer(2) general_sizer.Add(download_txt, flag=wx.LEFT, border=border) general_sizer.AddSpacer(5) general_sizer.Add(ask_radio_button, flag=wx.LEFT, border=border) general_sizer.AddSpacer(3) general_sizer.Add(set_path_sizer, flag=wx.LEFT, border=border) general_sizer.AddSpacer(2) general_sizer.Add(self.default_directory, flag=wx.LEFT, border=25) general_sizer.AddSpacer(6) general_sizer.Add(split_after_download, flag=wx.EXPAND | wx.RIGHT | wx.LEFT, border=border) self.SetSizer(general_sizer)
def __init__(self, provider, url): ProviderAtom.__init__( self, provider, url ) self.username = re.search(r'/traveller/([^/]+)', self.url).group(1) self.name = "Dopplr / %s"%self.username self.response = None self.fail = None self.token = NSUserDefaults.standardUserDefaults().stringForKey_("dopplrToken") if not self.token: return url = "https://www.dopplr.com/api/traveller_info.js?token=%s&traveller=%s"%( self.token, self.username ) Cache.getContentOfUrlAndCallback( callback = self.gotDopplrData, url = url, timeout = 3600, wantStale = True, failure = self.failed )
def getSocialGraphFor( self, url, more_urls = [] ): if not re.match(r'http', url): return if url in Extractor.SOCIAL_GRAPH_CACHE: print_info("using cached social graph data") self.addClues( Extractor.SOCIAL_GRAPH_CACHE[url], more_urls ) return api = "http://socialgraph.apis.google.com/lookup?pretty=1&fme=1&edo=1&edi=1" api += "&q=" + quote( url, '' ) print_info("Social graph API call to " + api ) # TODO _ respect more_urls here Cache.getContentOfUrlAndCallback( callback = self.gotSocialGraphData, url = api, timeout = 3600 * 48 ) # huge timeout here
def getParsedRenderedSite(url): site = Cache.loadRenderedSiteFromCache( ) #check if fully rendered page already downloaded if site: return BeautifulSoup(site, 'html.parser') #parse if already exists else: driver = webdriver.Chrome( './pokemon_scraper/chromedriver' ) #selenium webdriver automates web interaction, must have chrome installed driver.get(url) #run website and render content Cache.writeRenderedSiteToCache(driver.page_source) return BeautifulSoup( driver.page_source, 'html.parser' ) #take rendered HTML doc and parse with beautiful soup
def main(): if len(sys.argv) == 4: file = open(sys.argv[1]) if file: if sys.argv[2].lower() == "lru": try: cacheSize = int(sys.argv[3]) cache = Cache(cacheSize) cache.politicaLRU(file) except ValueError: print "Favor ingresar un numero en el parametro cache size" elif sys.argv[2].lower() == "clock": try: cacheSize = int(sys.argv[3]) cache = Cache(cacheSize) cache.politicaClock(file) except ValueError: print "Favor ingresar un numero en el parametro cache size" else: print "Lo ingresado en el parametro <politica> es incorrecto" else: print "Archivo ingresado en parametro <file> no encontrado" else: print "Cantidad de parametros incorrectos favor ingresar de la siguiente forma" print "python cacheSimulator.py <file> <politica> <cacheSize>"
def main(): cache = Cache() print("ESCREVA O COMANDO SEGUINDO O FORMATO:\nread endereco\nwrite endereco valor\nshow\nsair\n") while True: entrada = list(map(str, input().split())) command = entrada[0] if command.casefold() == 'read': cache.read(int(entrada[1])) if command.casefold() == 'write': try: cache.write(int(entrada[1]), int(entrada[2])) except IndexError: print("VALOR NÃO INFORMADO") continue if command.casefold() == 'show': cache.show() if command.casefold() == 'exit' or command.casefold() == 'sair': return else: print("Comando não encontrado")
def emTraining(sentences, sTest): print 'Beginning EM training...' globalStart = time.time() stTable = initStTable(sentences) epsilon = estimateEpsilon(sentences) if model is 2: alignCj, alignC = initAligns(sentences) unifAlignP = 1.0 / len(alignCj) #alignProbs = {align:unifAlignP for align in alignCj} alignProbs = dict(zip(alignCj, [unifAlignP] * len(alignCj))) else: alignProbs = None likelihoodCache = Cache.Cache(runType + '.likelihood', []) i = 0 while i < iterations: print "Iteration " + str(i) start = time.time() stCache = Cache.Cache('stTable.' + runType + '.iter' + str(i), [], True) if not stCache.cache: if model is 1: counts = collectCounts(sentences, stTable) stTable = translationTable(counts) else: counts, alignCj, alignC = collectCounts( sentences, stTable, alignProbs) stTable = translationTable(counts) alignProbs = alignments(alignCj, alignC) stCache.cache = stTable if i is 0 or (i + 1) % 5 is 0: if not evaluate: stCache.save() #else: # stTable = stCache.cache viterbiFile = 'Output/' + runType + '.viterbi.iter' + str(i) outputViterbi(sTest, stTable, viterbiFile, alignProbs) likelihood = logLikelihood(sentences, stTable, epsilon, alignProbs) likelihoodCache.cache.append(likelihood) if not evaluate: likelihoodCache.save() i += 1 if evaluate: outputViterbi(sentences, stTable, 'alignments.out', alignProbs) print "EMtraining finished after", iterations, "iterations in", getDuration( globalStart, time.time()), "."
def main(): instruction_cache = cache.Cache(16, 1, 1024) data_cache = cache.Cache(16, 8, 256) trace_addresses = cacheFunctions.read_trace_from_file() print("Starting timer now") start = time() cacheFunctions.analyse_the_trace(trace_addresses, instruction_cache, data_cache) end = time() print("Timer ended..") time_taken = end - start print_final_results(time_taken, instruction_cache, data_cache)
def callComponent(self, callProtocol, name, argDict, cache, compType, srcModTime): """ a protocol for component calls where the actual component called is the first encountered with a given base name, starting at the specified path and ascending upwards until the cascade root (by default, /). """ # parse name to find if a root is specified. i=name.find('...') if i==-1: root='/' path=name else: root=name[:i] or '/' path=name[i+3:] path=rectifyRelativeName(path) root=rectifyRelativeName(root) # this will raise an exception if not found comp=Cache._findPath(path, root) return defaultHandler.callComponent(None, comp, argDict, cache, compType, srcModTime)
def getVocabularies(sentences, sFile, tFile): start = time.time() srcVoc = Cache.Cache(sFile + '.voc', []) tarVoc = Cache.Cache(tFile + '.voc', []) if not srcVoc.cache or not tarVoc.cache: for sSnt, tSnt in sentences: for s in sSnt: if not s in srcVoc.cache: srcVoc.cache.append(s) for t in tSnt: if not t in tarVoc.cache: tarVoc.cache.append(t) srcVoc.save() tarVoc.save() print 'Vocabularies obtained in', getDuration(start, time.time()) return srcVoc.cache, tarVoc.cache
def Analyze(cache_filepath): # Load the cache. data = Cache.Load(cache_filepath) # Choose two random indices. random.seed() i1 = random.randint(0, len(data)) i2 = random.randint(0, len(data)) # Get the random data points. d1 = data[i1] d2 = data[i2] # Get the list of prime fibonacci numbers. FIBS = [ 89, 233, 1597, 28657, 514229, 433494437L, 2971215073L, 99194853094755497L ] # Try all combinations of fibonacci numbers. for fib1 in FIBS: for fib2 in FIBS: nmod1 = (d1['input'] * (fib1 + d1['name']) - d1['output']) nmod2 = (d2['input'] * (fib2 + d2['name']) - d2['output']) print(fractions.gcd(nmod1, nmod2))
def RAZ(self): self.pc = 0 self.regs = [0 for k in range(self.n_reg)] self.Cach.Mem.data = [None for k in range(self.n_mem)] S = self.Cach.S B = self.Cach.B self.Cach.Cache = [] for k in range(S): self.Cach.Cache += [C.Bloc(0, None, B)] self.pc = 0 self.instrNum = 0 self.reg1 = 0 self.reg2 = 0 self.imm = 0 self.o = 0 self.a = 0 self.n = 0 self.c_cycle = 0 self.instruction = '' self.running = 1 # voir si nécess ou non.
def LexicalAnalyze(Sentence, schema = "full"): try: logging.debug("-Start LexicalAnalyze: tokenize") Sentence = invalidchar_pattern.sub(u'\uFFFD', Sentence) if Sentence in Cache.SentenceCache: Dag = DependencyTree.DependencyTree() Dag.transform(Cache.SentenceCache[Sentence]) #return Cache.SentenceCache[Sentence], Dag, None # assume ResultWinningRules is none. ResultNodeList, Dag, ResultWinningRules = LexicalAnalyzeTask(Sentence, schema) if schema == "full" and utils.runtype != "debug" and utils.DisableDB is False: if len(Cache.SentenceCache) < utils.maxcachesize: Cache.SentenceCache[Sentence] = ResultNodeList Cache.WriteSentenceDB(Sentence, ResultNodeList) # if ParserConfig.get("main", "runtype").lower() == "debug": # t = Thread(target=Cache.WriteWinningRules_Async, args=(Sentence, ResultWinningRules)) # t.start() #Cache.WriteWinningRules(Sentence, ResultWinningRules) logging.debug("-End LexicalAnalyze") except Exception as e: logging.error("Overall Error in LexicalAnalyze({}) :".format(Sentence)) logging.error(e) logging.error(traceback.format_exc()) return None, None, None return ResultNodeList, Dag, ResultWinningRules
def test_cache_replacement_LRU(self): sa_cache = Cache.Cache(2, 8, "lru") data = [(x, x**2) for x in range(11)] collision = dict() for i in range(8, 11): collision[data[i][0]] = [] for i in range(8): sa_cache.put(data[i][0], data[i][1]) for j in range(8, 11): if sa_cache._get_set_num(data[i][0]) == sa_cache._get_set_num( data[j][0]): collision[data[j][0]].append(data[i][0]) sa_cache.put(data[8][0], data[8][1]) self.assertIsNone(sa_cache.get(collision[data[8][0]][0])) self.assertEqual(data[8][1], sa_cache.get(data[8][0])) # update timestamp by accessing through get, check if it is not replaced sa_cache.get(collision[data[9][0]][0]) sa_cache.put(data[9][0], data[9][1]) self.assertIsNotNone(sa_cache.get(collision[data[9][0]][0])) self.assertEqual(data[9][1], sa_cache.get(data[9][0])) # update timestamp by accessing through update, check if it is not replaced sa_cache.get(collision[data[10][0]][0]) sa_cache.put(data[10][0], data[10][1]) self.assertIsNotNone(sa_cache.get(collision[data[10][0]][0])) self.assertEqual(data[10][1], sa_cache.get(data[10][0]))
def _renderComponentAndCache(name, argDict, auxArgs, compType, srcModTime, cached): DEBUG(COMPONENT, "_renderComponentAndCache") try: mashed = argDict.copy() (out, cache_exp_time) = _realRenderComponent(name, argDict, auxArgs, compType, srcModTime) mashed.update(auxArgs) Cache.putCachedComponent(name, mashed, out, cache_exp_time) except: if cfg.Configuration.fallbackToCache and cached: DEBUG(COMPONENT, "execution explosion and fallback active") Error.logException() DEBUG(COMPONENT, "After logexc") return cached.out raise return out
def fetch(self, data='', page=''): if not page: page = self._page post_url = self.BASE_URL + '/gdweb/CombinationScarch.aspx' get_url = self.BASE_URL + '/gdweb/ScarchList.aspx?page='+str(page) if not data: data = self.rule.make() cache_key = 'search' + ':' + data + ':' + str(page) html = Cache.get(cache_key) if not html: r = Request() r.post(post_url, data) html = r.get(get_url) Cache.set(cache_key, html, 3*30*24*60*60) self._html = html return self
def setCache(self, S, E, B, m): self.mem.extend([0]*((1>>m)-self.size)) # self.cache.S = S # self.cache.E = E # self.cache.B = B # self.cache.m = m self.cache = Cache(S, E, B, m)
def __init__(self, *args): # Default values for some important options... self.ip_cache = {} self.cache = Cache.Cache() self.associations = Associations.Associations() self.setDefaultOpts() self.greenLight()
def callComponent(self, callProtocol, name, argDict, cache, compType, srcModTime): """ a protocol for component calls where the actual component called is the first encountered with a given base name, starting at the specified path and ascending upwards until the cascade root (by default, /). """ # parse name to find if a root is specified. i=name.find('...') if i==-1: root='/' path=name else: root=name[:i] or '/' path=name[i+3:] path=rectifyRelativeName(path) root=rectifyRelativeName(root) # this will raise an exception if not found comp=Cache._findPath(path, root) return defaultHandler.callComponent(None, comp, argDict, cache, compType, srcModTime)
def insert_raw(self, item): '''insert a new item into the cache return the information (stamp, hash) on success None otherwise item -- a file like object containing an image ''' if item is None: return None position = item.tell() item.seek(0) hash_ = Cache.get_file_hash(item) if hash_ is None: return None path = os.path.join(self.path, hash_) last_path = os.path.join(self.path, 'last') item.seek(0) handle = file(path, 'w+b', 0700) handle.write(item.read()) handle.close() shutil.copy2(path, last_path) item.seek(position) return self.__add_entry(hash_)
def insert_raw(self, item): '''insert a new item into the cache return the information (stamp, hash) on success None otherwise item -- a file like object containing an image ''' if item is None: return None position = item.tell() item.seek(0) hash_ = Cache.get_file_hash(item) if hash_ is None: return None path = os.path.join(self.path, hash_) last_path = os.path.join(self.path, 'last') item.seek(0) handle = file(path, 'w+b', 0700) handle.write(item.read()) handle.close() shutil.copy2(path, last_path) item.seek(position) return self.__add_entry(hash_)
def getFeedUrl(self): # it's very unlikely that the feed source will move # TODO - check stale cache first. Man, the feed provider is too complicated. special = self.specialCaseFeedUrl( self.url ) # return None to mean 'no special case', blank string to mean "no feed here" if special != None: if len(special) > 0: print_info("special-case feed url %s"%special) self.getFeed( special ) else: # bad feed self.dead = True self.changed() return Cache.getContentOfUrlAndCallback( callback = self.gotMainPage, url = self.url, timeout = self.timeout() * 10, wantStale = False, failure = self.failed ) # TODO - use stale version somehow
def __init__(self, parent): wx.Panel.__init__(self, parent) explain_txt = wx.StaticText(self, label="Past Paper Crawler caches viewed web pages to memory \nand disk to boost efficiency.") hint_txt = wx.StaticText(self, label="Current cache on the disk: ") open_button = wx.Button(self, label="open folder") self.Bind(wx.EVT_BUTTON, self.on_open, open_button) self.cache_folder = Cache.customized_directory() cache_list = sorted([file for file in os.listdir(self.cache_folder) if not file.startswith(".")]) self.cache_checklist = wx.CheckListBox(self, choices=cache_list, size=(0, 295)) open_cache_sizer = wx.BoxSizer(wx.HORIZONTAL) open_cache_sizer.Add(hint_txt, flag=wx.ALIGN_CENTER_VERTICAL | wx.RIGHT, border=5) open_cache_sizer.Add(open_button, flag=wx.ALIGN_CENTER_VERTICAL) select_all_button = wx.Button(self, label="Select all") self.Bind(wx.EVT_BUTTON, self.on_select_all, select_all_button) remove_button = wx.Button(self, label="Remove") self.Bind(wx.EVT_BUTTON, self.on_remove, remove_button) button_sizer = wx.BoxSizer(wx.HORIZONTAL) button_sizer.Add(select_all_button) button_sizer.Add(remove_button, flag=wx.LEFT, border=8) cache_sizer = wx.BoxSizer(wx.VERTICAL) cache_sizer.Add(explain_txt, flag=wx.ALL, border=10) cache_sizer.Add(open_cache_sizer, flag=wx.BOTTOM | wx.LEFT | wx.RIGHT, border=10) cache_sizer.Add(self.cache_checklist, flag=wx.EXPAND | wx.BOTTOM | wx.LEFT | wx.RIGHT, border=10) cache_sizer.Add(button_sizer, flag=wx.ALIGN_RIGHT | wx.BOTTOM | wx.LEFT | wx.RIGHT, border=10) self.SetSizer(cache_sizer)
def test_put(self): sa_cache = Cache.Cache(2, 8) data = (3, 5) sa_cache.put(data[0], data[1]) tag = sa_cache._custom_hash( tuple([ sa_cache._custom_hash(data[0]), sa_cache._custom_hash(data[1]) ])) self.assertIn( [tag, data[1]], [[entry[0], entry[1]] for entry in sa_cache._cache[sa_cache._get_set_num(data[0])]]) data2 = (-62, 172) sa_cache.put(data2[0], data2[1]) tag2 = sa_cache._custom_hash( tuple([ sa_cache._custom_hash(data2[0]), sa_cache._custom_hash(data2[1]) ])) self.assertIn( [tag, data[1]], [[entry[0], entry[1]] for entry in sa_cache._cache[sa_cache._get_set_num(data[0])]]) self.assertIn( [tag2, data2[1]], [[entry[0], entry[1]] for entry in sa_cache._cache[sa_cache._get_set_num(data2[0])]])
def showRoom(year, month, date, room): if (not re.match('^[a-zA-Z]{1,2}\\d+[a-zA-Z]?$', room)): return jsonify(errMsg='Invalid room'), 500 try: scheduleList = Cache.readCache(year, month, date) except IOError: html = Fetcher.fetch_html(int(year), int(month), int(date), int("0830"), int("2200")) scheduleList = Fetcher.parseHTML2List(html) Cache.saveCache(year, month, date, scheduleList) response = Scheduler.selectRoom(scheduleList, room) res = jsonify(response) res.mimetype = 'application/json' res.headers['Access-Control-Allow-Origin'] = '*' return res
def __init__(self): ##https://developers.facebook.com/tools/explorer/ user_token = "CAACEdEose0cBAO9m1ZChz3qqjqcS3HBlerOSd3wteZC7EqQYlcbngZCmQtvhKQmMmE1sORZAzZA07PboTUXgskIcZBFKeA05FpASH2hEoZCW4im9ZCuNLAlgnOHc00YM5tZByxLCZBo33JkGmq4aPFDNil7FnJGCI7dxgJVwd1ZApkurkCB8WGdZBSJTdNc6zFoCUXFHAKZCZC8OAF6RSV0ljZAw9TA" extended_access_token = "CAABfbgndG3ABAOGqO5oQ1HwqVOYrlZB6CofIOciVPgMFC4zIRRk7wJvjrZBTIpFlJ3eTZA72fs4UKmyPgasMZA6MEtAaCSegvAju2zsUXAgaTRCxAfFjwrh9x8ZBLJ4lRlEHVWg0m6ZAWk9mMWdYpAVc27cIZCwi5IXO4t0U2fWkVzbAn0UrWJS" self.graph = GraphAPI(extended_access_token) self.fb_cache = Cache(FACEBOOK_CACHE,INPUT_LANGUAGE) self.fb_cache.load() self.sleep_count = 0
def _rectifyRelativeName( name ): #if no previous components, or it's absolute, just return it as is if topOfComponentStack == -1: return '/' + name elif name[0] == '/': return name path, fname = os.path.split( componentStack[topOfComponentStack].name ) return Cache._normpath("%s/%s" % (path, name))
class DuckDuckGoWordOccurrenceClassifier: def __init__(self): self.duckduckgo_search = DuckduckgoSearch(False) self.word_occurrence_classifier = WordOccurrenceClassifier() self.cache = Cache(DUCKDUCK_WORD_OCCURRENCE_CACHE, INPUT_LANGUAGE) self.cache.load() def classify(self, term): cache_result = self.cache.search_cache(term) if cache_result is not None: return self.word_occurrence_classifier.normalize_results(ClassificationResult(term, cache_result.Matches)) try: search_result = self.duckduckgo_search.general_search(term) except Exception, e: return ClassificationResult(term, {key: -1 for key in categories}) result = self.word_occurrence_classifier.calculate_score(term, search_result) self.cache.update_cache(term, result) return self.word_occurrence_classifier.normalize_results(result)
def main(): if len(sys.argv) == 4: file = open(sys.argv[1]) if file: if sys.argv[2].lower() == "lru": try: cacheSize = int(sys.argv[3]) cache = Cache(cacheSize) cache.politicaLRU(file) except ValueError: print "Favor ingresar un numero en el parametro cache size" elif sys.argv[2].lower() == "clock": try: cacheSize = int(sys.argv[3]) cache = Cache(cacheSize) cache.politicaClock(file) except ValueError: print "Favor ingresar un numero en el parametro cache size" else: print "Lo ingresado en el parametro <politica> es incorrecto" else: print "Archivo ingresado en parametro <file> no encontrado" else: print "Cantidad de parametros incorrectos favor ingresar de la siguiente forma" print "python cacheSimulator.py <file> <politica> <cacheSize>"
def getCurrentMonth(username, password): # Try to return from cache cachekey = 'metricsathome.Data.iiNet-(' + iiNetCM + ',' + username + ')' iinetusage = Cache.read(cachekey) if iinetusage is not None: return iinetusage # Fall back to the net url = iiNetCM + '&username='******'&password='******'error')): raise Exception('iiNet API Error: ' + xmlDoc.getElementsByTagName('error')[0].firstChild.nodeValue) result = {} processQuotaReset(xmlDoc.getElementsByTagName('quota_reset')[0], result) processExpectedTraffic(xmlDoc.getElementsByTagName('expected_traffic_types')[0], result) processDailyUsage(xmlDoc.getElementsByTagName('day_hour'), result) # Save in cache Cache.write(cachekey, result, 7200) return result
class CompanyDuckDuckClassifier: def __init__(self): self.duckduckgo_search = DuckduckgoSearch(True) self.company_postfix = ['corp', 'corporation', 'company', 'inc', 'headquarters'] self.cache = Cache(DUCKDUCK_COMPANY_CACHE, INPUT_LANGUAGE) self.min_hits_to_match = DUCKDUCK_COMPANY_MIN_HITS_TO_MATCH self.cache.load() def classify(self, term): cache_result = self.cache.search_cache(term) if self.cache.search_cache(term) is not None: return self.normalize_results(ClassificationResult(term, cache_result.Matches)) result = ClassificationResult(term) for company_word in self.company_postfix: term_to_search = term + ' ' + company_word try: if self.duckduckgo_search.general_search(term_to_search) != '': result.Matches['company'] += 1 except Exception, e: return ClassificationResult(term, {key: -1 for key in categories}) self.cache.update_cache(term, result) return self.normalize_results(result)
def getData(self, aac, xml): "Get XML codes here: http://www.bom.gov.au/info/precis_forecasts.shtml" forecasts = Cache.read(BOM.modcachekey + '-data(' + aac + ',' + xml + ')') if forecasts is not None: return forecasts wdom = parse(urllib2.urlopen(self._xmlbase + xml)) areas = wdom.getElementsByTagName('area') area = filter(lambda a: a.attributes['aac'].value == aac, areas)[0] fcasts = filter(lambda a: a.attributes != None, area.childNodes) forecasts = [] for f in fcasts: values = {} values['date'] = dateutil.parser.parse(f.attributes['end-time-local'].value) - timedelta(days=1) for e in f.getElementsByTagName('element'): values[e.attributes['type'].value] = e.firstChild.nodeValue for e in f.getElementsByTagName('text'): values[e.attributes['type'].value] = e.firstChild.nodeValue forecasts.append(values) forecasts = sorted(forecasts, key=lambda f: f['date']) Cache.write(BOM.modcachekey + '-data(' + aac + ')', forecasts, 3600) return forecasts
def get_cache(hostinfo, req_path): """ req_path is a URL path ref including query-part, the backend will determine real cache location """ # Prepare default cache location cache_location = '%s:%i/%s' % (hostinfo + (req_path,)) cache_location = cache_location.replace(':80', '') cache = Cache.load_backend_type(Params.CACHE)(cache_location) Params.log("Init cache: %s %s" % (Params.CACHE, cache), 3) Params.log('Prepped cache, position: %s' % cache.path, 2) # XXX: use unrewritten path as descriptor key, need unique descriptor per resource cache.descriptor_key = cache_location return cache
def insert(self, item): '''insert a new item into the cache return the shortcut and the hash on success None otherwise item -- a tuple containing the shortcut and the path to an image ''' shortcut, path = item hash_ = Cache.get_file_path_hash(path) if hash_ is None: return None new_path = os.path.join(self.path, hash_) shutil.copy2(path, new_path) return self.__add_entry(shortcut, hash_)
def __init__(self, **kwargs): """Initialise the service. If guid is not provided, one will be requested (returned in the callback). Pass callback= or error= to receive notification of readiness.""" self.guid = Cache.read('metricsathome.Data.YarraTrams-guid') imp = Import('http://www.w3.org/2001/XMLSchema') imp.filter.add('http://www.yarratrams.com.au/pidsservice/') doctor = ImportDoctor(imp) self.client = Client(url, doctor=doctor) if self.guid is None: self.guid = self.client.service.GetNewClientGuid() Cache.write('metricsathome.Data.YarraTrams-guid', self.guid, 2592000) # Keep for 1 month of no use headers = self.client.factory.create('PidsClientHeader') headers.ClientGuid = self.guid headers.ClientType = 'DASHBOARDWIDGET' headers.ClientVersion = '1.0' headers.ClientWebServiceVersion = '6.4.0.0' self.client.set_options(soapheaders=headers)
def insert(self, item): '''insert a new item into the cache return the information (stamp, hash) on success None otherwise item -- a path to an image ''' hash_ = Cache.get_file_path_hash(item) if hash_ is None: return None path = os.path.join(self.path, hash_) last_path = os.path.join(self.path, 'last') shutil.copy2(item, path) shutil.copy2(item, last_path) return self.__add_entry(hash_)
def init(self, path): Params.log("FileTree.init %r" % path, 5) path2 = path if Params.ARCHIVE: path2 = time.strftime( Params.ARCHIVE, time.gmtime() ) + path2 path2 = os.path.join(Params.ROOT, path2) if len(path2) >= Params.MAX_PATH_LENGTH: sep = Cache.min_pos(path2.find('#'), path2.find( '?' )) if sep != -1: if (len(path2[:sep])+34) < Params.MAX_PATH_LENGTH: FileTreeQH.init(self, path) else: PartialMD5Tree.init(self, path) else: FileTreeQ.init(self, path)
def buildOverlay(self, code, layers=deflayers): cachekey = BOM.modcachekey + '-overlay(' + code + ',(' + ','.join(layers) + '))' background = Cache.read(cachekey) if background is not None: return Image.open(StringIO(background)) im = Image.new('RGBA', (512, 564), (255, 255, 255, 0)) ftp = FTP(self._ftphost) ftp.login() ftp.cwd(self._transparentcypath) files = ftp.nlst('IDR' + code + '.*.png') files += ftp.nlst('IDR.*.png') alllayers = {} for f in files: b = f.find('.') + 1 e = f.rfind('.') alllayers[f[b:e]] = f for l in layers: imgout = StringIO() ftp.retrbinary('RETR ' + alllayers[l], imgout.write) imgin = StringIO(imgout.getvalue()) img = Image.open(imgin).convert('RGBA') im.paste(img, (0,0), img) imgout.close() imgin.close() imgout = StringIO() im.save(imgout, format='PNG') Cache.write(cachekey, imgout.getvalue(), 604800) # cache for a week imgout.close() return im
def __init__(self, path, text="", size=32, shadow=True): self.font = pygame.font.Font(os.path.join("..", "data", "fonts", path), int(size)) self.texture = Texture() self.cache = Cache() # attributes self.scale = (1.0, 1.0) # image bounds (width, height) self.position = (0, 0) # where in the window it should render self.angle = 0 # angle which the image is drawn self.color = (255, 255, 255, 255) # colour of the image self.rect = (0.0, 0.0, 1.0, 1.0) # left, top, right, bottom, crops the texture self.alignment = 1 # alignment of the text (left, center , right) self.shadow = True # does the font project a shadow self.text = None self.setText(text) # it is not necessary to enter a string upon initialization,
def insert_raw(self, item): '''insert a new item into the cache return the information (stamp, hash) on success None otherwise item -- a tuple containing the shortcut and a file like object with the image ''' shortcut, image = item position = image.tell() image.seek(0) hash_ = Cache.get_file_hash(image) if hash_ is None: return None path = os.path.join(self.path, hash_) self.create_file(path, image) image.seek(position) return self.__add_entry(shortcut, hash_)
def insert_resized(self, item, filename, height = 50, width = 50): '''insert a new item into the cache with the specified filename resizing it if posible. return the shortcut and the hash on success None otherwise item -- a tuple containing the shortcut and the path to an image ''' shortcut, image = item path = os.path.join(tempfile.gettempdir(), "emote") # save the incoming file so we can resize it using imagemagick self.create_file(path, image) self.resize_with_imagemagick(path, path, height, width) hash_ = Cache.get_file_path_hash(path) if hash_ is None: return None new_path = os.path.join(self.path, filename) shutil.copy2(path, new_path) return self.__add_entry(shortcut, filename)
def __init__(self): self.u_cache = Cache() self.u_vlc = [] ivtile = 0 for iarch in range(Architecture.getCfgByName("threadnum")): self.u_vlc.append(VLC()) self.u_vlc[iarch].initialize() self.u_PCTracer = [] for iarch in range(Architecture.getCfgByName("threadnum")): self.u_PCTracer.append(PCTracer(iarch, ivtile)) self.u_PCTracer[iarch].initialize(GlobalVar.allcontents_trc[int(Architecture.getCfgByName("threadlist")[iarch])]) self.u_cache.initialize() self.PCTracer_nextState = [] self.PCTracer_state = [] self.PCTracer_counter = [] self.VLC_nextState = [] self.VLC_state = [] # self.VLC_counter = [] self.current_CYC = [] self.cur_abstime = [] self.cur_clock = [] self.cur_PC = [] for iarch in range(Architecture.getCfgByName("threadnum")): self.PCTracer_nextState.append(0) self.PCTracer_state.append(0) self.PCTracer_counter.append(0) self.VLC_nextState.append(0) self.VLC_state.append(0) # self.VLC_counter.append(0) self.current_CYC.append(0) self.cur_abstime.append(0) self.cur_clock.append(0) self.cur_PC.append(0)
def getRadarLoop(self, code): rdrimgs = [] cachekey = BOM.modcachekey + '-radarloop(' + code + ')' rdrloopstr = Cache.read(cachekey) if rdrloopstr is not None: for s in rdrloopstr: rdrimgs.append(Image.open(StringIO(s)).convert('RGBA')) return rdrimgs background = self.buildOverlay(code, BOM.backlayers) foreground = self.buildOverlay(code, BOM.forelayers) ftp = FTP(self._ftphost) ftp.login() ftp.cwd(self._radarpath) files = ftp.nlst('IDR' + code + '.T.*.png') for f in files: ftpcachekey = BOM.modcachekey + '-ftp://' + self._ftphost + self._radarpath + f imgstr = Cache.read(ftpcachekey) imageio = None if imgstr is not None: imageio = StringIO(imgstr) else: imgget = StringIO() ftp.retrbinary('RETR ' + f, imgget.write) Cache.write(ftpcachekey, imgget.getvalue(), 10800) imageio = StringIO(imgget.getvalue()) imgget.close() try: rdrtrans = Image.open(imageio).convert('RGBA') im = Image.new('RGBA', (512, 564), (255, 255, 255, 0)) im.paste(background, (0, 0), background) im.paste(rdrtrans, (0, 0), rdrtrans) im.paste(foreground, (0, 0), foreground) rdrimgs.append(im) except IOError: print 'WARNING: BOM is including bad images again'; rdrloopstr = [] for i in rdrimgs: imgout = StringIO() i.save(imgout, format='PNG') rdrloopstr.append(imgout.getvalue()) Cache.write(cachekey, rdrloopstr, 360) return rdrimgs
def init(self, path): Params.log("FileTreeQ.init %r" % path, 5) psep = Params.ENCODE_PATHSEP # encode query and/or fragment parts sep = Cache.min_pos(path.find('#'), path.find( '?' )) # sort query vals and turn into dirs if sep != -1: if psep: path = path[:sep] + path[sep:].replace('%2F', psep) path = path[:sep] + path[sep:].replace('/', psep) if '&' in path[sep:]: parts = path[sep+1:].split('&') elif ';' in path[sep:]: parts = path[sep+1:].split(';') else: parts = [path[sep+1:]] if Params.FileTreeQ_SORT: parts.sort() while '' in parts: parts.remove('') path = path[ :sep+1 ] if parts: path = path + '/' + '/'.join(parts) # optional removal of directories in path if psep: if sep == -1 or Params.FileTreeQ_ENCODE: # entire path path = path.replace( '/', psep) else: # URL path-part only path = path[:sep].replace( '/', psep) + path[sep:] # make archive path if Params.ARCHIVE: path = time.strftime( Params.ARCHIVE, time.gmtime() ) + path self.path = os.path.join(Params.ROOT, path) self.file = None