def __init__(self, *args, **kwargs): super(GMusicLibraryProvider, self).__init__(*args, **kwargs) # tracks, albums, and artists here refer to what is explicitly # in our library. self.tracks = {} self.albums = {} self.artists = {} # aa_* caches are *only* used for temporary objects. Library # objects will never make it here. self.aa_artists = LRUCache(1024) self.aa_tracks = LRUCache(1024) self.aa_albums = LRUCache(1024) self._radio_stations_in_browse = ( self.backend.config['gmusic']['radio_stations_in_browse']) self._radio_stations_count = ( self.backend.config['gmusic']['radio_stations_count']) self._radio_tracks_count = ( self.backend.config['gmusic']['radio_tracks_count']) self._top_tracks_count = ( self.backend.config['gmusic']['top_tracks_count']) # Setup the root of library browsing. self._root = [ Ref.directory(uri='gmusic:album', name='Albums'), Ref.directory(uri='gmusic:artist', name='Artists'), Ref.directory(uri='gmusic:track', name='Tracks') ] if self._radio_stations_in_browse: self._root.append(Ref.directory(uri='gmusic:radio', name='Radios'))
class CoapLRUCache(CoapCache): def __init__(self, max_dim): """ :param max_dim: """ self.cache = LRUCache(maxsize=max_dim) def update(self, key, element): """ :param key: :param element: :return: """ self.cache.update([(key.hashkey, element)]) def get(self, key): """ :param key: :return: CacheElement """ try: response = self.cache[key.hashkey] except KeyError: print "problem here" response = None return response def is_full(self): """ :return: """ if self.cache.currsize == self.cache.maxsize: return True return False def is_empty(self): """ :return: """ if self.cache.currsize == 0: return True return False def debug_print(self): """ :return: """ print "size = ", self.cache.currsize
def __init__(self, token, name): self.name = name self.token = token self.agent = Agent(reactor) self.last_update_id = -1 self.message_handlers = [] self.message_subscribers = LRUCache(maxsize=10000) self.message_prehandlers = [] self.message_next_handlers = LRUCache(maxsize=1000) self.retry_update = 0 self.running = False self.inline_query_handler = None self.callback_query_handler = None self.chosen_inline_result_handler = None self.botan = None
def test_lru(self): cached = Cached(LRUCache(maxsize=2)) self.assertEqual(cached.get(0), 0) self.assertEqual(cached.get(1), 1) self.assertEqual(cached.get(1), 1) self.assertEqual(cached.get(1.0), 1) self.assertEqual(cached.get(1.0), 1) cached.cache.clear() self.assertEqual(cached.get(1), 2)
def __init__(self, capacity, cache_policy=None, ttl_policy=None, *args, **kwargs): self.cache_policy = cache_policy or self.DEFAULT_CACHE_POLICY self.ttl_policy = ttl_policy or self.DEFAULT_TTL_POLICY self.cache = LRUCache(maxsize=capacity, missing=self.missing) self.args = args self.kwargs = kwargs
def __init__(self, shape, dtype=np.float): dtype = np.dtype(dtype) super().__init__() self.current_list_size = 0 self.shape = shape self.dtype = dtype self.array_byte_size = reduce(lambda x, y: x * y, shape) * dtype.itemsize self.open() self.cache = LRUCache(maxsize=128) self.misses = 0
class MISPFileCache(object): # cache up to 150 JSON structures in class attribute @staticmethod @cached(cache=LRUCache(maxsize=150)) def _load_json(path): if not os.path.exists(path): return None with open(path, 'r') as f: data = load(f) return data
def test_typed_lru(self): cached = Cached(LRUCache(maxsize=2)) self.assertEqual(cached.cache, cached.get_typed.cache(cached)) self.assertEqual(cached.get_typed(0), 0) self.assertEqual(cached.get_typed(1), 1) self.assertEqual(cached.get_typed(1), 1) self.assertEqual(cached.get_typed(1.0), 2) self.assertEqual(cached.get_typed(1.0), 2) self.assertEqual(cached.get_typed(0.0), 3) self.assertEqual(cached.get_typed(0), 4)
class WxWork(object): def __init__(self, url): self.host = settings.get('WX_WORK_API_BASE_URL') self.url = f"{self.host}{url}" async def return_result(self, result): return result @cached(cache=LRUCache(maxsize=1024)) async def get(self, *args, **kwargs): url = self.url payload = "{}" headers = { 'Content-Type': "application/json", 'cache-control': "no-cache" } response = requests.request("GET", url, data=payload, headers=headers, params=kwargs) result = await self.return_result(response) return result @cached(cache=LRUCache(maxsize=1024)) async def post(self, *args, **kwargs): url = self.url payload = "{}" headers = { 'Content-Type': "application/json", 'cache-control': "no-cache" } response = requests.request("POST", url, data=payload, headers=headers, params=kwargs) result = await self.return_result(response) return result
class DemixFile(DataFile): """A DataFile which contains methods for accessing and loading demixed traces. """ def __init__(self, filepath: Union[str, Path]): super().__init__(filepath=filepath) @classmethod @cached(cache=LRUCache(maxsize=10), key=from_json_cache_key) def from_json(cls, dict_repr: dict) -> "DemixFile": filepath = dict_repr["demix_file"] return cls(filepath=filepath) def to_json(self) -> Dict[str, str]: return {"demix_file": str(self.filepath)} @classmethod @cached(cache=LRUCache(maxsize=10), key=from_lims_cache_key) def from_lims(cls, db: PostgresQueryMixin, ophys_experiment_id: Union[int, str]) -> "DemixFile": query = """ SELECT wkf.storage_directory || wkf.filename AS demix_file FROM ophys_experiments oe JOIN well_known_files wkf ON wkf.attachable_id = oe.id JOIN well_known_file_types wkft ON wkft.id = wkf.well_known_file_type_id WHERE wkf.attachable_type = 'OphysExperiment' AND wkft.name = 'DemixedTracesFile' AND oe.id = {}; """.format(ophys_experiment_id) filepath = db.fetchone(query, strict=True) return cls(filepath=filepath) @staticmethod def load_data(filepath: Union[str, Path]) -> pd.DataFrame: with h5py.File(filepath, 'r') as in_file: traces = in_file['data'][()] roi_id = in_file['roi_names'][()] idx = pd.Index(roi_id, name='cell_roi_id', dtype=int) return pd.DataFrame({'corrected_fluorescence': list(traces)}, index=idx)
def __init__(self): super(MetricCassandraRepository, self).__init__() self._lock = threading.RLock() LOG.debug("prepare cql statements...") self._measurement_insert_stmt = self._session.prepare(MEASUREMENT_INSERT_CQL) self._measurement_insert_stmt.is_idempotent = True self._measurement_update_stmt = self._session.prepare(MEASUREMENT_UPDATE_CQL) self._measurement_update_stmt.is_idempotent = True self._metric_insert_stmt = self._session.prepare(METRICS_INSERT_CQL) self._metric_insert_stmt.is_idempotent = True self._metric_update_stmt = self._session.prepare(METRICS_UPDATE_CQL) self._metric_update_stmt.is_idempotent = True self._dimension_stmt = self._session.prepare(DIMENSION_INSERT_CQL) self._dimension_stmt.is_idempotent = True self._dimension_metric_stmt = self._session.prepare(DIMENSION_METRIC_INSERT_CQL) self._dimension_metric_stmt.is_idempotent = True self._metric_dimension_stmt = self._session.prepare(METRIC_DIMENSION_INSERT_CQL) self._metric_dimension_stmt.is_idempotent = True self._retrieve_metric_dimension_stmt = self._session.prepare(RETRIEVE_METRIC_DIMENSION_CQL) self._metric_batch = MetricBatch( self._cluster.metadata, self._cluster.load_balancing_policy, self._max_batches) self._metric_id_cache = LRUCache(self._cache_size) self._dimension_cache = LRUCache(self._cache_size) self._metric_dimension_cache = LRUCache(self._cache_size) self._load_dimension_cache() self._load_metric_dimension_cache()
class Database(object): def __init__(self): self.db_user = settings.get('DB_USER') self.db_pass = settings.get('DB_PASS') self.db_name = settings.get('DB_NAME') self.db_host = settings.get('DB_HOST') self.db_port = settings.get('DB_PORT') def get_db_host(self): return self.db_host def get_db_user(self): return self.db_user def get_db_pass(self): return self.db_pass def get_db_name(self): return self.db_name @cached(cache=LRUCache(maxsize=1024)) async def execute(self, sql): conn = await asyncpg.connect(user=self.db_user, password=self.db_pass, database=self.db_name, host=self.db_host, port=self.db_port) values = await conn.fetch(sql) await conn.close() return values @cached(cache=LRUCache(maxsize=1024)) async def fetchrow(self, sql): conn = await asyncpg.connect(user=self.db_user, password=self.db_pass, database=self.db_name, host=self.db_host, port=self.db_port) values = await conn.fetchrow(sql) await conn.close() return values
def __init__(self, trading_calendar, reader, adjustment_reader, sid_cache_size=1000): self.trading_calendar = trading_calendar self._reader = reader self._adjustments_reader = adjustment_reader self._window_blocks = { field: ExpiringCache(LRUCache(maxsize=sid_cache_size)) for field in self.FIELDS }
def load_model(model_dir, leaf_cache_size=10, points_cache_size=100000): """ Loads a model from `index_dir` and returns an instance of `HKMNearestNeighbor` :param model_dir location where model is saved :param leaf_cache_size how many leaves to keep in the cache :param points_cache_size how many individual points to keep in cache """ # load skeleton file_name = os.path.join(model_dir, 'skeleton.pickle') new_hkmnn_model = joblib.load(file_name) new_hkmnn_model.model_dir = model_dir # compute inverse index new_hkmnn_model.inverse_idx = new_hkmnn_model._get_idx_paths() # cache calls to get_vector and _get_nn_model get_nn_model_cache = LRUCache(maxsize=leaf_cache_size) get_vector_cache = LRUCache(maxsize=points_cache_size) new_hkmnn_model.get_vector = cached(get_vector_cache)(new_hkmnn_model.get_vector) new_hkmnn_model._get_nn_model = cached(get_nn_model_cache)(new_hkmnn_model._get_nn_model) return new_hkmnn_model
class ProfanityCommand(Command): extract = re.compile(r"1:.*?\s+warning\s+(.*?)\s\S+$") handlers = "!analyze", "!sjw", "!profanity" lru = LRUCache(maxsize=20) def handles(self, cmd): return True @staticmethod def alex(string): with Popen("alex -t".split(" "), stdout=PIPE, stdin=PIPE) as process: try: string = string.encode("utf-8") result = str(process.communicate(string, timeout=3)[0], "utf-8").split("\n") except TimeoutExpired: process.kill() raise result = (ProfanityCommand.extract.search(l.strip()) for l in result) result = set(m.group(1).strip() for m in result if m) return result def handle_cmd(self, cmd, remainder, msg): if cmd not in self.handlers: self.lru[msg.nick.casefold()] = (msg.nick, msg.msg) return False if not self.allowed(msg): return False try: if remainder: user = remainder.strip().casefold() user, text = self.lru.get(user, (None, None)) else: user, text = self.lru.items()[0][1] if not text: return False anal = ", ".join(self.alex(text)) if not anal: anal = "No issues found, SJW approved message" lanal = len(anal) if lanal > 220: anal = anal[0:219] + "…" lanal = len(anal) lrem = 295 - lanal quote = ">{user}: {text}".format(user=user, text=text) if len(quote) > lrem: quote = quote[0:lrem] + "…" self.post("{}\n{}", quote, anal) return True except Exception: LOGGER.exception("failed to analyze") return False
def get_cache(cache_type, cache_size): caches = { 'lfu': LFUCache(cache_size), 'lru': LRUCache(cache_size), #'rl' : RLCache(cache_size), 'rr': RRCache(cache_size) } try: return caches[cache_type] except KeyError: return default()
def __init__(self, db, quark_chain_config, count_minor_blocks=False): # TODO: evict old blocks from memory self.db = db self.quark_chain_config = quark_chain_config self.max_num_blocks_to_recover = ( quark_chain_config.ROOT.MAX_ROOT_BLOCKS_IN_MEMORY) self.count_minor_blocks = count_minor_blocks # TODO: May store locally to save memory space (e.g., with LRU cache) self.tip_header = None self.__recover_from_db() self.rblock_cache = LRUCache(maxsize=1024)
def load_media_cache(self, server): server, _ = Servers.get_or_create(name=server) media_cache = LRUCache(maxsize=MAX_LOADED_MEDIA) for i, m in enumerate(server.media): if i > MAX_LOADED_MEDIA: break media = MediaInfo(m.mxc_server, m.mxc_path, m.key, m.iv, m.hashes) media_cache[(m.mxc_server, m.mxc_path)] = media return media_cache
class RigidMotionTransformFile(DataFile): """A DataFile which contains methods for accessing and loading rigid motion transform output. """ def __init__(self, filepath: Union[str, Path]): super().__init__(filepath=filepath) @classmethod @cached(cache=LRUCache(maxsize=10), key=from_json_cache_key) def from_json(cls, dict_repr: dict) -> "RigidMotionTransformFile": filepath = dict_repr["rigid_motion_transform_file"] return cls(filepath=filepath) def to_json(self) -> Dict[str, str]: return {"rigid_motion_transform_file": str(self.filepath)} @classmethod @cached(cache=LRUCache(maxsize=10), key=from_lims_cache_key) def from_lims( cls, db: PostgresQueryMixin, ophys_experiment_id: Union[int, str] ) -> "RigidMotionTransformFile": query = """ SELECT wkf.storage_directory || wkf.filename AS transform_file FROM ophys_experiments oe JOIN well_known_files wkf ON wkf.attachable_id = oe.id JOIN well_known_file_types wkft ON wkft.id = wkf.well_known_file_type_id WHERE wkf.attachable_type = 'OphysExperiment' AND wkft.name = 'OphysMotionXyOffsetData' AND oe.id = {}; """.format(ophys_experiment_id) filepath = safe_system_path(db.fetchone(query, strict=True)) return cls(filepath=filepath) @staticmethod def load_data(filepath: Union[str, Path]) -> pd.DataFrame: motion_correction = pd.read_csv(filepath) return motion_correction[['x', 'y']]
def __init__( self, consensus_type: ConsensusType, create_block_async_func: Callable[..., Awaitable[Optional[Block]]], add_block_async_func: Callable[[Block], Awaitable[None]], get_mining_param_func: Callable[[], Dict[str, Any]], get_header_tip_func: Callable[[], Header], remote: bool = False, root_signer_private_key: Optional[KeyAPI.PrivateKey] = None, ): """Mining will happen on a subprocess managed by this class create_block_async_func: takes no argument, returns a block (either RootBlock or MinorBlock) add_block_async_func: takes a block, add it to chain get_mining_param_func: takes no argument, returns the mining-specific params """ self.consensus_type = consensus_type self.create_block_async_func = create_block_async_func self.add_block_async_func = add_block_async_func self.get_mining_param_func = get_mining_param_func self.get_header_tip_func = get_header_tip_func self.enabled = False self.process = None self.input_q = AioQueue() # [(MiningWork, param dict)] self.output_q = AioQueue() # [MiningResult] # header hash -> block under work # max size (tx max 258 bytes, gas limit 12m) ~= ((12m / 21000) * 258) * 128 = 18mb self.work_map = LRUCache(maxsize=128) if not remote and consensus_type != ConsensusType.POW_SIMULATE: Logger.warning("Mining locally, could be slow and error-prone") # remote miner specific attributes self.remote = remote # coinbase address -> header hash # key can be None, meaning default coinbase address from local config self.current_works = LRUCache(128) self.root_signer_private_key = root_signer_private_key
def preprocess(self, current_file): """On-demand feature extraction Parameters ---------- current_file : dict Generated by a pyannote.database.Protocol Returns ------- current_file : dict Current file with additional "features" entry Notes ----- Does nothing when self.feature_extraction is a pyannote.audio.features.Precomputed instance. """ # if "features" are precomputed on disk, do nothing # as "process_segment" will load just the part we need if isinstance(self.feature_extraction, Precomputed): return current_file # if (by chance) current_file already contains "features" # do nothing. if 'features' in current_file: return current_file # if we get there, it means that we need to extract features # for current_file. let's create a cache to store them... if not hasattr(self, 'preprocessed_'): self.preprocessed_ = LRUCache(maxsize=CACHE_MAXSIZE) # this is the key that will be used to know if "features" # already exist in cache uri = get_unique_identifier(current_file) # if "features" are not cached for current file # compute and cache them... if uri not in self.preprocessed_: features = self.feature_extraction(current_file) self.preprocessed_[uri] = features # create copy of current_file to prevent "features" # from consuming increasing memory... preprocessed = dict(current_file) # add "features" key preprocessed['features'] = self.preprocessed_[uri] return preprocessed
def backend(worker_pool, **kwargs): app = Flask(__name__) app.config['SECRET_KEY'] = os.environ['SECRET_KEY'] socketio = SocketIO(app) buf = LRUCache(1024) @socketio.on('message', namespace="/upload") def on_message(data): if request.sid not in buf: buf[request.sid] = [] buf[request.sid].append(json.loads(data)) @socketio.on('reset', namespace="/upload") def on_reset(): if request.sid in buf: buf[request.sid] = [] @app.route('/anim/<path:path>') def send_anim(path): return send_from_directory('anim', path) @app.route('/dist/<path:path>') def send_dist(path): return send_from_directory('dist', path) @socketio.on('render', namespace="/upload") def on_render(): if request.sid not in buf or buf[request.sid] == []: emit("render_error", "empty_data") else: room = str(uuid4()) path = f"anim/{room}.mp4" join_room(room) try: res = worker_pool.apply_async( func=render, args=( path, buf[request.sid], ), error_callback=lambda x: socketio.emit( "render_error", "server_error", room=room ) # Outside request context -> cannot use top-level emit() ) res.get(timeout=60) emit("render_done", path), except: emit("render_error", "server_error") raise return socketio.run(app, **kwargs)
def _query( sparql, timeout, q, cache=LRUCache(maxsize=config.CACHE_SIZE), **_): """Cached low level function to perform a single SPARQL query. :param sparql: SPARQLWrapper endpoint :param timeout: a timeout in seconds. The endpoint 'timeout' parameter will be set to 3/4 this value in ms (Virtuoso seems to treat non zero timeouts < 1000ms as 1000ms), instructing the server to give us a partial result up to this soft limit. We also set a hard timeout via the socket to really cancel the request if there's no result after timeout seconds. :param q: The SPARQL Query as string :param cache: a cache object like cachetools.LRUCache or None :return: a (t, res) pair with querytime t as float and res as dict. """ assert isinstance(sparql, SPARQLWrapper.SPARQLWrapper) assert isinstance(q, six.string_types) sparql.resetQuery() sparql.setTimeout(timeout) sparql.setReturnFormat(SPARQLWrapper.JSON) # sparql.setMethod(SPARQLWrapper.POST) # sparql.setRequestMethod(SPARQLWrapper.POSTDIRECTLY) # set query timeout parameter to half the hard timeout time sparql.addParameter('timeout', str(int(timeout * 1000 * 3 / 4))) logger.debug('performing sparql query: \n%s', q) c = cache.get(q) if cache is not None else None if c is None: logger.debug('cache miss') try: q_short = ' '.join((line.strip() for line in q.split('\n'))) sparql.setQuery(q_short) c = time(sparql.queryAndConvert) except socket.timeout: c = (timeout, {}) except ValueError: # e.g. if the endpoint gives us bad JSON for some unicode chars logger.warning( 'Could not parse result for query, assuming empty result...\n' 'Query:\n%s\nException:', q, exc_info=1, # appends exception to message ) c = (timeout, {}) if cache is not None: cache[q] = c else: logger.debug('cache hit') t, res = c logger.debug('orig query took %.4f s, result:\n%s\n', t, res) return t, res
class DiscordMessage: @classmethod @CacheManager.attach_cachedmethod(self2cache=lambda x: LRUCache(maxsize=DiscordMessageCache.Constant.MAXSIZE), ) def id2message(cls, channel_id, message_id): channel = DiscordChannel.id2channel(channel_id) return channel.messages.fetch(message_id) @classmethod def add_message2cache(cls, message): channel = message.channel DiscordChannel.add_channel2cache(channel) CacheManager.add2cache(cls.id2message, message, [channel.id, message.id])
class CommitFileMapper(ArtifactMapper): def __init__(self): super().__init__(CommitFile, foreign_key="commit_id") cache = LRUCache(512) def map(self, x: DataPoint) -> Optional[CommitFile]: return CommitFile( x.filename, x.status, x.patch if not isinstance(x.patch, float) else None, x.change if not isinstance(x.change, float) else None, )
def init(self, args, db, hosts, logger): self.args = args self.db = db self.hosts = hosts self.logger = logger self.bind = args.bind self.forward = args.forward self.peers = {} self.requests = {} self.cache = LRUCache(maxsize=args.cachesize) if args.daemon: Daemon(args.pidfile).register(self) if args.debug: Debugger(events=args.verbose, logger=logger).register(self) self.transport = UDPServer(self.bind).register(self) self.protocol = DNS().register(self)
def __init__( self, logger: Logger = None, num_of_attempts: int = 1, session: Session = Session(), verify: bool = True, cachesize: int = 128, ): self.logger = get_logger( self.__class__.__name__) if logger is None else logger self.num_of_attempts = num_of_attempts self.session = session self.verify = verify self.cache = LRUCache(maxsize=cachesize)
def __init__(self, host="127.0.0.1", port=9033, model="en", embeddings_path=None, verbose=False, attributes=None): super(Client, self).__init__(model, embeddings_path) self.host = host self.port = port self.rpc = RPCClient(host, port) self.verbose = verbose self.attributes = attributes self.cache = LRUCache(maxsize=3000000)
def __init__(self, conf, glib, matrix_client, clients, db_session, account_storage, message_storage): self.glib = glib self.matrix_client = matrix_client self.clients = clients self.db_session = db_session self.account_storage = account_storage self.message_storage = message_storage self.networks = conf["networks"] self.hs_host = _parse_hs_host(conf["hs_server"]) self.users_blacklist = conf["users_blacklist"] self.users_whitelist = conf["users_whitelist"] self.accounts = defaultdict(list) self.rooms = defaultdict(Room) self.transaction_callbacks = defaultdict(list) self.clients_callbacks = defaultdict(list) self.mxids_to_ext_contacts = LRUCache(maxsize=conf["max_cache_items"]) self.ext_contacts_to_mxids = LRUCache(maxsize=conf["max_cache_items"]) self.senders_access = LRUCache(maxsize=conf["max_cache_items"]) if "user_power_level" in conf: self.user_power_level = conf["user_power_level"] else: self.user_power_level = None
def default(): processor = getattr(AnalysisProcessor._context, 'analysis_processor', None) if processor is None: logger.warning("Default analysis processor not set for thread %s, using a simple one.", threading.current_thread().name) from vot.utilities import ThreadPoolExecutor from cachetools import LRUCache executor = ThreadPoolExecutor(1) cache = LRUCache(1000) processor = AnalysisProcessor(executor, cache) AnalysisProcessor._context.analysis_processor = processor return processor
def __init__(self, parameters=None, name=''): self.name = gl.utils.nameScope.rstrip( '/') if not name else gl.utils.nameScope + name if parameters: for k, p in parameters.items(): if type(p) != gl.Parameter: # Assume this is int, float, ... parameters[k] = gl.Parameter(p, name=self.name + '/' + k) self.parameters = parameters self.cache = LRUCache(maxsize=128)
def __init__(self, token, name, skip_offset=False, allowed_updates=None, agent=None, timeout=None): self.id = int(token.split(':')[0]) self.name = name self.token = token self.agent = agent self.last_update_id = -2 if skip_offset else -1 self.update_prehandlers = [] self.message_handlers = [] self.message_subscribers = LRUCache(maxsize=10000) self.message_prehandlers = [] self.message_next_handlers = LRUCache(maxsize=1000) self.retry_update = 0 self.allowed_updates = allowed_updates self.running = False self.inline_query_handler = None self.callback_query_handler = None self.chosen_inline_result_handler = None self.channel_post_handler = None self.on_updated_listener = None self.on_api_request_listener = None self.botan = None self.timeout = timeout self._noisy = False
class StimulusFile(DataFile): """A DataFile which contains methods for accessing and loading visual behavior stimulus *.pkl files. This file type contains a number of parameters collected during a behavior session including information about stimulus presentations, rewards, trials, and timing for all of the above. """ def __init__(self, filepath: Union[str, Path]): super().__init__(filepath=filepath) @classmethod @cached(cache=LRUCache(maxsize=10), key=from_json_cache_key) def from_json(cls, dict_repr: dict) -> "StimulusFile": filepath = dict_repr["behavior_stimulus_file"] return cls(filepath=filepath) def to_json(self) -> Dict[str, str]: return {"behavior_stimulus_file": str(self.filepath)} @classmethod @cached(cache=LRUCache(maxsize=10), key=from_lims_cache_key) def from_lims( cls, db: PostgresQueryMixin, behavior_session_id: Union[int, str] ) -> "StimulusFile": query = STIMULUS_FILE_QUERY_TEMPLATE.format( behavior_session_id=behavior_session_id ) filepath = db.fetchone(query, strict=True) return cls(filepath=filepath) @staticmethod def load_data(filepath: Union[str, Path]) -> dict: filepath = safe_system_path(file_name=filepath) return pd.read_pickle(filepath)
def __init__(self, capp, io_loop): """Monitors events that are received from celery. capp - The celery app io_loop - The event loop to use for dispatch """ super().__init__() self.capp = capp self.timer = PeriodicCallback(self.on_enable_events, self.events_enable_interval) self.monitor = EventMonitor(self.capp, io_loop) self.listeners = {} self.finished_tasks = LRUCache(self.max_finished_history)
def __init__(self, arch, collection, db = None, host = None, port = None): self.arch = arch self.db_name = self.DEF_DB if db is None else db self.collection_name = collection self.host = self.DEF_HOST if host is None else host self.port = self.DEF_PORT if port is None else port # instructions cache self.cache = LRUCache(maxsize = self.CACHE_SIZE) # connect to the server self.client = pymongo.Connection(self.host, self.port) self.db = self.client[self.db_name] # get collection self.collection = self.db[self.collection_name] self.collection.ensure_index(self.INDEX)
def __init__(self, text, spacy_pipeline=None, lang='auto', metadata=None, max_cachesize=5): self.metadata = {} if metadata is None else metadata self.lang = text_utils.detect_language(text) if lang == 'auto' else lang if spacy_pipeline is None: self.spacy_pipeline = data.load_spacy_pipeline(lang=self.lang) else: # check for match between text and supplied spacy pipeline language if spacy_pipeline.lang != self.lang: msg = 'TextDoc.lang {} != spacy_pipeline.lang {}'.format( self.lang, spacy_pipeline.lang) raise ValueError(msg) else: self.spacy_pipeline = spacy_pipeline self.spacy_vocab = self.spacy_pipeline.vocab self.spacy_stringstore = self.spacy_vocab.strings self.spacy_doc = self.spacy_pipeline(text) self._term_counts = Counter() self._cache = LRUCache(maxsize=max_cachesize)
class CodeStorageMongo(REIL.CodeStorageMem): # mongodb host DEF_HOST = '127.0.0.1' DEF_PORT = 27017 # defult database name DEF_DB = 'openreil' # index for instructions collection INDEX = [('addr', pymongo.ASCENDING), ('inum', pymongo.ASCENDING)] CACHE_SIZE = 1024 def __init__(self, arch, collection, db = None, host = None, port = None): self.arch = arch self.db_name = self.DEF_DB if db is None else db self.collection_name = collection self.host = self.DEF_HOST if host is None else host self.port = self.DEF_PORT if port is None else port # instructions cache self.cache = LRUCache(maxsize = self.CACHE_SIZE) # connect to the server self.client = pymongo.Connection(self.host, self.port) self.db = self.client[self.db_name] # get collection self.collection = self.db[self.collection_name] self.collection.ensure_index(self.INDEX) def __iter__(self): for item in self.collection.find().sort(self.INDEX): yield REIL.Insn(self._insn_from_item(item)) def _insn_to_item(self, insn): insn = REIL.Insn(insn) def _arg_in(arg): if arg.type == REIL.A_NONE: return () elif arg.type == REIL.A_CONST: return ( arg.type, arg.size, _U64IN(arg.val) ) else: return ( arg.type, arg.size, arg.name ) if insn.has_attr(REIL.IATTR_BIN): # JSON doesn't support binary data insn.set_attr(REIL.IATTR_BIN, base64.b64encode(insn.get_attr(REIL.IATTR_BIN))) # JSON doesn't support numeric keys attr = [ (key, val) for key, val in insn.attr.items() ] return { 'addr': _U64IN(insn.addr), 'size': insn.size, 'inum': insn.inum, 'op': insn.op, \ 'a': _arg_in(insn.a), 'b': _arg_in(insn.b), 'c': _arg_in(insn.c), \ 'attr': attr } def _insn_from_item(self, item): attr, attr_dict = item['attr'], {} def _arg_out(arg): if len(arg) == 0: return () elif REIL.Arg_type(arg) == REIL.A_CONST: arg = ( REIL.Arg_type(arg), REIL.Arg_size(arg), _U64OUT(REIL.Arg_val(arg)) ) return arg for key, val in attr: attr_dict[key] = val if attr_dict.has_key(REIL.IATTR_BIN): # get instruction binary data from base64 attr_dict[REIL.IATTR_BIN] = base64.b64decode(attr_dict[REIL.IATTR_BIN]) return ( ( _U64OUT(item['addr']), item['size'] ), item['inum'], item['op'], \ ( _arg_out(item['a']), _arg_out(item['b']), _arg_out(item['c']) ), \ attr_dict ) def _get_key(self, ir_addr): return { 'addr': ir_addr[0], 'inum': ir_addr[1] } def _find(self, ir_addr): return self.collection.find_one(self._get_key(ir_addr)) def _get_insn(self, ir_addr): # get item from cache try: return self.cache[ir_addr] except KeyError: pass # get item from collection insn = self._find(ir_addr) if insn is not None: insn = self._insn_from_item(insn) # update cache self.cache[ir_addr] = insn return insn else: raise REIL.StorageError(*ir_addr) def _del_insn(self, ir_addr): insn = self._find(ir_addr) if insn is not None: # remove item from collection self.collection.remove(self._get_key(ir_addr)) # remove item from cache try: del self.cache[ir_addr] except KeyError: pass else: raise REIL.StorageError(*ir_addr) def _put_insn(self, insn): ir_addr = REIL.Insn_ir_addr(insn) if self._find(ir_addr) is not None: # update existing item self.collection.update(self._get_key(ir_addr), self._insn_to_item(insn)) else: # add a new item self.collection.insert(self._insn_to_item(insn)) # update cache self.cache[ir_addr] = insn def size(self): return self.collection.find().count() def clear(self): self.cache.clear() # remove all items of collection return self.collection.remove()
class Server(Component): channel = "server" def init(self, args, db, hosts, logger): self.args = args self.db = db self.hosts = hosts self.logger = logger self.bind = args.bind self.forward = args.forward self.peers = {} self.requests = {} self.cache = LRUCache(maxsize=args.cachesize) if args.daemon: Daemon(args.pidfile).register(self) if args.debug: Debugger(events=args.verbose, logger=logger).register(self) self.transport = UDPServer( self.bind, channel=self.channel ).register(self) self.protocol = DNS(channel=self.channel).register(self) def ready(self, server, bind): self.logger.info( "DNS Server Ready! Listening on {0:s}:{1:d}".format(*bind) ) # Timer(1, Event.create("ttl"), persist=True, channel=self.channel).register(self) def ttl(self): for k, rrs in self.cache.items()[:]: if any(rr.ttl == 0 for rr in rrs): qname, qtype, qclass = k self.logger.info( "Expired Entry: {0:s} {1:s} {2:s}".format( CLASS.get(qclass), QTYPE.get(qtype), qname ) ) del self.cache[k] else: for rr in rrs: rr.ttl -= 1 def request(self, peer, request): qname = str(request.q.qname) qtype = request.q.qtype qclass = request.q.qclass key = (qname, qtype, qclass) if key in self.cache: self.logger.info( "Cached Request ({0:s}): {1:s} {2:s} {3:s}".format( "{0:s}:{1:d}".format(*peer), CLASS.get(qclass), QTYPE.get(qtype), qname ) ) reply = request.reply() for rr in self.cache[key]: reply.add_answer(rr) self.fire(write(peer, reply.pack())) return if key in self.hosts: self.logger.info( "Local Hosts Request ({0:s}): {1:s} {2:s} {3:s}".format( "{0:s}:{1:d}".format(*peer), CLASS.get(qclass), QTYPE.get(qtype), qname ) ) reply = request.reply() for rdata in self.hosts[key]: rr = RR( qname, rclass=CLASS.IN, rtype=QTYPE.AAAA if ":" in rdata else QTYPE.A, rdata=AAAA(rdata) if ":" in rdata else A(rdata) ) reply.add_answer(rr) self.cache[key] = rr self.fire(write(peer, reply.pack())) return records = Record.objects.filter(rname=qname) if not records: self.logger.info( "Request ({0:s}): {1:s} {2:s} {3:s} -> {4:s}:{5:d}".format( "{0:s}:{1:d}".format(*peer), CLASS.get(qclass), QTYPE.get(qtype), qname, self.forward, 53 ) ) lookup = DNSRecord(q=DNSQuestion(qname, qtype, qclass)) id = lookup.header.id self.peers[id] = peer self.requests[id] = request self.fire(write((self.forward, 53), lookup.pack())) return self.logger.info( "Authoritative Request ({0:s}): {1:s} {2:s} {3:s}".format( "{0:s}:{1:d}".format(*peer), CLASS.get(qclass), QTYPE.get(qtype), qname ) ) rr = [] reply = request.reply() if len(records) == 1 and records[0].rtype == CNAME: rr.append(records[0].rr) records = Record.objects.filter(rname=records[0].rdata) for record in records: rr.append(record.rr) reply.add_answer(*rr) self.cache[key] = rr self.fire(write(peer, reply.pack())) def response(self, peer, response): id = response.header.id qname = str(response.q.qname) qtype = response.q.qtype qclass = response.q.qclass if id not in self.peers: self.logger.info( "Unknown Response ({0:s}): {1:s} {2:s} {3:s}".format( "{0:s}:{1:d}".format(*peer), CLASS.get(qclass), QTYPE.get(qtype), qname ) ) return peer = self.peers[id] request = self.requests[id] key = (str(request.q.qname), request.q.qtype, request.q.qclass) reply = request.reply() reply.add_answer(*response.rr) self.cache[key] = reply.rr self.fire(write(peer, reply.pack())) del self.peers[id] del self.requests[id]
def __init__(self, upstreams, cache_size=None): super().__init__() self._upstreams = upstreams self._cache = LRUCache(cache_size) if cache_size else None self._cache_lock = RLock()
class TextDoc(object): """ Class that tokenizes, tags, and parses a text document, and provides an easy interface to information extraction, alternative document representations, and statistical measures of the text. Args: text (str) spacy_pipeline (``spacy.<lang>.<Lang>()``, optional) lang (str, optional) metadata (dict, optional) max_cachesize (int, optional) """ def __init__(self, text, spacy_pipeline=None, lang='auto', metadata=None, max_cachesize=5): self.metadata = {} if metadata is None else metadata self.lang = text_utils.detect_language(text) if lang == 'auto' else lang if spacy_pipeline is None: self.spacy_pipeline = data.load_spacy_pipeline(lang=self.lang) else: # check for match between text and supplied spacy pipeline language if spacy_pipeline.lang != self.lang: msg = 'TextDoc.lang {} != spacy_pipeline.lang {}'.format( self.lang, spacy_pipeline.lang) raise ValueError(msg) else: self.spacy_pipeline = spacy_pipeline self.spacy_vocab = self.spacy_pipeline.vocab self.spacy_stringstore = self.spacy_vocab.strings self.spacy_doc = self.spacy_pipeline(text) self._term_counts = Counter() self._cache = LRUCache(maxsize=max_cachesize) def __repr__(self): return 'TextDoc({} tokens: {})'.format( len(self.spacy_doc), repr(self.text[:50].replace('\n',' ').strip() + '...')) def __len__(self): return self.n_tokens def __getitem__(self, index): return self.spacy_doc[index] def __iter__(self): for tok in self.spacy_doc: yield tok @property def tokens(self): """Yield the document's tokens as tokenized by spacy; same as ``__iter__``.""" for tok in self.spacy_doc: yield tok @property def sents(self): """Yield the document's sentences as segmented by spacy.""" for sent in self.spacy_doc.sents: yield sent def merge(self, spans): """ Merge spans *in-place* within doc so that each takes up a single token. Note: All cached methods on this doc will be cleared. Args: spans (iterable(``spacy.Span``)): for example, the results from :func:`extract.named_entities() <textacy.extract.named_entities>` or :func:`extract.pos_regex_matches() <textacy.extract.pos_regex_matches>` """ with LOCK: self._cache.clear() spacy_utils.merge_spans(spans) ############### # DOC AS TEXT # @property def text(self): """Return the document's raw text.""" return self.spacy_doc.text_with_ws @property def tokenized_text(self): """Return text as an ordered, nested list of tokens per sentence.""" return [[token.text for token in sent] for sent in self.spacy_doc.sents] @property def pos_tagged_text(self): """Return text as an ordered, nested list of (token, POS) pairs per sentence.""" return [[(token.text, token.pos_) for token in sent] for sent in self.spacy_doc.sents] ####################### # DOC REPRESENTATIONS # def as_bag_of_terms(self, weighting='tf', normalized=True, binary=False, idf=None, lemmatize='auto', ngram_range=(1, 1), include_nes=False, include_nps=False, include_kts=False): """ Represent doc as a "bag of terms", an unordered set of (term id, term weight) pairs, where term weight may be by TF or TF*IDF. Args: weighting (str {'tf', 'tfidf'}, optional): weighting of term weights, either term frequency ('tf') or tf * inverse doc frequency ('tfidf') idf (dict, optional): if `weighting` = 'tfidf', idf's must be supplied externally, such as from a `TextCorpus` object lemmatize (bool or 'auto', optional): if True, lemmatize all terms when getting their frequencies ngram_range (tuple(int), optional): (min n, max n) values for n-grams to include in terms list; default (1, 1) only includes unigrams include_nes (bool, optional): if True, include named entities in terms list include_nps (bool, optional): if True, include noun phrases in terms list include_kts (bool, optional): if True, include key terms in terms list normalized (bool, optional): if True, normalize term freqs by the total number of unique terms binary (bool optional): if True, set all (non-zero) term freqs equal to 1 Returns: :class:`collections.Counter <collections.Counter>`: mapping of term ids to corresponding term weights """ term_weights = self.term_counts( lemmatize=lemmatize, ngram_range=ngram_range, include_nes=include_nes, include_nps=include_nps, include_kts=include_kts) if binary is True: term_weights = Counter({key: 1 for key in term_weights.keys()}) elif normalized is True: # n_terms = sum(term_freqs.values()) n_tokens = self.n_tokens term_weights = Counter({key: val / n_tokens for key, val in term_weights.items()}) if weighting == 'tfidf' and idf: term_weights = Counter({key: val * idf[key] for key, val in term_weights.items()}) return term_weights def as_bag_of_concepts(self): raise NotImplementedError() def as_semantic_network(self): raise NotImplementedError() ########################## # INFORMATION EXTRACTION # @cachedmethod(attrgetter('_cache'), key=partial(hashkey, 'words')) def words(self, **kwargs): """ Extract an ordered list of words from a spacy-parsed doc, optionally filtering words by part-of-speech (etc.) and frequency. .. seealso:: :func:`extract.words() <textacy.extract.words>` for all function kwargs. """ return list(extract.words(self.spacy_doc, **kwargs)) @cachedmethod(attrgetter('_cache'), key=partial(hashkey, 'ngrams')) def ngrams(self, n, **kwargs): """ Extract an ordered list of n-grams (``n`` consecutive words) from doc, optionally filtering n-grams by the types and parts-of-speech of the constituent words. Args: n (int): number of tokens to include in n-grams; 1 => unigrams, 2 => bigrams .. seealso:: :func:`extract.ngrams() <textacy.extract.ngrams>` for all function kwargs. """ return list(extract.ngrams(self.spacy_doc, n, **kwargs)) @cachedmethod(attrgetter('_cache'), key=partial(hashkey, 'named_entities')) def named_entities(self, **kwargs): """ Extract an ordered list of named entities (PERSON, ORG, LOC, etc.) from doc, optionally filtering by the entity types and frequencies. .. seealso:: :func:`extract.named_entities() <textacy.extract.named_entities>` for all function kwargs. """ return list(extract.named_entities(self.spacy_doc, **kwargs)) @cachedmethod(attrgetter('_cache'), key=partial(hashkey, 'noun_chunks')) def noun_chunks(self, **kwargs): """ Extract an ordered list of noun phrases from doc, optionally filtering by frequency and dropping leading determiners. .. seealso:: :func:`extract.noun_chunks() <textacy.extract.noun_chunks>` for all function kwargs. """ return list(extract.noun_chunks(self.spacy_doc, **kwargs)) @cachedmethod(attrgetter('_cache'), key=partial(hashkey, 'pos_regex_matches')) def pos_regex_matches(self, pattern): """ Extract sequences of consecutive tokens from a spacy-parsed doc whose part-of-speech tags match the specified regex pattern. Args: pattern (str): Pattern of consecutive POS tags whose corresponding words are to be extracted, inspired by the regex patterns used in NLTK's ``nltk.chunk.regexp``. Tags are uppercase, from the universal tag set; delimited by < and >, which are basically converted to parentheses with spaces as needed to correctly extract matching word sequences; white space in the input doesn't matter. Examples (see :obj:`POS_REGEX_PATTERNS <textacy.regexes_etc.POS_REGEX_PATTERNS>`): * noun phrase: r'<DET>? (<NOUN>+ <ADP|CONJ>)* <NOUN>+' * compound nouns: r'<NOUN>+' * verb phrase: r'<VERB>?<ADV>*<VERB>+' * prepositional phrase: r'<PREP> <DET>? (<NOUN>+<ADP>)* <NOUN>+' """ return list(extract.pos_regex_matches(self.spacy_doc, pattern)) @cachedmethod(attrgetter('_cache'), key=partial(hashkey, 'subject_verb_object_triples')) def subject_verb_object_triples(self): """ Extract an *un*ordered list of distinct subject-verb-object (SVO) triples from doc. """ return list(extract.subject_verb_object_triples(self.spacy_doc)) @cachedmethod(attrgetter('_cache'), key=partial(hashkey, 'acronyms_and_definitions')) def acronyms_and_definitions(self, **kwargs): """ Extract a collection of acronyms and their most likely definitions, if available, from doc. If multiple definitions are found for a given acronym, only the most frequently occurring definition is returned. .. seealso:: :func:`extract.acronyms_and_definitions() <textacy.extract.acronyms_and_definitions>` for all function kwargs. """ return extract.acronyms_and_definitions(self.spacy_doc, **kwargs) @cachedmethod(attrgetter('_cache'), key=partial(hashkey, 'semistructured_statements')) def semistructured_statements(self, entity, **kwargs): """ Extract "semi-structured statements" from doc, each as a (entity, cue, fragment) triple. This is similar to subject-verb-object triples. Args: entity (str): a noun or noun phrase of some sort (e.g. "President Obama", "global warming", "Python") .. seealso:: :func:`extract.semistructured_statements() <textacy.extract.semistructured_statements>` for all function kwargs. """ return list(extract.semistructured_statements( self.spacy_doc, entity, **kwargs)) @cachedmethod(attrgetter('_cache'), key=partial(hashkey, 'direct_quotations')) def direct_quotations(self): """ Baseline, not-great attempt at direction quotation extraction (no indirect or mixed quotations) using rules and patterns. English only. """ return list(extract.direct_quotations(self.spacy_doc)) @cachedmethod(attrgetter('_cache'), key=partial(hashkey, 'key_terms')) def key_terms(self, algorithm='sgrank', n=10): """ Extract key terms from a document using `algorithm`. Args: algorithm (str {'sgrank', 'textrank', 'singlerank'}, optional): name of algorithm to use for key term extraction n (int or float, optional): if int, number of top-ranked terms to return as keyterms; if float, must be in the open interval (0.0, 1.0), representing the fraction of top-ranked terms to return as keyterms Raises: ValueError: if ``algorithm`` not in {'sgrank', 'textrank', 'singlerank'} """ if algorithm == 'sgrank': return keyterms.sgrank(self.spacy_doc, window_width=1500, n_keyterms=n) elif algorithm == 'textrank': return keyterms.textrank(self.spacy_doc, n_keyterms=n) elif algorithm == 'singlerank': return keyterms.singlerank(self.spacy_doc, n_keyterms=n) else: raise ValueError('algorithm {} not a valid option'.format(algorithm)) ############## # STATISTICS # @cachedmethod(attrgetter('_cache'), key=partial(hashkey, 'term_counts')) def term_counts(self, lemmatize='auto', ngram_range=(1, 1), include_nes=False, include_nps=False, include_kts=False): """ Get the number of occurrences ("counts") of each unique term in doc; terms may be words, n-grams, named entities, noun phrases, and key terms. Args: lemmatize (bool or 'auto', optional): if True, lemmatize all terms when getting their frequencies; if 'auto', lemmatize all terms that aren't proper nouns or acronyms ngram_range (tuple(int), optional): (min n, max n) values for n-grams to include in terms list; default (1, 1) only includes unigrams include_nes (bool, optional): if True, include named entities in terms list include_nps (bool, optional): if True, include noun phrases in terms list include_kts (bool, optional): if True, include key terms in terms list Returns: :class:`collections.Counter() <collections.Counter>`: mapping of unique term ids to corresponding term counts """ if lemmatize == 'auto': get_id = lambda x: self.spacy_stringstore[spacy_utils.normalized_str(x)] elif lemmatize is True: get_id = lambda x: self.spacy_stringstore[x.lemma_] else: get_id = lambda x: self.spacy_stringstore[x.text] for n in range(ngram_range[0], ngram_range[1] + 1): if n == 1: self._term_counts = self._term_counts | Counter( get_id(word) for word in self.words()) else: self._term_counts = self._term_counts | Counter( get_id(ngram) for ngram in self.ngrams(n)) if include_nes is True: self._term_counts = self._term_counts | Counter( get_id(ne) for ne in self.named_entities()) if include_nps is True: self._term_counts = self._term_counts | Counter( get_id(np) for np in self.noun_chunks()) if include_kts is True: # HACK: key terms are currently returned as strings # TODO: cache key terms, and return them as spacy spans get_id = lambda x: self.spacy_stringstore[x] self._term_counts = self._term_counts | Counter( get_id(kt) for kt, _ in self.key_terms()) return self._term_counts def term_count(self, term): """ Get the number of occurrences ("count") of term in doc. Args: term (str or ``spacy.Token`` or ``spacy.Span``) Returns: int """ # figure out what object we're dealing with here; convert as necessary if isinstance(term, str): term_text = term term_id = self.spacy_stringstore[term_text] term_len = term_text.count(' ') + 1 elif isinstance(term, spacy_token): term_text = spacy_utils.normalized_str(term) term_id = self.spacy_stringstore[term_text] term_len = 1 elif isinstance(term, spacy_span): term_text = spacy_utils.normalized_str(term) term_id = self.spacy_stringstore[term_text] term_len = len(term) term_count_ = self._term_counts[term_id] if term_count_ > 0: return term_count_ # have we not already counted the appropriate `n` n-grams? if not any(self.spacy_stringstore[t].count(' ') == term_len for t in self._term_counts): get_id = lambda x: self.spacy_stringstore[spacy_utils.normalized_str(x)] if term_len == 1: self._term_counts += Counter(get_id(w) for w in self.words()) else: self._term_counts += Counter(get_id(ng) for ng in self.ngrams(term_len)) term_count_ = self._term_counts[term_id] if term_count_ > 0: return term_count_ # last resort: try a regular expression return sum(1 for _ in re.finditer(re.escape(term_text), self.text)) @property def n_tokens(self): """The number of tokens in the document -- including punctuation.""" return len(self.spacy_doc) def n_words(self, filter_stops=False, filter_punct=True, filter_nums=False): """ The number of words in the document, with optional filtering of stop words, punctuation (on by default), and numbers. """ return len(self.words(filter_stops=filter_stops, filter_punct=filter_punct, filter_nums=filter_nums)) @property def n_sents(self): """The number of sentences in the document.""" return sum(1 for _ in self.spacy_doc.sents) def n_paragraphs(self, pattern=r'\n\n+'): """The number of paragraphs in the document, as delimited by ``pattern``.""" return sum(1 for _ in re.finditer(pattern, self.text)) + 1 @cachedmethod(attrgetter('_cache'), key=partial(hashkey, 'readability_stats')) def readability_stats(self): return text_stats.readability_stats(self)
def __init__(self,cachefilename,CACHE_SIZE,logger=logger('detoursCache.log')): self.lock = threading.RLock() self.cachefilename = cachefilename self.entry = LRUCache(maxsize=CACHE_SIZE) self.logger=logger self.hitcount=0
class Cache(): def __init__(self,cachefilename,CACHE_SIZE,logger=logger('detoursCache.log')): self.lock = threading.RLock() self.cachefilename = cachefilename self.entry = LRUCache(maxsize=CACHE_SIZE) self.logger=logger self.hitcount=0 def hit(self): self.lock.acquire(blocking=1) try: self.hitcount+=1 finally: self.lock.release() def reset(self): self.lock.acquire(blocking=1) try: self.hitcount=0 finally: self.lock.release() def push(self,key,val): self.lock.acquire(blocking=1) try: self.entry[key]=val except: return finally: self.lock.release() def get(self,key): self.lock.acquire(blocking=1) try: return self.entry[key] except: return False finally: self.lock.release() def write_to_disk(self): self.lock.acquire(blocking=1) try: cachefile = open(self.cachefilename,'w') for key,val in self.entry.items(): print(key+'\t'+val,file=cachefile) cachefile.close() finally: self.lock.release() def load_from_disk(self): self.lock.acquire(blocking=1) try: if os.path.exists(self.cachefilename): with open(self.cachefilename, 'r') as f: for line in f: if line == "": continue rline = line.strip() splitvals=rline.split('\t') if len(splitvals) == 2: key=splitvals[0] valstr=splitvals[1] self.entry[key]=valstr else: continue except: self.logger.error("Failed to read existing cache file") raise("Error in loading previous cache file") finally: self.lock.release()
class GMusicLibraryProvider(backend.LibraryProvider): root_directory = Ref.directory(uri='gmusic:directory', name='Google Music') def __init__(self, *args, **kwargs): super(GMusicLibraryProvider, self).__init__(*args, **kwargs) # tracks, albums, and artists here refer to what is explicitly # in our library. self.tracks = {} self.albums = {} self.artists = {} # aa_* caches are *only* used for temporary objects. Library # objects will never make it here. self.aa_artists = LRUCache(1024) self.aa_tracks = LRUCache(1024) self.aa_albums = LRUCache(1024) self._radio_stations_in_browse = ( self.backend.config['gmusic']['radio_stations_in_browse']) self._radio_stations_count = ( self.backend.config['gmusic']['radio_stations_count']) self._radio_tracks_count = ( self.backend.config['gmusic']['radio_tracks_count']) self._top_tracks_count = ( self.backend.config['gmusic']['top_tracks_count']) # Setup the root of library browsing. self._root = [ Ref.directory(uri='gmusic:album', name='Albums'), Ref.directory(uri='gmusic:artist', name='Artists'), Ref.directory(uri='gmusic:track', name='Tracks') ] if self._radio_stations_in_browse: self._root.append(Ref.directory(uri='gmusic:radio', name='Radios')) @property def all_access(self): return self.backend.session.all_access def _browse_tracks(self): tracks = list(self.tracks.values()) tracks.sort(key=lambda ref: ref.name) refs = [] for track in tracks: refs.append(track_to_ref(track)) return refs def _browse_albums(self): refs = [] for album in self.albums.values(): refs.append(album_to_ref(album)) refs.sort(key=lambda ref: ref.name) return refs def _browse_album(self, uri): refs = [] for track in self._lookup_album(uri): refs.append(track_to_ref(track, True)) return refs def _browse_artists(self): refs = [] for artist in self.artists.values(): refs.append(artist_to_ref(artist)) refs.sort(key=lambda ref: ref.name) return refs def _browse_artist(self, uri): refs = [] for album in self._get_artist_albums(uri): refs.append(album_to_ref(album)) refs.sort(key=lambda ref: ref.name) if len(refs) > 0: refs.insert(0, Ref.directory(uri=uri + ':all', name='All Tracks')) is_all_access = uri.startswith('gmusic:artist:A') if is_all_access: refs.insert(1, Ref.directory(uri=uri + ':top', name='Top Tracks')) return refs else: # Show all tracks if no album is available return self._browse_artist_all_tracks(uri) def _browse_artist_all_tracks(self, uri): artist_uri = ':'.join(uri.split(':')[:3]) refs = [] tracks = self._lookup_artist(artist_uri, True) for track in tracks: refs.append(track_to_ref(track)) return refs def _browse_artist_top_tracks(self, uri): artist_uri = ':'.join(uri.split(':')[:3]) refs = [] tracks = self._get_artist_top_tracks(artist_uri) for track in tracks: refs.append(track_to_ref(track)) return refs def _browse_radio_stations(self, uri): stations = self.backend.session.get_radio_stations( self._radio_stations_count) # create Ref objects refs = [] for station in stations: refs.append(Ref.directory(uri='gmusic:radio:' + station['id'], name=station['name'])) return refs def _browse_radio_station(self, uri): station_id = uri.split(':')[2] tracks = self.backend.session.get_station_tracks( station_id, self._radio_tracks_count) # create Ref objects refs = [] for track in tracks: mopidy_track = self._to_mopidy_track(track) self.aa_tracks[mopidy_track.uri] = mopidy_track refs.append(track_to_ref(mopidy_track)) return refs def browse(self, uri): logger.debug('browse: %s', str(uri)) if not uri: return [] if uri == self.root_directory.uri: return self._root parts = uri.split(':') # tracks if uri == 'gmusic:track': return self._browse_tracks() # albums if uri == 'gmusic:album': return self._browse_albums() # a single album # uri == 'gmusic:album:album_id' if len(parts) == 3 and parts[1] == 'album': return self._browse_album(uri) # artists if uri == 'gmusic:artist': return self._browse_artists() # a single artist # uri == 'gmusic:artist:artist_id' if len(parts) == 3 and parts[1] == 'artist': return self._browse_artist(uri) # all tracks of a single artist # uri == 'gmusic:artist:artist_id:all' if len(parts) == 4 and parts[1] == 'artist' and parts[3] == 'all': return self._browse_artist_all_tracks(uri) # top tracks of a single artist # uri == 'gmusic:artist:artist_id:top' if len(parts) == 4 and parts[1] == 'artist' and parts[3] == 'top': return self._browse_artist_top_tracks(uri) # all radio stations if uri == 'gmusic:radio': return self._browse_radio_stations(uri) # a single radio station # uri == 'gmusic:radio:station_id' if len(parts) == 3 and parts[1] == 'radio': return self._browse_radio_station(uri) logger.debug('Unknown uri for browse request: %s', uri) return [] def lookup(self, uri): if uri.startswith('gmusic:track:'): return self._lookup_track(uri) elif uri.startswith('gmusic:album:'): return self._lookup_album(uri) elif uri.startswith('gmusic:artist:'): return self._lookup_artist(uri) else: return [] def _lookup_track(self, uri): is_all_access = uri.startswith('gmusic:track:T') try: return [self.tracks[uri]] except KeyError: logger.debug('Track not a library track %r', uri) pass if is_all_access and self.all_access: track = self.aa_tracks.get(uri) if track: return [track] song = self.backend.session.get_track_info(uri.split(':')[2]) if song is None: logger.warning('There is no song %r', uri) return [] if 'artistId' not in song: logger.warning('Failed to lookup %r', uri) return [] mopidy_track = self._to_mopidy_track(song) self.aa_tracks[mopidy_track.uri] = mopidy_track return [mopidy_track] else: return [] def _lookup_album(self, uri): is_all_access = uri.startswith('gmusic:album:B') if self.all_access and is_all_access: tracks = self.aa_albums.get(uri) if tracks: return tracks album = self.backend.session.get_album_info( uri.split(':')[2], include_tracks=True) if album and album.get('tracks'): tracks = [self._to_mopidy_track(track) for track in album['tracks']] for track in tracks: self.aa_tracks[track.uri] = track tracks = sorted(tracks, key=lambda t: (t.disc_no, t.track_no)) self.aa_albums[uri] = tracks return tracks logger.warning('Failed to lookup all access album %r: %r', uri, album) # Even if the album has an all access ID, we need to look it # up here (as a fallback) because purchased tracks can have a # store ID, but only show up in your library. try: album = self.albums[uri] except KeyError: logger.debug('Failed to lookup %r', uri) return [] tracks = self._find_exact( dict(album=album.name, artist=[artist.name for artist in album.artists], date=album.date)).tracks return sorted(tracks, key=lambda t: (t.disc_no, t.track_no)) def _get_artist_top_tracks(self, uri): is_all_access = uri.startswith('gmusic:artist:A') artist_id = uri.split(':')[2] if not is_all_access: logger.debug("Top Tracks not available for non-all-access artists") return [] artist_info = self.backend.session.get_artist_info(artist_id, include_albums=False, max_top_tracks=self._top_tracks_count, max_rel_artist=0) top_tracks = [] for track_dict in artist_info['topTracks']: top_tracks.append(self._to_mopidy_track(track_dict)) return top_tracks def _get_artist_albums(self, uri): is_all_access = uri.startswith('gmusic:artist:A') artist_id = uri.split(':')[2] if is_all_access: # all access artist_infos = self.backend.session.get_artist_info( artist_id, max_top_tracks=0, max_rel_artist=0) if artist_infos is None or 'albums' not in artist_infos: return [] albums = [] for album in artist_infos['albums']: albums.append( self._aa_search_album_to_mopidy_album({'album': album})) return albums elif self.all_access and artist_id in self.aa_artists: albums = self._get_artist_albums( 'gmusic:artist:%s' % self.aa_artists[artist_id]) if len(albums) > 0: return albums # else fall back to non aa albums if uri in self.artists: artist = self.artists[uri] return [album for album in self.albums.values() if artist in album.artists] else: logger.debug('0 albums available for artist %r', uri) return [] def _lookup_artist(self, uri, exact_match=False): def sorter(track): return ( track.album.date, track.album.name, track.disc_no, track.track_no, ) if self.all_access: try: all_access_id = self.aa_artists[uri.split(':')[2]] artist_infos = self.backend.session.get_artist_info( all_access_id, max_top_tracks=0, max_rel_artist=0) if not artist_infos or not artist_infos['albums']: logger.warning('Failed to lookup %r', artist_infos) tracks = [ self._lookup_album('gmusic:album:' + album['albumId']) for album in artist_infos['albums']] tracks = reduce(lambda a, b: (a + b), tracks) return sorted(tracks, key=sorter) except KeyError: pass try: artist = self.artists[uri] except KeyError: logger.debug('Failed to lookup %r', uri) return [] tracks = self._find_exact( dict(artist=artist.name)).tracks if exact_match: tracks = filter(lambda t: artist in t.artists, tracks) return sorted(tracks, key=sorter) def refresh(self, uri=None): self.tracks = {} self.albums = {} self.artists = {} album_tracks = {} for track in self.backend.session.get_all_songs(): mopidy_track = self._to_mopidy_track(track) self.tracks[mopidy_track.uri] = mopidy_track self.albums[mopidy_track.album.uri] = mopidy_track.album # We don't care about the order because we're just using # this as a temporary variable to grab the proper album # artist out of the album. if mopidy_track.album.uri not in album_tracks: album_tracks[mopidy_track.album.uri] = [] album_tracks[mopidy_track.album.uri].append(mopidy_track) # Yes, this is awful. No, I don't have a better solution. Yes, # I'm annoyed at Google for not providing album artist IDs. for album in self.albums.values(): artist_found = False for album_artist in album.artists: for track in album_tracks[album.uri]: for artist in track.artists: if album_artist.name == artist.name: artist_found = True self.artists[artist.uri] = artist if not artist_found: for artist in album.artists: self.artists[artist.uri] = artist def search(self, query=None, uris=None, exact=False): if exact: return self._find_exact(query=query, uris=uris) lib_tracks, lib_artists, lib_albums = self._search_library(query, uris) if query: aa_tracks, aa_artists, aa_albums = self._search(query, uris) for aa_artist in aa_artists: lib_artists.add(aa_artist) for aa_album in aa_albums: lib_albums.add(aa_album) lib_tracks = set(lib_tracks) for aa_track in aa_tracks: lib_tracks.add(aa_track) return SearchResult(uri='gmusic:search', tracks=lib_tracks, artists=lib_artists, albums=lib_albums) def _find_exact(self, query=None, uris=None): # Find exact can only be done on gmusic library, # since one can't filter all access searches lib_tracks, lib_artists, lib_albums = self._search_library(query, uris) return SearchResult(uri='gmusic:search', tracks=lib_tracks, artists=lib_artists, albums=lib_albums) def _search(self, query=None, uris=None): for (field, values) in query.iteritems(): if not hasattr(values, '__iter__'): values = [values] # Since gmusic does not support search filters, just search for the # first 'searchable' filter if field in [ 'track_name', 'album', 'artist', 'albumartist', 'any']: logger.info( 'Searching Google Play Music for: %s', values[0]) res = self.backend.session.search(values[0], max_results=50) if res is None: return [], [], [] albums = [ self._aa_search_album_to_mopidy_album(album_res) for album_res in res['album_hits']] artists = [ self._aa_search_artist_to_mopidy_artist(artist_res) for artist_res in res['artist_hits']] tracks = [ self._aa_search_track_to_mopidy_track(track_res) for track_res in res['song_hits']] return tracks, artists, albums return [], [], [] def _search_library(self, query=None, uris=None): if query is None: query = {} self._validate_query(query) result_tracks = self.tracks.values() for (field, values) in query.iteritems(): if not hasattr(values, '__iter__'): values = [values] # FIXME this is bound to be slow for large libraries for value in values: if field == 'track_no': q = self._convert_to_int(value) else: q = value.strip().lower() def uri_filter(track): return q in track.uri.lower() def track_name_filter(track): return q in track.name.lower() def album_filter(track): return q in getattr(track, 'album', Album()).name.lower() def artist_filter(track): return ( any(q in a.name.lower() for a in track.artists) or albumartist_filter(track)) def albumartist_filter(track): album_artists = getattr(track, 'album', Album()).artists return any(q in a.name.lower() for a in album_artists) def track_no_filter(track): return track.track_no == q def date_filter(track): return track.date and track.date.startswith(q) def any_filter(track): return any([ uri_filter(track), track_name_filter(track), album_filter(track), artist_filter(track), albumartist_filter(track), date_filter(track), ]) if field == 'uri': result_tracks = filter(uri_filter, result_tracks) elif field == 'track_name': result_tracks = filter(track_name_filter, result_tracks) elif field == 'album': result_tracks = filter(album_filter, result_tracks) elif field == 'artist': result_tracks = filter(artist_filter, result_tracks) elif field == 'albumartist': result_tracks = filter(albumartist_filter, result_tracks) elif field == 'track_no': result_tracks = filter(track_no_filter, result_tracks) elif field == 'date': result_tracks = filter(date_filter, result_tracks) elif field == 'any': result_tracks = filter(any_filter, result_tracks) else: raise LookupError('Invalid lookup field: %s' % field) result_artists = set() result_albums = set() for track in result_tracks: result_artists |= track.artists result_albums.add(track.album) return result_tracks, result_artists, result_albums def _validate_query(self, query): for (_, values) in query.iteritems(): if not values: raise LookupError('Missing query') for value in values: if not value: raise LookupError('Missing query') def _to_mopidy_track(self, song): track_id = song.get('id', song.get('nid')) if track_id is None: raise ValueError if track_id[0] != "T" and "-" not in track_id: track_id = "T"+track_id return Track( uri='gmusic:track:' + track_id, name=song['title'], artists=[self._to_mopidy_artist(song)], album=self._to_mopidy_album(song), track_no=song.get('trackNumber', 1), disc_no=song.get('discNumber', 1), date=unicode(song.get('year', 0)), length=int(song['durationMillis']), bitrate=320) def _to_mopidy_album(self, song): name = song.get('album', '') artist = self._to_mopidy_album_artist(song) date = unicode(song.get('year', 0)) album_id = song.get('albumId') if album_id is None: album_id = create_id(artist.name + name + date) uri = 'gmusic:album:' + album_id images = get_images(song) return Album( uri=uri, name=name, artists=[artist], num_tracks=song.get('totalTrackCount'), num_discs=song.get('totalDiscCount'), date=date, images=images) def _to_mopidy_artist(self, song): name = song.get('artist', '') artist_id = song.get('artistId') if artist_id is not None: artist_id = artist_id[0] else: artist_id = create_id(name) uri = 'gmusic:artist:' + artist_id return Artist(uri=uri, name=name) def _to_mopidy_album_artist(self, song): name = song.get('albumArtist', '') if name.strip() == '': name = song.get('artist', '') uri = 'gmusic:artist:' + create_id(name) return Artist(uri=uri, name=name) def _aa_search_track_to_mopidy_track(self, search_track): track = search_track['track'] aa_artist_id = create_id(track['artist']) if 'artistId' in track: aa_artist_id = track['artistId'][0] else: logger.warning('No artistId for Track %r', track) artist = Artist( uri='gmusic:artist:' + aa_artist_id, name=track['artist']) album = Album( uri='gmusic:album:' + track['albumId'], name=track['album'], artists=[artist], date=unicode(track.get('year', 0))) return Track( uri='gmusic:track:' + track['storeId'], name=track['title'], artists=[artist], album=album, track_no=track.get('trackNumber', 1), disc_no=track.get('discNumber', 1), date=unicode(track.get('year', 0)), length=int(track['durationMillis']), bitrate=320) def _aa_search_artist_to_mopidy_artist(self, search_artist): artist = search_artist['artist'] uri = 'gmusic:artist:' + artist['artistId'] return Artist(uri=uri, name=artist['name']) def _aa_search_album_to_mopidy_album(self, search_album): album = search_album['album'] uri = 'gmusic:album:' + album['albumId'] name = album['name'] artist = self._aa_search_artist_album_to_mopidy_artist_album(album) date = unicode(album.get('year', 0)) return Album( uri=uri, name=name, artists=[artist], date=date) def _aa_search_artist_album_to_mopidy_artist_album(self, album): name = album.get('albumArtist', '') if name.strip() == '': name = album.get('artist', '') uri = 'gmusic:artist:' + create_id(name) return Artist(uri=uri, name=name) def _convert_to_int(self, string): try: return int(string) except ValueError: return object()
def popitem(self): key, val = LRUCache.popitem(self) if self._evict: self._evict(key, val) return key, val
class TelegramBot: def __init__(self, token, name): self.name = name self.token = token self.agent = Agent(reactor) self.last_update_id = -1 self.message_handlers = [] self.message_subscribers = LRUCache(maxsize=10000) self.message_prehandlers = [] self.message_next_handlers = LRUCache(maxsize=1000) self.retry_update = 0 self.running = False self.inline_query_handler = None self.callback_query_handler = None self.chosen_inline_result_handler = None self.botan = None def method_url(self, method): return API_URL + 'bot' + self.token + '/' + method def start_update(self): self.running = True @inlineCallbacks def update_bot(): if not self.running: return try: yield self.get_update() self.retry_update = 0 reactor.callWhenRunning(update_bot) except: log.failure("Couldn't get updates. Delaying for %d seconds" % self.retry_update) reactor.callLater(self.retry_update, update_bot) self.retry_update = min(self.retry_update + 3, 20) reactor.callWhenRunning(update_bot) def stop_update(self): self.running = False @inlineCallbacks def get_update(self): payload = {'timeout': 20, 'offset': self.last_update_id + 1} updates = yield _request(self.token, 'getUpdates', params=payload, timeout=25) new_messages_ids = set() new_messages = [] for update in updates: log.debug("New update. ID: {update_id}", update_id=update['update_id']) if update['update_id'] > self.last_update_id: self.last_update_id = update['update_id'] if 'inline_query' in update.keys(): inline_query = InlineQuery.de_json(update['inline_query']) self.process_inline_query(inline_query) elif 'chosen_inline_result' in update.keys(): chosen_inline_result = ChosenInlineResult.de_json(update['chosen_inline_result']) self.process_chosen_inline_query(chosen_inline_result) elif 'message' in update.keys(): msg = Message.de_json(update['message']) msg.bot_name = self.name if not msg.from_user.id in new_messages_ids: new_messages.append(msg) new_messages_ids.add(msg.from_user.id) elif 'callback_query' in update.keys(): callback_query = CallbackQuery.de_json(update['callback_query']) self.process_callback_query(callback_query) else: log.debug("Unknown update type: {update}", update=json.dumps(update, skipkeys=True, ensure_ascii=False, default=lambda o: o.__dict__)) if len(new_messages) > 0: self.process_new_messages(new_messages) def process_callback_query(self, callback_query): if self.callback_query_handler: self.callback_query_handler(callback_query, self) def process_new_messages(self, new_messages): self._notify_message_prehandlers(new_messages) not_processed = [] for message in new_messages: if not self._notify_message_next_handler(message): not_processed.append(message) new_messages = not_processed self._notify_command_handlers(new_messages) self._notify_message_subscribers(new_messages) def process_inline_query(self, inline_query): if self.inline_query_handler: self.inline_query_handler(inline_query, self) def process_chosen_inline_query(self, chosen_inline_result): if self.chosen_inline_result_handler: self.chosen_inline_result_handler(chosen_inline_result, self) def _notify_message_prehandlers(self, new_messages): for message in new_messages: for handler in self.message_prehandlers: handler(message, self) def _notify_command_handlers(self, new_messages): for message in new_messages: for message_handler in self.message_handlers: if self._test_message_handler(message_handler, message): message_handler['function'](message, self) break def _notify_message_subscribers(self, new_messages): for message in new_messages: if not hasattr(message, 'reply_to_message'): continue handler = self.message_subscribers.pop(message.reply_to_message.message_id, None) if handler is not None: handler(message, self) def _notify_message_next_handler(self, message): handler = self.message_next_handlers.pop(message.chat.id, None) if handler is not None: handler(message, self) return True return False def register_message_handler(self, fn, commands=None, regexp=None, func=None, content_types=None): if not content_types: content_types = ['text'] func_dict = {'function': fn, 'content_types': content_types} if regexp: func_dict['regexp'] = regexp if 'text' in content_types else None if func: func_dict['lambda'] = func if commands: func_dict['commands'] = commands if 'text' in content_types else None self.message_handlers.append(func_dict) def message_handler(self, commands=None, regexp=None, func=None, content_types=None): def decorator(fn): self.register_message_handler(fn, commands, regexp, func, content_types) return fn return decorator @staticmethod def _test_message_handler(message_handler, message): if message.content_type not in message_handler['content_types']: return False if 'commands' in message_handler and message.content_type == 'text': cmd = extract_command(message.text) if cmd: for command_pattern in message_handler['commands']: if not command_pattern.endswith('$'): command_pattern += '$' if re.match(command_pattern, cmd): return True return False if 'regexp' in message_handler \ and message.content_type == 'text' \ and re.search(message_handler['regexp'], message.text): return True if 'lambda' in message_handler: return message_handler['lambda'](message) return False @inlineCallbacks def send_message(self, chat_id, text, disable_web_page_preview=None, reply_to_message_id=None, reply_markup=None, parse_mode=None): method = r'sendMessage' payload = {'chat_id': str(chat_id), 'text': text} if disable_web_page_preview: payload['disable_web_page_preview'] = disable_web_page_preview if reply_to_message_id: payload['reply_to_message_id'] = reply_to_message_id if reply_markup: if isinstance(reply_markup, JsonSerializable): payload['reply_markup'] = reply_markup.to_json() elif isinstance(reply_markup, dict): payload['reply_markup'] = json.dumps(reply_markup) if parse_mode: payload['parse_mode'] = parse_mode request = yield _request(self.token, method, 'POST', params=payload) returnValue(Message.de_json(request)) @inlineCallbacks def answer_to_inline_query(self, query_id, results, personal=False): request = yield _request(self.token, 'answerInlineQuery', 'POST', params={ 'inline_query_id': str(query_id), 'results': json.dumps(results, ensure_ascii=False), 'is_personal': personal }) returnValue(request) @inlineCallbacks def edit_message_text(self, chat_id, message_id, text, parse_mode=None, disable_web_page_preview=None, reply_markup=None): method = r'editMessageText' payload = {'chat_id': str(chat_id), 'message_id': str(message_id), 'text': text} if disable_web_page_preview: payload['disable_web_page_preview'] = disable_web_page_preview if reply_markup: if isinstance(reply_markup, JsonSerializable): payload['reply_markup'] = reply_markup.to_json() elif isinstance(reply_markup, dict): payload['reply_markup'] = json.dumps(reply_markup) if parse_mode: payload['parse_mode'] = parse_mode request = yield _request(self.token, method, 'POST', params=payload) returnValue(Message.de_json(request)) @inlineCallbacks def answer_callback_query(self, callback_query_id, text=None, show_alert=None): method = r'answerCallbackQuery' payload = {'callback_query_id': str(callback_query_id)} if text: payload['text'] = text if show_alert: payload['show_alert'] = show_alert request = yield _request(self.token, method, 'POST', params=payload) returnValue(request) @inlineCallbacks def get_file(self, file_id): method = r'getFile' payload = {'file_id': str(file_id)} request = yield _request(self.token, method, 'POST', params=payload) returnValue(File.de_json(request)) def get_file_url(self, file): return "https://api.telegram.org/file/bot%s/%s" % (self.token, file.path) @inlineCallbacks def send_audio(self, chat_id, audio, filename='audio', duration=None, performer=None, title=None, reply_to_message_id=None, reply_markup=None, timeout=30): method = r'sendAudio' payload = {'chat_id': chat_id} files = None if not is_string(audio): files = {'audio': (filename, audio)} else: payload['audio'] = audio if duration: payload['duration'] = duration if performer: payload['performer'] = performer if title: payload['title'] = title if reply_to_message_id: payload['reply_to_message_id'] = reply_to_message_id # if reply_markup: # payload['reply_markup'] = _convert_markup(reply_markup) request = yield _request(self.token, method, 'POST', params=payload, files=files, timeout=timeout) returnValue(Message.de_json(request)) def reply_to(self, message, text, **kwargs): return self.send_message(message.chat.id, text, reply_to_message_id=message.message_id, **kwargs) def send_chat_action(self, chat_id, action): method = r'sendChatAction' payload = {'chat_id': chat_id, 'action': action} return _make_request(self.token, method, 'POST', params=payload) def register_for_reply(self, message, callback): self.message_subscribers[message.message_id] = callback def register_next_chat_handler(self, chat_id, callback): self.message_next_handlers[chat_id] = callback
class ProxyResolver(BaseResolver): _Record = recordclass('_Record', 'expire data') def __init__(self, upstreams, cache_size=None): super().__init__() self._upstreams = upstreams self._cache = LRUCache(cache_size) if cache_size else None self._cache_lock = RLock() def _query_cache(self, key): with self._cache_lock: records = self._cache.get(key, None) if records is None: records = [] else: records = [record for record in records if record.expire > time()] if len(records): records = records[1:] + [records[0]] self._cache[key] = records else: del self._cache[key] return records def _add_to_cache(self, key, record): if self._cache is None: return with self._cache_lock: records = self._cache.get(key, None) if records is None: self._cache[key] = [record] else: for erecord in records: if erecord.data == record.data: erecord.expire = record.expire break else: records.append(record) def _resolve_in_cache(self, questions, oq, oa, now): for q in questions: key = (q.qname, QTYPE.CNAME, q.qclass) cnames = self._query_cache(key) if len(cnames): recursive_questions = [] for cname in cnames: oa.add_answer(RR(ttl=max(cname.expire - now, 0), **cname.data)) recursive_questions.append(DNSQuestion( qname=cname.data['rdata'].label, qtype=q.qtype, qclass=q.qclass )) self._resolve_in_cache(recursive_questions, oq, oa, now) else: if q.qtype != QTYPE.CNAME: key = (q.qname, q.qtype, q.qclass) record_list = self._query_cache(key) if len(record_list): for record in record_list: oa.add_answer(RR(ttl=max(record.expire - now, 0), **record.data)) else: oq.add_question(q) else: oq.add_question(q) def resolve(self, request, handler): now = int(time()) a = request.reply() uq = DNSRecord() self._resolve_in_cache(request.questions, uq, a, now) if len(uq.questions): for upstream in self._upstreams: try: ua_pkt = uq.send( str(upstream.address), upstream.port, upstream.tcp, upstream.timeout, upstream.ipv6 ) ua = DNSRecord.parse(ua_pkt) except: continue for rr in ua.rr: key = (rr.rname, rr.rtype, rr.rclass) cr = self._Record(now + rr.ttl, { 'rname': rr.rname, 'rtype': rr.rtype, 'rclass': rr.rclass, 'rdata': rr.rdata, }) self._add_to_cache(key, cr) a.add_answer(*ua.rr) break else: raise IOError return a
def __init__(self, maxsize, getsizeof=None, evict=None): LRUCache.__init__(self, maxsize, getsizeof) self._evict = evict
class MetricCassandraRepository(abstract_repository.AbstractCassandraRepository): def __init__(self): super(MetricCassandraRepository, self).__init__() self._lock = threading.RLock() LOG.debug("prepare cql statements...") self._measurement_insert_stmt = self._session.prepare(MEASUREMENT_INSERT_CQL) self._measurement_insert_stmt.is_idempotent = True self._measurement_update_stmt = self._session.prepare(MEASUREMENT_UPDATE_CQL) self._measurement_update_stmt.is_idempotent = True self._metric_insert_stmt = self._session.prepare(METRICS_INSERT_CQL) self._metric_insert_stmt.is_idempotent = True self._metric_update_stmt = self._session.prepare(METRICS_UPDATE_CQL) self._metric_update_stmt.is_idempotent = True self._dimension_stmt = self._session.prepare(DIMENSION_INSERT_CQL) self._dimension_stmt.is_idempotent = True self._dimension_metric_stmt = self._session.prepare(DIMENSION_METRIC_INSERT_CQL) self._dimension_metric_stmt.is_idempotent = True self._metric_dimension_stmt = self._session.prepare(METRIC_DIMENSION_INSERT_CQL) self._metric_dimension_stmt.is_idempotent = True self._retrieve_metric_dimension_stmt = self._session.prepare(RETRIEVE_METRIC_DIMENSION_CQL) self._metric_batch = MetricBatch( self._cluster.metadata, self._cluster.load_balancing_policy, self._max_batches) self._metric_id_cache = LRUCache(self._cache_size) self._dimension_cache = LRUCache(self._cache_size) self._metric_dimension_cache = LRUCache(self._cache_size) self._load_dimension_cache() self._load_metric_dimension_cache() def process_message(self, message): (dimensions, metric_name, region, tenant_id, time_stamp, value, value_meta) = parse_measurement_message(message) with self._lock: dim_names = [] dim_list = [] for name in sorted(dimensions.iterkeys()): dim_list.append('%s\t%s' % (name, dimensions[name])) dim_names.append(name) hash_string = '%s\0%s\0%s\0%s' % (region, tenant_id, metric_name, '\0'.join(dim_list)) metric_id = hashlib.sha1(hash_string.encode('utf8')).hexdigest() metric = Metric(id=metric_id, region=region, tenant_id=tenant_id, name=metric_name, dimension_list=dim_list, dimension_names=dim_names, time_stamp=time_stamp, value=value, value_meta=json.dumps(value_meta, ensure_ascii=False)) id_bytes = bytearray.fromhex(metric.id) if self._metric_id_cache.get(metric.id, None): measurement_bound_stmt = self._measurement_update_stmt.bind((self._retention, metric.value, metric.value_meta, id_bytes, metric.time_stamp)) self._metric_batch.add_measurement_query(measurement_bound_stmt) metric_update_bound_stmt = self._metric_update_stmt.bind((self._retention, metric.time_stamp, metric.region, metric.tenant_id, metric.name, metric.dimension_list, metric.dimension_names)) self._metric_batch.add_metric_query(metric_update_bound_stmt) return metric self._metric_id_cache[metric.id] = metric.id metric_insert_bound_stmt = self._metric_insert_stmt.bind((self._retention, id_bytes, metric.time_stamp, metric.time_stamp, metric.region, metric.tenant_id, metric.name, metric.dimension_list, metric.dimension_names)) self._metric_batch.add_metric_query(metric_insert_bound_stmt) for dim in metric.dimension_list: (name, value) = dim.split('\t') dim_key = self._get_dimnesion_key(metric.region, metric.tenant_id, name, value) if not self._dimension_cache.get(dim_key, None): dimension_bound_stmt = self._dimension_stmt.bind((metric.region, metric.tenant_id, name, value)) self._metric_batch.add_dimension_query(dimension_bound_stmt) self._dimension_cache[dim_key] = dim_key metric_dim_key = self._get_metric_dimnesion_key( metric.region, metric.tenant_id, metric.name, name, value) if not self._metric_dimension_cache.get(metric_dim_key, None): dimension_metric_bound_stmt = self._dimension_metric_stmt.bind( (metric.region, metric.tenant_id, name, value, metric.name)) self._metric_batch.add_dimension_metric_query(dimension_metric_bound_stmt) metric_dimension_bound_stmt = self._metric_dimension_stmt.bind( (metric.region, metric.tenant_id, metric.name, name, value)) self._metric_batch.add_metric_dimension_query(metric_dimension_bound_stmt) self._metric_dimension_cache[metric_dim_key] = metric_dim_key measurement_insert_bound_stmt = self._measurement_insert_stmt.bind( (self._retention, metric.value, metric.value_meta, metric.region, metric.tenant_id, metric.name, metric.dimension_list, id_bytes, metric.time_stamp)) self._metric_batch.add_measurement_query(measurement_insert_bound_stmt) return metric def write_batch(self, metrics): with self._lock: batch_list = self._metric_batch.get_all_batches() results = execute_concurrent(self._session, batch_list, raise_on_first_error=True) self._handle_results(results) self._metric_batch.clear() LOG.info("flushed %s metrics", len(metrics)) @staticmethod def _handle_results(results): for (success, result) in results: if not success: raise result def _load_dimension_cache(self): rows = self._session.execute(RETRIEVE_DIMENSION_CQL) if not rows: return for row in rows: key = self._get_dimnesion_key(row.region, row.tenant_id, row.name, row.value) self._dimension_cache[key] = key LOG.info( "loaded %s dimension entries cache from database into cache." % self._dimension_cache.currsize) @staticmethod def _get_dimnesion_key(region, tenant_id, name, value): return '%s\0%s\0%s\0%s' % (region, tenant_id, name, value) def _load_metric_dimension_cache(self): qm = token_range_query_manager.TokenRangeQueryManager(RETRIEVE_METRIC_DIMENSION_CQL, self._process_metric_dimension_query) token_ring = self._cluster.metadata.token_map.ring qm.query(token_ring) def _process_metric_dimension_query(self, rows): cnt = 0 for row in rows: key = self._get_metric_dimnesion_key( row.region, row.tenant_id, row.metric_name, row.dimension_name, row.dimension_value) self._metric_dimension_cache[key] = key cnt += 1 LOG.info("loaded %s metric dimension entries from database into cache." % cnt) LOG.info( "total loaded %s metric dimension entries in cache." % self._metric_dimension_cache.currsize) @staticmethod def _get_metric_dimnesion_key(region, tenant_id, metric_name, dimension_name, dimension_value): return '%s\0%s\0%s\0%s\0%s' % (region, tenant_id, metric_name, dimension_name, dimension_value)
def __init__(self, max_dim): """ :param max_dim: """ self.cache = LRUCache(maxsize=max_dim)
def popitem(self): key, val = LRUCache.popitem(self) val.Reset() return key, val