def worker(queue, eviction_policy): timings = {'get': [], 'set': [], 'delete': []} cache = Cache('tmp', eviction_policy=eviction_policy) for index, (action, key, value) in enumerate(iter(queue.get, None)): start = time.time() if action == 'set': cache.set(key, value, expire=EXPIRE) elif action == 'get': result = cache.get(key) else: assert action == 'delete' cache.delete(key) stop = time.time() if action == 'get' and PROCESSES == 1 and THREADS == 1 and EXPIRE is None: assert result == value if index > WARMUP: timings[action].append(stop - start) queue.put(timings) cache.close()
class URLhaus: """Simple client to query URLhaus by abuse.ch. :param query: domain, url or hash. :param cache_duration: Duration before refreshing the cache (in seconds). Ignored if `cache_duration` is 0. :param cache_root: Path where to store the cached file. :type query: string :type cache_duration: int :type cache_root: str """ def __init__(self, query, cache_duration=300, cache_root="/tmp/cortex/URLhaus"): self.URL = "https://urlhaus.abuse.ch/browse.php" self.query = query self.cache = None if cache_duration > 0: self.cache = Cache(cache_root) self.cache_duration = cache_duration def _get_raw_data(self): try: return self.cache[self.query.encode('utf-8')] except(AttributeError, TypeError): return self.fetch() except KeyError: self.cache.set( self.query.encode('utf-8'), self.fetch(), expire=self.cache_duration) return self.cache[self.query.encode('utf-8')] def search(self): res = self._get_raw_data() return self.parse(res) def fetch(self): payload = {"search": self.query} return requests.get(self.URL, params=payload).text def parse(self, doc): results = [] soup = BeautifulSoup(doc, "html.parser") table = soup.find("table", class_="table") rows = table.find_all("tr")[1:] for row in rows: cols = row.find_all("td") results.append({ "dateadded": cols[0].text, "malware_url": cols[1].text, "link": cols[1].find("a").attrs.get("href"), "status": cols[2].text, "tags": cols[3].text.split(), "gsb": cols[4].text, "reporter": cols[5].text }) return results
def _set_cached(path, content): # 1/ memory cache JUMBO_FIELDS_MEMORY_CACHE[path] = content # 2/ disk cache if SIMPLEFLOW_ENABLE_DISK_CACHE: try: cache = Cache(constants.CACHE_DIR) cache_key = "jumbo_fields/" + path.split("/")[-1] logger.debug("diskcache: setting key={} on cache_dir={}".format(cache_key, constants.CACHE_DIR)) cache.set(cache_key, content, expire=3 * constants.HOUR) except OperationalError: logger.warning("diskcache: got an OperationalError on write, skipping cache write")
def _set_cached(path, content): # 1/ memory cache JUMBO_FIELDS_MEMORY_CACHE[path] = content # 2/ disk cache if SIMPLEFLOW_ENABLE_DISK_CACHE: try: cache = Cache(constants.CACHE_DIR) cache_key = "jumbo_fields/" + path.split("/")[-1] logger.debug("diskcache: setting key={} on cache_dir={}".format( cache_key, constants.CACHE_DIR)) cache.set(cache_key, content, expire=3 * constants.HOUR) except OperationalError: logger.warning( "diskcache: got an OperationalError on write, skipping cache write" )
class Cache(object): def __init__(self): from diskcache import Cache self.cache = Cache('/tmp/navan') self.cache.stats(enable=True) def get(self, *args): return self.cache.get(':'.join(args)) def set(self, *args, **kwargs): expire = kwargs.get('expire') if len(args) < 2: raise Exception('cache set must contain `key` and `value`') key, value = args[:-1], args[-1] key = ':'.join(key) return self.cache.set(key, value, expire) def get_json(self, *args): ret = self.get(*args) if not ret: return ret return json.loads(ret) def set_json(self, *args, **kwargs): args = list(args) args[-1] = json.dumps(args[-1]) return self.set(*args, **kwargs)
class DiscCacheWrapper(): DISC_CACHE_PATH = "./tmp/flickr-auth-disc-cache" def __init__(self): self.disc_cache = Cache(directory=DiscCacheWrapper.DISC_CACHE_PATH) def get(self, key): # This can raise a diskcache.Timeout error if it fails to talk to its database return self.disc_cache.get(key=key, default=None, retry=False) def set(self, key, value, timeout=None): # This can raise a diskcache.Timeout error if it fails to talk to its database self.disc_cache.set(key=key, value=value, expire=timeout, retry=False) def delete(self, key): self.disc_cache.delete(key=key)
class Cache(object): def __init__(self): try: self.cache = DC('./tmp') except Exception as ex: print('Get an exception with diskcache open: {}'.format(ex)) self.cache = None def __del__(self): try: self.cache.close() except Exception as ex: print('Get an exception with diskcache close: {}'.format(ex)) def set(self, key, value): if self.cache is not None: self.cache.set(key, BytesIO(value), read=True, tag=u'data') def get(self, key): if self.cache is not None: value = self.cache.get(key, default=b'', read=True, tag=True) if value is not None and value != b'': return value return None def pop(self, key): if self.cache is not None: value = self.cache.pop(key, default=b'', read=True, tag=True) if value is not None and value != b'': return value return None def delete(self, key): if self.cache is not None: self.cache.delete(key) def create_index(self): if self.cache is not None: self.cache.create_tag_index() return self.cache.tag_index return None def clear_all(self): if self.cache is not None: self.cache.clear()
def get_ingress_status(domain, code): cache = Cache('/tmp/check_k8s_status.cache') data_cache = cache.get(domain, default=b'', read=True, expire_time=False) if domain in cache: # print(data_cache) if code in data_cache: print(data_cache[code]) else: print(0.0) exit(0) data = get_status_code_ratio(domain) expire_time = es_query_duration / 1000 cache.set(domain, data, expire=expire_time) if code in data: print(data[code]) else: print(0.0)
class DiskCache(): def __init__(self, cache_dir, ttl=None): self.ttl = ttl self.cache = Cache(cache_dir, eviction_policy='least-recently-used') def __getitem__(self, key): return self.cache[key] def __setitem__(self, key, value): return self.cache.set(key, value, expire=self.ttl) def get(self, key, default=None): return self.cache.get(key, default=default) def set(self, key, value): return self.cache.set(key, value, expire=self.ttl) def clear(self): self.cache.clear()
class CachedClient(Client): def __init__(self, apikey, agent="unknown", host=None, cache_dir=None): super().__init__(apikey, agent=agent, host=host) self.cache = Cache(cache_dir, disk=VtCache, disk_compress_level=6, tag_index=True) self.cache_dir = cache_dir self.logger = logging.getLogger('kfinny.cachedvt.CachedClient') def _get(self, resource): data, tag = self.cache.get(resource, tag=True) if data and tag in ['sha1', 'md5']: data, tag = self.cache.get(data, tag=True) if data and tag == 'object': data = Object.from_dict(data) return data, tag def _put_object(self, obj): self.cache.set(obj.sha256, obj.to_dict(), tag='object') self.cache.set(obj.sha1, obj.sha256, tag='sha1') self.cache.set(obj.md5, obj.sha256, tag='md5') def _put_error(self, resource, error): self.cache.set(resource, { 'resource': resource, 'code': error.code, 'message': error.message }, tag='error') def yield_file_report(self, resource, include_notfound=False): queryset = set() if isinstance(resource, str): resource = resource.split(',') if isinstance(resource, (tuple, list, set, frozenset)): for r in resource: data, tag = self._get(r) if data is not None: if tag == 'object' or include_notfound: yield data else: queryset.add(r) resource = sorted(queryset) for i in resource: try: obj = self.get_object(f'/files/{i}') self._put_object(obj) yield obj except APIError as e: self._put_error(i, e) self.logger.debug("hits = {}, misses = {}".format(*self.cache.stats()))
class BaseCacheAnalyzer(BaseAnalyzer): def __init__(self, cache_location=None, force=False, delete_cache_when_done=False, **kwargs): super().__init__(**kwargs) self.cache_location = cache_location self.cache = None self.force = force self.delete_cache_when_done = delete_cache_when_done def initialize(self): from diskcache import Cache self.cache = Cache(self.cache_location or self.uid + "_cache") def filter(self, simulation): return self.force or not self.is_in_cache(simulation.id) def to_cache(self, key, data): self.cache.set(key, data) def from_cache(self, key): return self.cache.get(key) def is_in_cache(self, key): return key in self.cache def destroy(self): if self.cache: self.cache.close() if self.cache and self.delete_cache_when_done and os.path.exists( self.cache.directory): cache_directory = self.cache.directory del self.cache shutil.rmtree(cache_directory) @property def keys(self): return list(self.cache.iterkeys()) if self.cache else None
def main(source, clear_cache): comi_dir_path = '/tmp/comi' comi_dir_path_object = Path(comi_dir_path) comi_dir_path_object.mkdir(exist_ok=True) cache_dir_path = '/tmp/comi/cache' cache = Cache(cache_dir_path) if clear_cache: cache.clear() print('Cache cleared') return True if source: if 'github.com' in source and 'blob' in source: url = source.replace('blob', 'raw') else: url = source else: url = 'https://github.com/commmands/commands/raw/master/commands_1.commands' cache_value = cache.get(url) temp_commands_path = '/tmp/comi/temp_commands' temp_commands_path_object = Path(temp_commands_path) temp_commands_path_object.parent.mkdir(parents=True, exist_ok=True) if cache_value: temp_commands_path_object.write_text(cache_value) else: commands_response = requests.get(url) commands = commands_response.text temp_commands_path_object.write_text(commands) cache.set(url, commands) perl_part = "perl -e 'ioctl STDOUT, 0x5412, $_ for split //, do{ chomp($_ = <>); $_ }'" command = f"cat {temp_commands_path} | fzf --tac | {perl_part} ; echo" subprocess.call(command, shell=True)
class KeyValueDB(Generic[NativeType, StorageType], abc.ABC): """Interface for concrete DB backend.""" _native_type: NativeType _storage_type: StorageType def __init__(self, datebase_dir: Path): if not datebase_dir.exists(): datebase_dir.mkdir(mode=0o750, parents=True) self._cache = Cache(str(datebase_dir)) def __contains__(self, key: Any) -> bool: return self._cache.__contains__(key) def __delitem__(self, key: Any) -> bool: return self._cache.__delitem__(key) def __getitem__(self, key: Any) -> NativeType: return self._storage_to_native_type(self._cache.__getitem__(key)) def __setitem__(self, key: Any, value: NativeType) -> None: return self._cache.__setitem__(key, self._native_to_storage_type(value)) def _native_to_storage_type(self, value: NativeType) -> StorageType: if self._native_type is self._storage_type or self._storage_type is None: return cast(StorageType, value) else: return self._storage_type(value) def _storage_to_native_type(self, value: StorageType) -> NativeType: if self._native_type is self._storage_type or self._native_type is None: return cast(NativeType, value) else: return self._native_type(value) def close(self, *args, **kwargs) -> None: return self._cache.close() def get(self, key: Any, default: Any = None, *args, **kwargs) -> Union[Any, NativeType]: value = self._cache.get(key, default, *args, **kwargs) if value is default: return default else: return self._storage_to_native_type(value) def set(self, key: Any, value: NativeType, *args, **kwargs) -> bool: return self._cache.set(key, self._native_to_storage_type(value), *args, **kwargs) def touch(self, *args, **kwargs) -> bool: return self._cache.touch(*args, **kwargs)
class CacheWrapper: def __init__(self, path, size): self._cache = Cache(path, size_limit=size) def update_cache_entries(self, entries: List[KVPair]): """ Update cache entries :param entries: a list of pairs key-value :return: """ for entry in entries: self._cache.set(entry.key, entry.value) # ALWAYS override! def set_entry(self, entry: KVPair): self._cache.set(entry.key, entry.value) def get_cached_entries(self, keys: List[Any]) -> Tuple[List[Any], List[Any]]: """ Retrieve cached entries :param keys: a list of keys to retrieve :return: a tuple (<cached results>, <missing entries>) """ to_compute = [] cached_entries = [] for key in keys: entry = self._cache.get(key) if entry is None: to_compute.append(key) else: cached_entries.append(entry) return cached_entries, to_compute def get_cached_entry(self, key: Any) -> Any: return self._cache.get(key)
class DiskCachePersistence: def __init__(self, dbname, dbpassphrase): self.dbname = dbname self.dbpassphrase = dbpassphrase self.db = Cache(dbname) # TODO: create encrypted Cache with kdf dbpassphrase # TODO: purge expired skippedMessageKey based on store_time def save_conversation(self, conversation): return self.db.set(b'conv:' + conversation.ks['CONVid'], prefix='conv', tag='conv', retry=True) def load_conversation(self, conv_id): return self.db.get(b'conv:' + conv_id, None, retry=True) def delete_conversation(self, conversation): return self.db.pop(b'conv:' + conversation.ks['CONVid'], None, retry=True) def get_other_names(self, name): names = [] for k in self.db: if k.startswith('conv:'): names.append(self.db[k].other_name) return names
class CacheManager(object): def __init__(self, cache_path=CACHE_PATH): self.cache = Cache(cache_path) def items(self): return self.cache.iterkeys() def has_key(self, key): return key in self.cache def set(self, key, value, ttl=TTL): return self.cache.set(key=key, value=value, expire=ttl) def get(self, key): return self.cache.get(key=key) def clear_cache(self): for key in self.cache: if key != 'censys_credentials': del self.cache[key] return True def close(self): self.cache.close()
class DiskPubSubCache(object): """A DiskCache-backed cache used for PubSub channels Attributes: cache (Cache): The cache which backs this pubsub cache _subscribers (dict{str: DiskSubscription}): The subscriptions tracked by this cache _threads_registered (set(str)): The names of the threads which have registered triggers on the database _push_partial (func): The function called when an insert or update happens on the cache Args: directory (str): The path to the directory used by this cache timeout (float, optional): The number of seconds to wait before an operation times out. Defaults to 0.01 seconds """ _insert_func_name = 'push_on_insert' _update_func_name = 'push_on_update' def __init__(self, directory, timeout=0.01): self.cache = Cache(directory, timeout=timeout) self._subscribers = {} # Would be nice to use a weakref to a set so that keys with no subscribers are self._threads_registered = set() self._insert_triggers() self._push_partial = partial(self.__push_to_subscribers) def publish(self, channel, data): """Publish data to a channel Args: channel (str): Channel to publish the data to data: The data to publish the data to. The data will arrive in the same format as it was set Returns: (int): The number of subscribers which received the published data """ self.cache.set(channel, data) return len(self._subscribers.get(channel, [])) def register_callbacks(self): """Registers the trigger functions for the current thread. A thread must have trigger functions registered before it can publish data """ if threading.current_thread().name not in self._threads_registered: con = self._con for func_name in (self._insert_func_name, self._update_func_name): con.create_function(func_name, 2, self._push_partial) self._threads_registered.add(threading.current_thread().name) def _insert_triggers(self): """Inserts the original triggers into the cache, but does not create or the functions which receive the triggers """ con = self._con for func_name, operation in [(self._insert_func_name, 'INSERT'), (self._update_func_name, 'UPDATE')]: con.execute('CREATE TRIGGER IF NOT EXISTS {0} AFTER {1} ON Cache BEGIN ' 'SELECT {0}(NEW.key, NEW.value); END;'.format(func_name, operation)) def subscribe(self, channel): """Subscribe to a channel Args: channel (str): The name of the channel to subscribe to Returns: (DiskSubscription): The subscription to this channel """ subscription = DiskSubscription(channel) if channel not in self._subscribers: self._subscribers[channel] = WeakSet([subscription]) else: self._subscribers[channel].add(subscription) return subscription def __push_to_subscribers(self, channel, value): try: value = self.__get_value(value) for subscriber in self._subscribers.get(str(channel), []): subscriber.push(value) except: import traceback traceback.print_exc() raise @staticmethod def __get_value(value): if value == unsubscribe_message or isinstance(value, string_types) or isinstance(value, int) or isinstance( value, float): return value if isinstance(value, binary_type): return value.decode('utf-8') try: return pickle.load(BytesIO(value)) except (KeyError, TypeError, IndexError): return str(value) @property def _con(self): con = getattr(self.cache._local, 'con', None) if con is None: con = self.cache._local.con = sqlite3.connect( os.path.join(self.cache._directory, DBNAME), timeout=self.cache._timeout, isolation_level=None, ) # Some SQLite pragmas work on a per-connection basis so query the # Settings table and reset the pragmas. The Settings table may not # exist so catch and ignore the OperationalError that may occur. try: select = 'SELECT key, value FROM Settings' settings = con.execute(select).fetchall() except sqlite3.OperationalError: pass else: for key, value in settings: if key.startswith('sqlite_'): self.cache.reset(key, value, update=False) return con def shutdown(self): """Shuts down the connection to the cache """ self.cache.close()
future = loop.run_in_executor(None, cache.set, key, val) result = await future return result async def get_async(key): loop = asyncio.get_running_loop() future = loop.run_in_executor(None, cache.get, key) result = await future return result ## # Utility Constants ## cache.set("running", True) app.last_request = time.time() app.image_server = "https://s2.mangadex.org" app.api_server = "https://api.mangadex.network" ## # Sanic Utility Functions ## @app.middleware('request') async def add_start_time(request): request.ctx.start_time = time.time() @app.middleware('response') async def add_spent_time(request, response):
class Webserver: def __init__( self, host="localhost", port=8080, hostname='hostname', workers=os.cpu_count() - 1, max_connections=socket.SOMAXCONN, max_cache_size=4e9, proto='http', ): self._max_connections = max_connections self.host = host self.port = port self.address = f'{proto}://{self.host}:{self.port}' self.hostname = hostname self.max_workers = workers self.routes = {} self.vservers_routes = {} # роуты для вирт. сервера self.regular_routes = {} self._response = None self._current_clients = [] self.data_end = b'\r\n\r\n' self.connect_refresh_timeout = 10 self.client_request_timeout = 0.1 self._max_keep_alive = timedelta(seconds=10) self.cache = Cache(size_limit=int(max_cache_size)) self.cache_expire = 100 self._running = True def set_routes(self, routes): self.routes = {**routes, **self.routes} def get_routes(self): return self.routes def handle_file(self, request, filename, root=os.getcwd(), content_type=None): path = os.path.join(root, filename) # Пробуем взять отдаваемый стат. файл из cache res = self.cache.get(path) if res: return res if not os.path.exists(path): self.cache.set(path, Errors.NOT_FOUND_PAGE, expire=self.cache_expire, tag='data') self.cache.cull() return Errors.NOT_FOUND_PAGE if content_type is None: content_type, _ = mimetypes.guess_type(path) logger.verbose('Content type for %s, was set automatically to %s', path, content_type) response = Response.response_file(request, path, content_type) if response.status == 200: # Если код 200 OK, то добавим response в cache self.cache.set(path, response, expire=self.cache_expire, tag='data') self.cache.cull() return response def handle_dir(self, request, dirname=os.getcwd()): path = os.path.abspath(dirname) if not os.path.exists(path): return Errors.NOT_FOUND_PAGE response = Response.response_dir(request, path) return response def get_routes_for_virtual_server(self, request): vserver_name = request.headers['Host'] routes = self.vservers_routes.get(vserver_name) if routes: logger.debug('Find routes for vservers %s', vserver_name) return routes return self.regular_routes def find_handler(self, request: Request): logger.debug('%s', self.regular_routes) try: response = Errors.NOT_FOUND_PAGE for path_info, handler in self.get_routes_for_virtual_server( request).items(): logger.debug('checking %s %s | %s', path_info.path, path_info.method, handler) path_info: PathInfo = path_info reg = path_info.path match = re.fullmatch(reg, request.url.path) if match and request.method == path_info.method: logger.debug('handler found %s for %s', handler, request.url) if len(match.groupdict().items()) == 0: response = handler(request) else: response = handler(request, match.group(1)) break logger.debug('handler returned %s', response) return response except Exception: logger.exception("Internal Error") return Errors.INTERNAL_SERVER_ERROR def _read_data(self, client, address): _keep_alive_deadline = datetime.now() + self._max_keep_alive request = Request() filled = False while not filled: try: line = client.recv(64 * 1024) if not line: raise ReceiveDone('Client done') split = Request.split_keep_sep(line, bytes(os.linesep, 'utf-8')) for s in split: if request.dynamic_fill(s): filled = True except BlockingIOError: # читали, а нам никто не написал, подождем и может напишут time.sleep(0.5) _current_time = datetime.now() if _current_time > _keep_alive_deadline: # ждали больше чем держим keep-alive соединение raise ConnectionError('Client dead') return request def _start_session(self, client, address): while True: request = self._read_data(client, address) request.print_headers() request.print_body() response = self.find_handler(request) logger.access(method=request.method, url=request.target, protocol=request.version, status=response.status) Response.response(client, response) # закончили обрабатывать запрос от клиента # (получили, ответили, пришли сюда) if request.headers.get('Connection') == 'keep-alive': logger.verbose("Client %s asked for keep-alive connection", address) # если браузер послал keep-alive (что он почти всегда # делает дабы избавиться от поспоянных созданий # подключений и ускорить общение с сервером) client.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) # и продолжаем крутиться в while True: else: # иначе выходим из while True: (клиент не хочет больше # с нами общаться) break def handle(self, client, address): init_thread_local(THREAD_LOCAL_DATA) THREAD_LOCAL_DATA.client = address with client: try: self._start_session(client, address) except ConnectionError: logger.verbose('Client was not alive for %s seconds', self._max_keep_alive) except OSError: logger.exception('Client had an issue') except ReceiveDone: logger.verbose('Client received the content') except Exception: logger.exception('Client had an error') Response.response(client, Errors.INTERNAL_SERVER_ERROR) logger.verbose("Disconnected") def run(self, stop: Event): self._running = True server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.make_regular_routes() with server_socket: server_socket.bind((self.host, self.port)) server_socket.listen(self._max_connections) server_socket.setblocking(False) logger.verbose(f'Start server on {self.address}') logger.debug('T %s', server_socket.gettimeout()) with ThreadPoolExecutor(max_workers=self.max_workers) as executor: logger.verbose('ThreadPoolExecutor started') while not stop.is_set(): try: self._check_is_running() try: client, address = server_socket.accept() THREAD_LOCAL_DATA.client = address logger.verbose(f'Got client') future = executor.submit(self.handle, client, address) self._current_clients.append(future) except (socket.timeout, BlockingIOError): # никто не подключился, можно подождать # logger.info(f'No client') time.sleep(1) pass except KeyboardInterrupt: logger.verbose('Server shutdown started') break logger.verbose(('ThreadPoolExecutor ' 'waiting for alive connections')) for future in self._current_clients: result = future.result() logger.verbose('ThreadPoolExecutor finished') def _check_is_running(self): if not self._running: raise KeyboardInterrupt def stop(self): self._running = False def route(self, path, method='GET', vserver='localhost'): logger.debug('Adding %s [%s]', path, method) if path in self.routes: raise AssertionError("Such route already exists.") def wrapper(handler): path_info = PathInfo(path, method) if vserver == 'localhost': self.set_routes({path_info: handler}) if vserver in self.vservers_routes: self.vservers_routes[vserver][path_info] = handler logger.debug('Add route %s for vserver %s', path_info, vserver) else: self.vservers_routes[vserver] = {path_info: handler} logger.debug('INIT route %s for vserver %s', path_info, vserver) return handler return wrapper def make_regular_routes(self): logger.debug('Processing routes %s', self.routes) for path_info, handler in self.routes.items(): reg = re.compile(path_info.path) path_info.path = reg self.regular_routes[path_info] = handler
class PandaXT: """A "ccxt" exchanges wrapper class over Pandas lib.""" def __init__(self, exchange, api_key=None, secret=None, user_agent=None, clear_cache=False) -> NoReturn: """Class constructor. >>> markets = PandaXT('binance').markets >>> isinstance(markets, Markets) True >>> market = markets.get('BTC/USDT') # type: Market >>> isinstance(market, Market) True >>> isinstance(market.precision.price, int) True :param Text exchange: a ccxt lib supported exchange :param Text api_key: API key :param Text secret: API secret :param U[Bool, Text] user_agent: if True, exchange API keys will be load from "$HOME/.env" file. :param Bool clear_cache: if True, current cache will be ignored and overwrite. """ # for cache purposes self._basemarkets = None self._currencies = None self._symbols = None self._markets = None if clear_cache: self._cache.clear() exchange = _get_version(exchange) # assert exchange is not None, f'"{exchange}" not supported' api = getattr(ccxt, exchange) settings = dict(timeout=25000, enableRateLimit=True, fetchTickersErrors=False) if user_agent or False: if isinstance(user_agent, bool): settings.update(userAgent=api.userAgents[1]) else: settings.update(userAgent=str(user_agent)) if not all((api_key, secret)): load_dotenv() field_name = exchange.upper().strip(' _012345') key_field, secret_field = f'{field_name}_KEY', f'{field_name}_SECRET' api_key = os.environ.get(key_field) secret = os.environ.get(secret_field) if api_key and secret: settings.update(apiKey=api_key, secret=secret) if exchange == 'binance': # noinspection PyUnresolvedReferences settings.update(adjustForTimeDifference=True, parseOrderToPrecision=True) self._api = api(settings) def _load_markets(self) -> Markets: """Markets metadata cache handler. >>> isinstance(PandaXT('binance')._load_markets(), Markets) True :return: markets metadata as "Markets" instance. """ if self._markets is not None: data = self._markets else: self._cache_dir = _DATA_DIR / 'pandaxt' / self.id self._cache_dir.mkdir(exist_ok=True, parents=True) self._cache = Cache(str(self._cache_dir)) data = self._cache.get('markets', dict()) if len(data) == 0: data = self._api.load_markets() data = {k: {x: y for x, y in v.items() if y} for k, v in data.items() if v} self._cache.set('markets', data, (60.0 ** 2.0) * 6.0) return Markets(**data) @property def id(self) -> Text: """Exchange unique reference (also know as ID). >>> PandaXT('binance').id 'binance' :return: exchange unique reference. """ return self._api.id @property def timeframes(self) -> List[Text]: """Return valid exchange timeframes as list. >>> '15m' in PandaXT('binance').timeframes True :return: valid exchange timeframes. """ items = self._api.timeframes.items() od = OrderedDict(sorted(items, key=lambda x: x[1])) return list(od.keys()) @property def delisted(self) -> Symbols: """Returns delisted symbols (active -> False) >>> binance = PandaXT('binance') >>> delisted = binance.delisted >>> isinstance(delisted, list) True :return: return delisted symbols """ return Symbols(*[m for m in self.markets if hasattr(m, 'active') and not m.active]) @property def name(self) -> Text: """Exchange long name. :return: exchange long name. """ return getattr(self._api, 'name') @property def symbols(self) -> Symbols: """Get all supported symbols by exchange as "Symbol" list.""" if self._symbols is None: self._symbols = sorted([m for m in self.markets if m not in self.delisted]) return Symbols(self._symbols) @property def base_markets(self) -> Currencies: """Get exchange base markets currencies as "Currency" list.""" if self._basemarkets is None: self._basemarkets = sorted(list({s.quote for s in self.symbols})) return Currencies(self._basemarkets) @property def currencies(self) -> Currencies: """Get supported currencies by exchange as "Currency" list. >>> exchange = PandaXT('binance') >>> len(exchange.currencies) > 0 True :return: all active currencies supported by exchange. """ # Initialize markets, symbols, currencies and basemarkets if self._currencies is None: self._currencies = [s.base for s in self.symbols] self._currencies = self._currencies + list(self.base_markets) self._currencies = sorted(list(set(self._currencies))) return Currencies(self._currencies) def get_market_precision(self, symbol, precision_type=None) -> Int: """Get precision set by exchange for a market. >>> PandaXT('binance').get_market_precision('MATIC/USDT') :param symbol: symbol of the market where precision will be return. :param precision_type: accepted types: "price", "amount", "cost", "base", "quote" (default "price") :return: precision length for the supplied market symbol. """ market: Market = self.markets.get(Symbol(symbol)) precision = market.precision return getattr(precision, precision or 'price') @property def markets(self) -> Markets: """Get all exchange markets metadata. >>> exchange = PandaXT('binance') >>> markets = exchange.markets >>> isinstance(markets, Markets) and len(markets) > 1 True :return Dict: all exchange markets metadata. """ if self._markets is None: self._markets = self._load_markets() return self._markets def parse_timeframe(self, timeframe) -> U[Int, NoReturn]: """Get amount of seconds for a supplied string formatted timeframe. :param timeframe: a supported and valid string formatted timeframe (example: "15m") :return: None if timeframe is not supported otherwise, amount of seconds for supplied timeframe. """ if hasattr(self._api, 'parse_timeframe'): return self._api.parse_timeframe(timeframe) def get_timeframe(self, timeframe) -> Text: """Timeframe sanitizer. >>> PandaXT('binance').get_timeframe(15) '15m' :param timeframe: timeframe to sanitize. :type timeframe: Text or Int :return str: sanitize timeframe. """ timeframe = str(timeframe) if timeframe.isdigit() or not timeframe[-1].isalpha(): timeframe = f'{timeframe}m' if timeframe not in self.timeframes: raise TimeframeError(timeframe, exchange=self.name) else: return timeframe # @check_symbol def get_ohlc(self, symbol, timeframe='15m', limit=25) -> pd.DataFrame: """Get OHLC data for specific symbol as pandas DataFrame type. :param Text symbol: symbol name use at ohlc data request. :param Text timeframe: an exchange supported timeframe. :param int limit: max rows limit. :return pd.DataFrame: data-frame with: open, high, low, close, volume, qvolume columns and 'date' as index. """ if Symbol(symbol) not in self.symbols: # print(symbol, symbol in self.symbols, len(self.symbols)) raise SymbolError(symbol, exchange=self.name) data = self._api.fetch_ohlcv(symbol, timeframe=timeframe, limit=limit) df = pd.DataFrame(data, columns=_OHLC_FIELDS) df['qvolume'] = df['volume'] * df['close'] df.index = pd.to_datetime(df.pop('date') // 1000, unit='s') df.name = 'date' return df def get_currency(self, currency) -> Currency: """Currency name sanitizer.""" if currency: if currency in self._api.commonCurrencies: currency = self._api.commonCurrencies.get(currency) if currency not in self.currencies: log.debug(f'Currency {str(currency)} is not supported by exchange.') return Currency(currency) def altname(self, name) -> U[Currency, Symbol]: """Retrieve alternative currency or symbol name used in a specific exchange. >>> PandaXT('binance').altname('YOYOW') 'YOYOW' >>> PandaXT('binance').altname('YOYOW/BTC') (Symbol:YOYOW/BTC) >>> PandaXT('binance').altname('YOYOW/BTC') (Symbol:YOYOW/BTC) :param name: currency or symbol name to check for alternative name. :type name: Text or Currency or Symbol :return: currency alternative name as Currency or Symbol instance. :rtype: Currency or Symbol """ _symbol = str(name).upper() if '/' in _symbol.strip('/'): # print(_symbol) base, quote = _symbol.split('/') assert quote in self.base_markets, f'{quote} is not a valid base market.' base = self.get_currency(base) s = self.get_currency(base) + Currency(quote) s.price2precision = functools.partial(self.price2precision, s) s.cost2precision = functools.partial(self.cost2precision, s) s.amount2precision = functools.partial(self.amount2precision, s) return s else: return self.get_currency(_symbol) def cost2precision(self, symbol, cost, to_str=True) -> U[Float, Text]: """Return cost rounded to symbol precision exchange specifications. :param symbol: a valid exchange symbol. :type symbol: Text or Symbol :param Float cost: cost to be rounded. :param Bool to_str: True to return result as str, otherwise result will be returned as float :return float: cost rounded to specific symbol exchange specifications. """ result = self._api.cost_to_precision(symbol, cost) return result if to_str else float(result) def amount2precision(self, symbol, amount, to_str=True) -> U[Float, Text]: """Return amount rounded to symbol precision exchange specifications. :param symbol: a valid exchange symbol. :type symbol: Text or Symbol :param Float amount: amount to be rounded. :param Bool to_str: True to return result as str, otherwise result will be returned as float :return: amount rounded to specific symbol exchange specifications. :rtype: float or Text """ result = self._api.amount_to_precision(symbol, amount) return result if to_str else float(result) def price2precision(self, symbol, price, to_str=True) -> U[Float, Text]: """Return price rounded to symbol precision exchange specifications. :param symbol: a valid exchange symbol. :type symbol: Text or Symbol :param price: price to be rounded. :param Bool to_str: True to return result as str, otherwise result will be returned as float :return float: price rounded to specific symbol exchange specifications. """ result = self._api.price_to_precision(symbol, price) return result if to_str else float(result) # m = self.markets.get(symbol) # type: Market # t = str if to_str else float # template = '{:.@f}'.replace('@', str(m.precision.price)) # return t(template.format(float(price))) # @check_symbol def get_orderbook(self, symbol, limit=5) -> pd.DataFrame: """Get order book data for a symbol. >>> book_entries = PandaXT('binance').get_orderbook('MDA/BTC', 5) >>> isinstance(book_entries['ask'].values[-1], float) True :param symbol: a valid exchange symbol. :type symbol: Text or Symbol :param int limit: limit entries returned to "limit" value. :return: DataFrame type with order book data for "symbol". :rtype: pd.DataFrame """ raw = self._api.fetch_order_book(symbol, limit=int(limit)) columns = ['ask', 'ask_vol', 'bid', 'bid_vol'] data = [ask + bid for ask, bid in zip(raw['asks'], raw['bids'])] df = pd.DataFrame(data, columns=columns) return df.round({'ask': 8, 'bid': 8, 'ask_vol': 0, 'bid_vol': 0}) def get_tickers(self, symbols, sorted_by=None) -> U[Ticker, Tickers]: """Get tickers dict with symbol name as keys for all symbols specified as positional args. >>> exchange = PandaXT('binance') >>> ticker = exchange.get_tickers('ADA/BTC', sorted_by='percentage') >>> isinstance(ticker, Ticker) True :param symbols: list of valid exchange symbols. :param sorted_by: a valid ticker field like percentage, last, ask, bid, quoteVolume, ... :return: Ticker or Tickers instance with returned data. :rtype: Ticker or Tickers """ result = list() if isinstance(symbols, (Symbols, List, Tuple, Set)) and len(symbols): symbols = [symbols] if isinstance(symbols, str) else list(symbols) for s in map(self.altname, symbols): if s not in self.markets: log.debug(f'Symbol {s or "NULL"} is not listed in {self.name} exchange.') continue try: raw = self._api.fetch_tickers(symbols) if len(raw) > 0: if str(sorted_by) in list(raw.values())[0].keys(): raw = OrderedDict(sorted(raw.items(), key=lambda k: k[1][sorted_by], reverse=True)) result = Tickers(**raw) result = result[symbols[0]] if len(symbols) == 1 else result except ccxt.ExchangeError as err: print(str(err)) return result def get_indicators(self, indicators, symbol, timeframe='15m', limit=25, **kwargs) -> Dict[Text, pd.Series]: """Get technical indicators value for a symbol. :param Dict indicators: indicators and their params as dict (params ara mandatory, there is no default values). :param Text symbol: a valid exchange symbol. :param Text timeframe: an exchange valid timeframe (default 15m). :param Int limit: a valid exchange limit for returned rows (check exchange official API) supplied as a param / value dict instance also. Example: "{'roc': {'period': 9}}" :param kwargs: if "ohlc" is set with OHLC data (DataFrame) it will be use for value calculations. :return: dict type with indicators name/value pairs. """ indicators = {k.lower(): v for k, v in indicators.items()} symbol = Symbol(symbol) return_value = OrderedDict.fromkeys(indicators.keys()) supported_ti = [_a for _a in dir(tulipy.lib) if _a[0].islower()] functions = OrderedDict({i: getattr(tulipy, i) for i in indicators if i in supported_ti}) data = kwargs.get('ohlc', self.get_ohlc(symbol, timeframe=timeframe, limit=limit)) for n, fn in functions.items(): inputs = ['close' if i in 'real' else i for i in fn.inputs] indicator_params = dict() if len(fn.options): options = [opt.replace(' ', '_') for opt in fn.options] indicator_params = indicators.get(n) indicator_params = {k: v for k, v in indicator_params.items() if k in options} try: raw = fn(*data[inputs].T.values, **indicator_params) di = data.index if n in 'roc': raw = raw * 100.0 sr = pd.Series(raw, name=n.upper()) sr.index = di.values[-len(sr):] return_value[n] = sr.copy(True) except InvalidOptionError as err: print(str(err)) return dict(ohlc=data, **{k: return_value[k.lower()] for k in indicators}) def create_market_order(self, symbol, side, amount=None) -> Dict: """Create a market order order. :param Text symbol: a valid exchange symbol. :param Text side: accepted values: "buy", "sell" :param float amount: amount used in order creation. :return: order creation result data as dict. """ return self.create_order(symbol=symbol, side=side, order_type='market', amount=amount) def buy(self, symbol, amount=None, price=None) -> Dict: """Create buy order. :param Text symbol: a valid exchange symbol. :param float amount: amount to buy or None to auto-fill :param float price: buy price or None to auto-fill :return Dict: order creation result data as dict. """ return self.create_order(symbol=symbol, order_type='limit', side='buy', amount=amount, price=price) def create_order(self, symbol, side, order_type=None, amount=None, price=None) -> Dict: """Create a new order. :param Text symbol: symbol to be use for order creation. :param Text side: order side: 'sell' or 'buy' :param Text order_type: order type (default 'limit') :param float amount: amount used in order creation. :param float price: price used in order creation. :return: order result info as dict. """ symbol = Symbol(symbol) response = dict() if symbol not in self.symbols: raise SymbolError(f'Invalid symbol: {symbol}') if side not in ['buy', 'sell']: raise SideError(side) currency = symbol.quote if side in 'buy' else symbol.base balance_field = 'free' if side in 'buy' else 'total' ticker_field = 'ask' if side in 'buy' else 'bid' amount = magic2num( amount or self.get_balances(balance_field).get( currency, 0.0)) if amount > 0.0: price = magic2num(price or self.get_tickers(symbol).get(ticker_field)) if side in 'buy': amount = amount / price try: response = self._api.create_order(symbol=symbol, type=order_type or 'limit', side=side, amount=amount, price=price) except ccxt.InvalidOrder as err: print(f' - [ERROR] {str(err)}', file=sys.stderr) response = dict() return response def sell(self, symbol, amount=None, price=None) -> Dict: """Create sell order. :param Text symbol: a valid exchange symbol. :param Float amount: amount to sell or None to auto-fill :param Float price: sell price or None to auto-fill :return dict: order creation result data as dict. """ return self.create_order(symbol=symbol, order_type='limit', side='sell', amount=amount, price=price) def get_balances(self, field=None, tradeables_only=True, fiat=None) -> Wallet: """Get balances. >>> balances = PandaXT('binance').get_balances('total') >>> isinstance(balances, Wallet) True :param Text field: accepted values: if None all balances will be loaded, (values; "total", "used", "free") :param Bool tradeables_only: :return: positive balances. """ def is_zero(v) -> U[Float, Dict]: if isinstance(v, float): return v <= 0.0 elif isinstance(v, dict): return v.get('total', 0.0) <= 0.0 else: return 0.0 def is_tradeable(currency, _tickers, balance, base_market=None) -> Bool: """Check if a currency balance is over minimum tradeable amount for a base market. :param Text currency: :param Tickers _tickers: :param balance: :type balance: Dict or Float :param Text base_market: :return: True if currency is tradeable """ if currency in self.base_markets: return True base_market = Currency(base_market or 'BTC') symbol: Symbol = self.altname(currency) + base_market market: Market = self.markets.get(symbol, False) if not market: return False # min amount in quote currency limits: Limit = market.limits quote_min_amount = limits.amount ticker: Ticker = _tickers[symbol] if currency == 'BTC' and 'USD' in base_market: last = 1.0 / ticker.last else: last = ticker.last # converting min_amount to base currency base_min_amount = last * quote_min_amount # subtracting a 0.01% fee base_min_amount = base_min_amount * 0.999 if isinstance(balance or [], dict): balance = balance.get('total', 0.0) else: balance = balance or 0.0 return balance > base_min_amount try: data = self._api.fetch_balance() if 'info' in data: del data['info'] data: dict = data.pop(field) if field else data data = {str(k): v for k, v in data.items() if not is_zero(v)} if tradeables_only: symbols = {Currency(c) + Currency('BTC') for c in data} tickers = self.get_tickers(symbols) data = {str(k): v for k, v in data.items() if k in self._basemarkets or is_tradeable(k, tickers, v)} wallet = Wallet(**data) if str(fiat or '').lower() in ['EUR', 'USD']: acum = 0 for k, v in Wallet(**data).items(): if fiat.lower() == 'eur': acum += v.to_eur elif fiat.lower() == 'usd': acum += v.to_usd wallet.update({fiat.upper(): acum}) return wallet except ccxt.NetworkError as err: print('PandaXT::get_ohlc', str(err)) def get_balance(self, currency, field=None) -> Balance: """Get balance for a currency. >>> PandaXT('binance').get_balance('STORJ', 'total') :param Text currency: a valid exchange currency. :param Text field: accepted values: total, used, free :return: currency with balance amount as float. """ currency = self.altname(str(currency)) if currency not in self.currencies: raise CurrencyError(currency) if field and field in ['total', 'free', 'used']: field = field else: field = None balance_data = self.get_balances(field=field) or Balance( **{'currency': currency, field: 0.0}) if currency not in balance_data: raise ccxt.InsufficientFunds() else: return balance_data[currency] def get_user_trades(self, symbol, side=None, limit=25) -> pd.DataFrame: """Get user trades filter by symbol. :param Text symbol: a valid exchange symbol :param Text side: "buy" or "sell" :param int limit: a valid limit for rows return (please, refer to official exchange API manual for details) :return pd.DataFrame: user trades as pandas DataFrame type. """ symbol = str(symbol).upper() if symbol not in (self.symbols, self.altname(symbol) or ''): raise SymbolError(symbol) else: symbol = Symbol( symbol) if symbol in self.symbols else self.altname(symbol) trades = self._api.fetch_my_trades(symbol, limit=limit) if trades: trades = [{k: v for k, v in t.items() if k not in 'info'} for t in trades] for idx, t in enumerate(trades.copy()): trades[idx].update(total_cost=trades[idx]['fee']['cost']) del trades[idx]['fee'] trades = pd.DataFrame(trades) trades['real_cost'] = trades['cost'] + \ (trades['cost'] * trades['price']) # TODO: not totally true so revise it trades['real_price'] = trades['price'] * 1.001 trades['real_amount'] = trades['real_cost'] * trades['price'] if str(side).lower() in ['buy', 'sell']: trades = trades.query(f'side == "{str(side).lower()}"') return trades.sort_index(ascending=False) def get_trades(self, symbol, side=None, limit=25) -> pd.DataFrame: """Get latest user trades for a symbol. :param Text symbol: a valid exchange symbol :param Text side: "buy" or "sell" :param int limit: a valid limit for rows return (please, refer to official exchange API manual for details) :return: latest trades for a symbol. """ symbol = self.altname(symbol) trades = self._api.fetch_trades(symbol, limit=limit) if trades: trades = [{k: v for k, v in t.items() if k not in 'info' and v} for t in trades] trades = pd.DataFrame(trades).set_index('timestamp') if str(side).lower() in ['buy', 'sell']: trades = trades.query(f'side == "{str(side).lower()}"') return trades.sort_index(ascending=False) def get_cost(self, symbol, **kwargs) -> Float: """FIXME do not work well sometimes giving a bad result by unknown reason. Get weighted average (from buy trades data) cost for a symbol. >>> api = PandaXT('binance') >>> symbol_cost = api.get_cost('AGI/BTC') >>> symbol_cost True :param U[Symbol,Text] symbol: a valid exchange symbol. :param kwargs: accepted keys are: balance (Balance) and trades (pd.DataFrame) :return: cost calculation result as float type. """ symbol = self.altname(symbol) if isinstance(symbol or 0, Currency): symbol = symbol + Currency('BTC') base, quote = symbol.parts # reuse provided balance and trades data (for rate limit save) if 'balance' in kwargs: cached = kwargs.get('balance') # type: Balance balance = cached[base] if isinstance(cached, Wallet) else cached else: balance = self.get_balance(base, field='total') # type: Balance total_balance = balance.total if balance and hasattr( balance, 'total') else balance if total_balance > 0.0: trades = kwargs.get('trades', []) if not trades: trades = self.get_user_trades(symbol, side='buy') elif not isinstance(trades, pd.DataFrame): trades = pd.DataFrame(trades) if trades['order'].isna().all(): trades['order'].update(trades['id']) # group-by operations per column columns_op = {'amount': 'mean', 'price': 'mean', 'cost': 'mean', 'timestamp': 'mean'} trades = trades.groupby('order').agg(columns_op).sort_index( ascending=False) # type: pd.DataFrame trades = trades[['price', 'amount']].reset_index(drop=True) for index, price, amount in trades.itertuples(): if round(total_balance - amount, 8) <= 0.0: if round(total_balance - amount, 8) != 0.0: prices = trades[:index + 1]['price'].values amounts = trades[:index + 1]['amount'].values amounts[-1] = total_balance else: prices, amounts = trades[:index + 1].T.values return round(pd.np.average(prices, weights=amounts), 10) else: total_balance -= amount def get_order_status(self, order_id, symbol=None) -> Text: """Get order status by order_id. :param Text order_id: a valid order id. :param Text symbol: a valid exchange market :return: order status as str. Possible values are: "closed", "canceled", "open" """ return self._api.fetch_order_status(order_id, symbol=symbol) def get_open_orders(self, symbol=None, order_id=None) -> List[Dict[Symbol, Dict]]: """Get open orders.md for a symbol. :param Text symbol: symbol used in opened orders.md server query. :param order_id: just return data for a specific order. :type order_id: Text or int :return List: list of open orders.md for specific symbol. """ raw = self._api.fetch_open_orders(symbol or order_id) if isinstance(raw or 0, list) and len([r for r in raw if r]): return [{Symbol(k): v for k, v in r.items() if k not in 'info'} for r in raw] else: return list() def get_profit(self, currency) -> U[Tuple, Float]: """Returns current profit for a currency and its weighted average buy cost. :param Text currency: a valid currency to use at profit and cost calc :return: current profit and weighted average buy cost as a tuple """ currency = Currency(str(currency)) btc_symbol = currency + Currency('BTC') # type: Symbol balance = self.get_balance(currency) if balance.used > 0.0: cost = self.get_cost(symbol=btc_symbol) return balance.to_btc - (cost * balance.total) else: return 0.0, 0.0 def get_withdrawals(self): """Get withdrawals info. :return: """ if self._api.has['fetchWithdrawals']: result = self._api.fetch_withdrawals() return [{k: v for k, v in r.items() if k != 'info'} for r in result] else: raise ccxt.NotSupported('Withdrawals not supported by this exchange.') def get_transactions(self): """Get transactions info. :return: """ if self._api.has['fetchTransactions']: result = self._api.fetch_transactions() return [{k: v for k, v in r.items() if k != 'info'} for r in result] else: raise ccxt.NotSupported('Transactions not supported by this exchange.') def get_deposits(self): """Get deposits info. :return: """ if self._api.has['fetchDeposits']: resultb = self._api.fetch_withdrawals() return [{k: v for k, v in r.items() if k != 'info'} for r in result] else: raise ccxt.NotSupported('Deposits not supported by this exchange.') def cancel_order(self, symbol, last_only=False) -> List: """Cancel symbols open orders.md for a symbol. :param symbol: the symbol with open orders.md. :type symbol: Bool or Symbol :param Bool last_only: if True, only last order sent will be cancelled. :return: list of dict with data about cancellations. """ symbol = Symbol(symbol) pending_orders = self.get_open_orders(symbol) if len(pending_orders): if last_only: return self._api.cancel_order(pending_orders[-1]['id'], symbol) else: canceled_orders = list() for p in pending_orders: return_value = self._api.cancel_order(p['id'], symbol) if return_value and return_value.get( 'status', '') in 'cancel': canceled_orders.append( {k: v for k, v in return_value.items() if v}) else: self._api.cancel_order(p['id'], symbol) return canceled_orders def __str__(self) -> Text: """PandaXT instance as "str" type representation. >>> str(PandaXT('binance')) 'binance' :return Text: PandaXT instance as "str" type representation. """ return self.id def __repr__(self): """PandaXT instance as "str" type representation. >>> PandaXT('binance') binance :return str: PandaXT instance as "str" type representation. """ return self.id def __contains__(self, item) -> Bool: """Check if a symbol or currency is supported by exchange. >>> exchange = PandaXT('binance') >>> Currency('ETH') in exchange True >>> Currency('ETH/BTC') in exchange True >>> Currency('MyCOIN') in exchange False :param item: currency or symbol for supports checking on exchange. :type item: Text or Currency or Symbol :return bool: True is item is supported, otherwise False. """ return str(item) in self.markets or str(item) in map(str, self.currencies)
class FileDirCache(MutableMapping): def __init__( self, use_listings_cache=True, listings_expiry_time=None, listings_cache_location=None, **kwargs, ): """ Parameters ---------- use_listings_cache: bool If False, this cache never returns items, but always reports KeyError, and setting items has no effect listings_expiry_time: int or float (optional) Time in seconds that a listing is considered valid. If None, listings do not expire. listings_cache_location: str (optional) Directory path at which the listings cache file is stored. If None, an autogenerated path at the user folder is created. """ import appdirs from diskcache import Cache listings_expiry_time = listings_expiry_time and float(listings_expiry_time) if listings_cache_location: listings_cache_location = Path(listings_cache_location) / str(listings_expiry_time) listings_cache_location.mkdir(exist_ok=True, parents=True) else: listings_cache_location = Path(appdirs.user_cache_dir(appname="fsspec_dircache")) / str( listings_expiry_time ) try: listings_cache_location.mkdir(exist_ok=True, parents=True) except Exception: logger.error(f"folder for dircache could not be created at {listings_cache_location}") self.cache_location = listings_cache_location self._cache = Cache(directory=listings_cache_location) self.use_listings_cache = use_listings_cache self.listings_expiry_time = listings_expiry_time def __getitem__(self, item): """Draw item as fileobject from cache, retry if timeout occurs""" return self._cache.get(key=item, read=True, retry=True) def clear(self): self._cache.clear() def __len__(self): return len(list(self._cache.iterkeys())) def __contains__(self, item): value = self._cache.get(item, retry=True) # None, if expired if value: return True return False def __setitem__(self, key, value): if not self.use_listings_cache: return self._cache.set(key=key, value=value, expire=self.listings_expiry_time, retry=True) def __delitem__(self, key): del self._cache[key] def __iter__(self): return (k for k in self._cache.iterkeys() if k in self) def __reduce__(self): return ( FileDirCache, (self.use_listings_cache, self.listings_expiry_time, self.cache_location), )
class RouteHandler(object): def __init__(self, loop, messenger, database): self._loop = loop self._messenger = messenger self._database = database self.cache = Cache() async def create_election(self, request): private_key, public_key, user = await self._authorize(request) body = await decode_request(request) required_fields = [ 'name', 'description', 'start_timestamp', 'end_timestamp', 'results_permission', 'can_change_vote', 'can_show_realtime', 'voting_options', 'poll_book' ] validate_fields(required_fields, body) if user.get('type') != 'ADMIN' and user.get('type') != 'SUPERADMIN': raise ApiForbidden('Voter must be an admin or superadmin') election_id = uuid.uuid1().hex voting_options = body.get('voting_options') admin = await self._database.fetch_voter_resource(public_key=public_key ) for voting_option in voting_options: if voting_option.get('name').upper( ) == "NULL" or voting_option.get('name').upper() == "BLANK": raise ApiInternalError('NULL and BLANK are default options') voting_options.append({"name": "NULL", "description": "VOTE NULL"}) voting_options.append({"name": "BLANK", "description": "VOTE BLANK"}) await self._messenger.send_create_election_transaction( private_key=private_key, election_id=election_id, name=body.get('name'), description=body.get('description'), start_timestamp=body.get('start_timestamp'), end_timestamp=body.get('end_timestamp'), results_permission=body.get('results_permission'), can_change_vote=body.get('can_change_vote'), can_show_realtime=body.get('can_show_realtime'), admin_id=admin.get('voter_id'), status=1, timestamp=get_time()) for voting_option in voting_options: voting_option_id = uuid.uuid1().hex await self._messenger.send_create_voting_option_transaction( private_key=private_key, voting_option_id=voting_option_id, name=voting_option.get('name'), description=voting_option.get('description'), election_id=election_id, status=1, timestamp=get_time()) await self._database.insert_voting_option_num_vote_resource( voting_option_id=voting_option_id, name=voting_option.get('name'), election_id=election_id) for poll_book in body.get('poll_book'): await self._messenger.send_create_poll_registration_transaction( private_key=private_key, voter_id=poll_book.get('id'), name=poll_book.get('name'), election_id=election_id, status=1, timestamp=get_time()) return json_response({'data': 'Create election transaction submitted'}) async def create_voter(self, request): body = await decode_request(request) required_fields = ['voter_id', 'name', 'password'] validate_fields(required_fields, body) if await self._database.is_voter_created(body.get("voter_id") ) is not None: raise ApiConflict("Voter ID must be unique") public_key, private_key = self._messenger.get_new_key_pair() await self._messenger.send_create_voter_transaction( private_key=private_key, voter_id=body.get('voter_id'), public_key=public_key, name=body.get('name'), created_at=get_time(), type='VOTER') encrypted_private_key = encrypt_private_key(request.app['aes_key'], public_key, private_key) hashed_password = hash_password(body.get('password')) await self._database.create_auth_entry(public_key, encrypted_private_key, hashed_password) user = { 'name': body.get('name'), 'voter_id': body.get('voter_id'), 'type': 'VOTER' } token = self.generate_auth_token(request.app['secret_key'], public_key, user) return json_response({'accessToken': token, 'user': user}) async def update_voter_type(self, request): private_key, public_key, user = await self._authorize(request) body = await decode_request(request) required_fields = ['type'] validate_fields(required_fields, body) voter_id = request.match_info.get('voterId', '') if voter_id == '': raise ApiBadRequest( 'The voter ID is a required query string parameter') if user.get('type') != 'SUPERADMIN': raise ApiForbidden('Forbidden') voter = await self._database.fetch_voter_resource(voter_id=voter_id) if voter is None: raise ApiNotFound('No voter found') if body.get('type') == 'ADMIN' and (voter.get('type') == 'ADMIN' or voter.get('type') == 'SUPERADMIN'): raise ApiConflict( 'Voter {} is already an admin or superadmin'.format(voter_id)) elif body.get('type') == 'VOTER' and voter.get('type') == 'VOTER': raise ApiConflict('Voter {} is already a voter. '.format(voter_id)) auth_info = await self._database.fetch_auth_resource( public_key=voter.get('public_key')) voter_private_key = decrypt_private_key( request.app['aes_key'], voter.get('public_key'), auth_info.get('encrypted_private_key')) await self._messenger.send_update_voter_transaction( private_key=voter_private_key, voter_id=voter_id, public_key=voter.get('public_key'), name=voter.get('name'), created_at=get_time(), type=body.get('type')) return json_response({ 'voter': { 'voter_id': voter_id, 'name': voter.get('name'), 'type': 'ADMIN' } }) async def create_vote(self, request): body = await decode_request(request) required_fields = [] validate_fields(required_fields, body) private_key, public_key, user = await self._authorize(request) voting_option_id = request.match_info.get('votingOptionId', '') if voting_option_id == '': raise ApiBadRequest( 'The voting option ID is a required query string parameter') voter = await self._database.fetch_voter_resource(public_key=public_key ) if voter is None: raise ApiNotFound('Voter with the public_key ' '{} was not found'.format(public_key)) voting_option = await self._database.fetch_voting_option_resource( voting_option_id=voting_option_id) vo_count_vote = await self._database.fetch_voting_option_num_vote_resource( voting_option_id=voting_option_id) if voting_option is None: raise ApiNotFound('Voting Option with the voting option id ' '{} was not found'.format(voting_option_id)) election_id = voting_option.get('election_id') election = await self._database.fetch_election_resource( election_id=election_id) if election.get('status') == 0: raise ApiBadRequest('Election with the election id ' '{} is cancelled'.format(election_id)) current_time = get_time() if election.get('end_timestamp') < current_time or election.get( 'start_timestamp') > current_time: raise ApiBadRequest('Not in election time.'.format()) poll_registration = await self._database.fetch_poll_book_registration( voter_id=user.get('voter_id'), election_id=election_id) if poll_registration is None: raise ApiBadRequest( 'Voter is not registered in the poll book of the election with the id ' '{} .'.format(election_id)) num_votes_update = vo_count_vote.get('num_votes') + 1 await self._messenger.send_create_vote_transaction( private_key=private_key, vote_id=uuid.uuid1().hex, timestamp=get_time(), voter_id=voter.get('voter_id'), election_id=voting_option.get('election_id'), voting_option_id=voting_option_id) await self._database.update_voting_option_num_vote_resource( voting_option_id=voting_option_id, num_votes=num_votes_update) return json_response({'data': 'Create vote transaction submitted'}) async def update_vote(self, request): private_key, public_key, user = await self._authorize(request) body = await decode_request(request) required_fields = ['voting_option_id'] validate_fields(required_fields, body) vote_id = request.match_info.get('voteId', '') if vote_id == '': raise ApiBadRequest( 'The vote ID is a required query string parameter') vote = await self._database.fetch_vote_resource(vote_id=vote_id) election_id = vote.get('election_id') if vote is None: raise ApiNotFound('Vote with the vote id ' '{} was not found'.format(vote_id)) if vote.get('voting_option_id') == body.get('voting_option_id'): raise ApiBadRequest('Vote must be different.') election = await self._database.fetch_election_resource( election_id=election_id) if election is None: raise ApiNotFound('Election with the election id ' '{} was not found'.format(election_id)) if election.get('can_change_vote') == 0: raise ApiInternalError( 'Election with the election id ' '{} was not found don\'t permit to change vote'.format( election_id)) if election.get('can_change_vote') == 0: raise ApiInternalError( 'Election with the election id ' '{} was not found don\'t permit to change vote'.format( election_id)) current_time = get_time() if election.get('end_timestamp') < current_time or election.get( 'start_timestamp') > current_time: raise ApiBadRequest('Not in election time.'.format()) if election.get('admin_id') == user.get('voter_id'): raise ApiBadRequest( 'User is not the owner of the election with the id ' '{} .'.format(election_id)) new_voting_option_id = body.get('voting_option_id') old_voting_option_id = vote.get('voting_option_id') old_num_vote = await self._database.fetch_voting_option_num_vote_resource( voting_option_id=old_voting_option_id) new_num_vote = await self._database.fetch_voting_option_num_vote_resource( voting_option_id=new_voting_option_id) num_votes_remove = old_num_vote.get('num_votes') - 1 num_votes_update = new_num_vote.get('num_votes') + 1 await self._messenger.send_update_vote_transaction( private_key=private_key, vote_id=vote_id, timestamp=get_time(), voting_option_id=new_voting_option_id) # remove -1 to old voting option await self._database.update_voting_option_num_vote_resource( voting_option_id=old_voting_option_id, num_votes=num_votes_remove) # add +1 to new voting option await self._database.update_voting_option_num_vote_resource( voting_option_id=new_voting_option_id, num_votes=num_votes_update) return json_response({'data': 'Update Vote transaction submitted'}) async def update_election(self, request): private_key, public_key, user = await self._authorize(request) body = await decode_request(request) election_id = request.match_info.get('electionId', '') if election_id == '': raise ApiBadRequest( 'The election ID is a required query string parameter') election = await self._database.fetch_election_resource( election_id=election_id) if election is None: raise ApiNotFound('Election with the election id ' '{} was not found'.format(election_id)) current_time = get_time() if election.get('start_timestamp') < current_time: raise ApiBadRequest('Election with the election id ' '{} already start.'.format(election_id)) if election.get('admin_id') != user.get('voter_id'): raise ApiBadRequest( 'User is not the owner of the election with the id ' '{} .'.format(election_id)) await self._messenger.send_update_election_transaction( private_key=private_key, election_id=election_id, name=body.get('name') if body.get('name') is not None else election.get('name'), description=body.get('description') if body.get('description') is not None else election.get('description'), start_timestamp=body.get('start_timestamp') if body.get('start_timestamp') is not None else election.get('start_timestamp'), end_timestamp=body.get('end_timestamp') if body.get('end_timestamp') is not None else election.get('end_timestamp'), results_permission=body.get('results_permission') if body.get('results_permission') is not None else election.get('results_permission'), can_change_vote=body.get('can_change_vote') if body.get('can_change_vote') is not None else election.get('can_change_vote'), can_show_realtime=body.get('can_show_realtime') if body.get('can_show_realtime') is not None else election.get('can_show_realtime'), admin_id=user.get('voter_id'), status=body.get('status') if body.get('status') is not None else election.get('status'), timestamp=get_time()) if body.get('voting_options') is not None: for voting_option in body.get('voting_options'): voting_option_id = uuid.uuid1().hex await self._messenger.send_create_voting_option_transaction( private_key=private_key, voting_option_id=voting_option_id, name=voting_option.get('name'), description=voting_option.get('description'), election_id=election_id, status=1, timestamp=get_time()) await self._database.insert_voting_option_num_vote_resource( voting_option_id=voting_option_id, name=voting_option.get('name'), election_id=election_id) if body.get('poll_book') is not None: for poll_book in body.get('poll_book'): await self._messenger.send_create_poll_registration_transaction( private_key=private_key, voter_id=poll_book.get('id'), name=poll_book.get('name'), election_id=election_id, status=1, timestamp=get_time()) return json_response({'data': 'Update Election transaction submitted'}) async def get_election(self, request): private_key, public_key, user = await self._authorize(request) election_id = request.match_info.get('electionId', '') if election_id == '': raise ApiBadRequest( 'The election ID is a required query string parameter') if 'asAdmin' in request.rel_url.query: election = await self._database.fetch_election_with_can_vote_resource_admin( voter_id=user.get('voter_id'), election_id=election_id) else: election = await self._database.fetch_election_with_can_vote_resource( voter_id=user.get('voter_id'), election_id=election_id) if election is None: raise ApiNotFound('Election with the election id ' '{} was not found'.format(election_id)) if election.get('results_permission') != 'PUBLIC': if election.get('can_vote') is False and election.get( 'admin_id') != user.get('voter_id'): raise ApiForbidden( 'Voter is not registered in the poll book of the election with the id ' '{}.'.format(election_id)) return json_response(election) async def get_election_votes(self, request): private_key, public_key, user = await self._authorize(request) election_id = request.match_info.get('electionId', '') if election_id == '': raise ApiBadRequest( 'The election ID is a required query string parameter') number_of_votes = await self._database.fetch_number_of_votes( election_id=election_id) if number_of_votes is None: raise ApiNotFound('No voting options with the election id ' '{} was not found'.format(election_id)) election = await self._database.fetch_election_resource( election_id=election_id) if election is None: raise ApiNotFound('Election with the election id ' '{} was not found'.format(election_id)) if election.get('results_permission') != 'PUBLIC': poll_registration = await self._database.fetch_poll_book_registration( voter_id=user.get('voter_id'), election_id=election_id) if poll_registration is None and election.get( 'admin_id') != user.get('voter_id'): raise ApiBadRequest( 'Voter is not registered in the poll book of the election with the id ' '{} .'.format(election_id)) return json_response(number_of_votes) async def get_poll_registrations(self, request): private_key, public_key, user = await self._authorize(request) election_id = request.match_info.get('electionId', '') if election_id == '': raise ApiBadRequest( 'The election ID is a required query string parameter') poll_book = await self._database.fetch_poll_book( election_id=election_id) if poll_book is None: raise ApiNotFound('No voters with the election id ' '{} was not found'.format(election_id)) election = await self._database.fetch_election_resource( election_id=election_id) if election is None: raise ApiNotFound('Election with the election id ' '{} was not found'.format(election_id)) if election.get('admin_id') != user.get('voter_id'): raise ApiBadRequest( 'User is not the owner of the election with the id ' '{} .'.format(election_id)) return json_response(poll_book) async def count_poll_registrations(self, request): private_key, public_key, user = await self._authorize(request) election_id = request.match_info.get('electionId', '') if election_id == '': raise ApiBadRequest( 'The election ID is a required query string parameter') count_poll_book = await self._database.count_poll_book( election_id=election_id) election = await self._database.fetch_election_resource( election_id=election_id) if election is None: raise ApiNotFound('Election with the election id ' '{} was not found'.format(election_id)) if election.get('results_permission') != 'PUBLIC': poll_registration = await self._database.fetch_poll_book_registration( voter_id=user.get('voter_id'), election_id=election_id) if poll_registration is None and election.get( 'admin_id') != user.get('voter_id'): raise ApiBadRequest( 'Voter is not registered in the poll book of the election with the id ' '{} .'.format(election_id)) return json_response(count_poll_book) async def list_voting_options_election(self, request): private_key, public_key, user = await self._authorize(request) election_id = request.match_info.get('electionId', '') if election_id == '': raise ApiBadRequest( 'The election ID is a required query string parameter') voting_options = await self._database.fetch_election_voting_options_resource( election_id=election_id) if voting_options is None: raise ApiNotFound('Voting Options in the election id ' '{} was not found'.format(election_id)) election = await self._database.fetch_election_resource( election_id=election_id) if election is None: raise ApiNotFound('Election with the election id ' '{} was not found'.format(election_id)) if election.get('results_permission') != 'PUBLIC': poll_registration = await self._database.fetch_poll_book_registration( voter_id=user.get('voter_id'), election_id=election_id) if poll_registration is None and election.get( 'admin_id') != user.get('voter_id'): raise ApiForbidden( 'Voter is not registered in the poll book of the election with the id ' '{} .'.format(election_id)) return json_response(voting_options) async def get_voting_option(self, request): private_key, public_key, user = await self._authorize(request) voting_option_id = request.match_info.get('votingOptionId', '') if voting_option_id == '': raise ApiBadRequest( 'The voting option ID is a required query string parameter') voting_option = await self._database.fetch_voting_option_resource( voting_option_id=voting_option_id) if voting_option is None: raise ApiNotFound('No voting options with the id ' '{} was not found'.format(voting_option_id)) return json_response(voting_option) async def update_voting_option_status(self, request): private_key, public_key, user = await self._authorize(request) voting_option_id = request.match_info.get('votingOptionId', '') if voting_option_id == '': raise ApiBadRequest( 'The voting option ID is a required query string parameter') voting_option = await self._database.fetch_voting_option_resource( voting_option_id=voting_option_id) if voting_option is None: raise ApiNotFound('Voting Option with the voting option id ' '{} was not found'.format(voting_option_id)) election_id = voting_option.get('election_id') election = await self._database.fetch_election_resource( election_id=election_id) if election is None: raise ApiNotFound('Election with the election id ' '{} was not found'.format(election_id)) current_time = get_time() if election.get('start_timestamp') < current_time: raise ApiBadRequest('Election with the election id ' '{} already start.'.format(election_id)) if election.get('admin_id') != user.get('voter_id'): raise ApiBadRequest( 'User is not the owner of the election with the id ' '{} .'.format(election_id)) if voting_option.get('status') is True: status = 0 else: status = 1 await self._messenger.send_update_voting_option_status_transaction( private_key=private_key, voting_option_id=voting_option_id, name=voting_option.get('name'), description=voting_option.get('description'), election_id=voting_option.get('election_id'), status=status, timestamp=get_time()) return json_response( {'data': 'Update Voting Option Status transaction submitted'}) async def update_poll_book_status(self, request): private_key, public_key, user = await self._authorize(request) voter_id = request.match_info.get('voterId', '') if voter_id == '': raise ApiBadRequest( 'The voter ID is a required query string parameter') election_id = request.match_info.get('electionId', '') if election_id == '': raise ApiBadRequest( 'The election ID is a required query string parameter') voter_poll_book = await self._database.fetch_poll_book_registration( election_id=election_id, voter_id=voter_id) if voter_poll_book is None: raise ApiNotFound('Voter with the voter id ' '{} was not found'.format(voter_id)) election = await self._database.fetch_election_resource( election_id=election_id) if election is None: raise ApiNotFound('Election with the election id ' '{} was not found'.format(election_id)) current_time = get_time() if election.get('start_timestamp') < current_time: raise ApiBadRequest('Election with the election id ' '{} already start.'.format(election_id)) if election.get('admin_id') != user.get('voter_id'): raise ApiBadRequest( 'User is not the owner of the election with the id ' '{} .'.format(election_id)) if voter_poll_book.get('status') is True: status = 0 else: status = 1 await self._messenger.send_update_voter_poll_book_status_transaction( private_key=private_key, voter_id=voter_id, name=voter_poll_book.get('name'), election_id=election_id, status=status, timestamp=get_time()) return json_response( {'data': 'Update Poll Registration Status transaction submitted'}) async def list_elections_current(self, request): private_key, public_key, user = await self._authorize(request) voterId = request.match_info.get('voterId', '') if voterId == '': raise ApiBadRequest( 'The voter ID is a required query string parameter') if user.get('voter_id') != voterId: raise ApiForbidden('Admin must be the authenticated one') current_elections_list = await self._database.fetch_current_elections_resources( voterId, get_time()) return json_response(current_elections_list) async def list_elections_past(self, request): private_key, public_key, user = await self._authorize(request) voterId = request.match_info.get('voterId', '') if voterId == '': raise ApiBadRequest( 'The voter ID is a required query string parameter') if user.get('voter_id') != voterId: raise ApiForbidden('Admin must be the authenticated one') past_elections_list = await self._database.fetch_past_elections_resources( voterId, get_time()) return json_response(past_elections_list) async def list_admin_elections(self, request): private_key, public_key, user = await self._authorize(request) voter_id = request.match_info.get('voterId', '') if voter_id == '': raise ApiBadRequest( 'The voter ID is a required query string parameter') if user.get('voter_id') != voter_id: raise ApiForbidden('Admin must be the authenticated one') if user.get('type') != 'ADMIN' and user.get('type') != 'SUPERADMIN': raise ApiForbidden('Voter must be an admin or superadmin') admin_elections_list = await self._database.fetch_admin_elections_resources( user.get('voter_id')) return json_response(admin_elections_list) async def list_public_elections(self, request): private_key, public_key, user = await self._authorize(request) public_elections_list = await self._database.fetch_public_elections_resources( get_time()) return json_response(public_elections_list) async def list_public_past_elections(self, request): private_key, public_key, user = await self._authorize(request) past_elections_list = await self._database.fetch_public_past_elections_resources( user.get('voter_id'), get_time()) return json_response(past_elections_list) async def list_admins(self, request): private_key, public_key, user = await self._authorize(request) if user.get('type') != 'SUPERADMIN': raise ApiForbidden('Forbidden') admin_list = await self._database.fetch_admins_resources() return json_response(admin_list) async def get_voters(self, request): private_key, public_key, user = await self._authorize(request) if user.get('type') != 'SUPERADMIN': raise ApiForbidden('Forbidden') voter_id = request.match_info.get('voterId', '') voters_list = await self._database.fetch_voters_resources( voter_id=voter_id) return json_response(voters_list) async def get_vote(self, request): private_key, public_key, user = await self._authorize(request) vote_id = request.match_info.get('voteId', '') if vote_id == '': raise ApiBadRequest( 'The vote ID is a required query string parameter') vote = await self._database.fetch_vote_resource(vote_id=vote_id) if vote is None: raise ApiNotFound('Vote with the vote id ' '{} was not found'.format(vote_id)) election_id = vote.get('election_id') election = await self._database.fetch_election_resource( election_id=election_id) if election is None: raise ApiNotFound('Election with the election id ' '{} was not found'.format(election_id)) return json_response(vote) async def get_vote_election(self, request): private_key, public_key, user = await self._authorize(request) voter_id = request.match_info.get('voterId', '') if voter_id == '': raise ApiBadRequest( 'The voter ID is a required query string parameter') election_id = request.match_info.get('electionId', '') if election_id == '': raise ApiBadRequest( 'The election ID is a required query string parameter') if user.get('voter_id') != voter_id: raise ApiForbidden('Admin must be the authenticated one') vote = await self._database.fetch_my_vote__election_resource( voter_id=voter_id, election_id=election_id) return json_response(vote) async def authenticate(self, request): body = await decode_request(request) required_fields = ['voter_id', 'password'] validate_fields(required_fields, body) password = bytes(body.get('password'), 'utf-8') voter = await self._database.fetch_voter_resource( voter_id=body.get('voter_id')) if voter is None: raise ApiUnauthorized('Incorrect voter_id or password') auth_info = await self._database.fetch_auth_resource( public_key=voter.get('public_key')) if auth_info is None: raise ApiUnauthorized('No voter with that public key exists') hashed_password = auth_info.get('hashed_password') if not bcrypt.checkpw(password, bytes.fromhex(hashed_password)): raise ApiUnauthorized('Incorrect public key or password') user = { 'name': voter.get('name'), 'voter_id': body.get('voter_id'), 'type': voter.get('type') } token = self.generate_auth_token(request.app['secret_key'], voter.get('public_key'), user) return json_response({'accessToken': token, 'user': user}) async def _authorize(self, request): token = request.headers.get('AUTHORIZATION') if token is None: raise ApiUnauthorized('No auth token provided') token_prefixes = ('Bearer', 'Token') for prefix in token_prefixes: if prefix in token: token = token.partition(prefix)[2].strip() try: token_dict = self.deserialize_auth_token(request.app['secret_key'], token) except BadSignature: raise ApiUnauthorized('Invalid auth token') public_key = token_dict.get('public_key') auth_resource = await self._database.fetch_auth_resource( public_key=public_key) if auth_resource is None: raise ApiUnauthorized('Token is not associated with an agent') user = self.cache.get(token) return decrypt_private_key( request.app['aes_key'], public_key, auth_resource['encrypted_private_key']), public_key, user async def logout(self, request): await self._authorize(request) token = request.headers.get('AUTHORIZATION') token_prefixes = ('Bearer', 'Token') for prefix in token_prefixes: if prefix in token: token = token.partition(prefix)[2].strip() self.cache.delete(token) return json_response("Successful logout") def generate_auth_token(self, secret_key, public_key, user): serializer = Serializer(secret_key, expires_in=3600) token = serializer.dumps({'public_key': public_key}) decoded_token = token.decode('ascii') self.cache.set(decoded_token, user, expire=3600) self.cache.close() return decoded_token def deserialize_auth_token(self, secret_key, token): token_status = self.cache.get(token) if token_status is None: raise BadSignature("") serializer = Serializer(secret_key) return serializer.loads(token)
def search( self, name=None, ip=None, hexadecimal=None, type="ANY", bailiwick=None, wildcard_left=None, wildcard_right=None, inverse=False, sort=True, return_limit=10000, remote_limit=50000, epoch=False, time_first_before=None, time_first_after=None, time_last_before=None, time_last_after=None, ): """ A method of the DNSDB Class to search the DNSDB API. :param name: string (required) fully qualified domain name :param ip: string IPv4 or IPv6 address, CIDR notation is valid :param hexadecimal: string hexadecimal digits specifying a raw octet string :param type: string (optional: default="ANY") dns resource record types (ANY, A, MX, SIG, etc) :param bailiwick: string (optional: default=None) a label in a fqdn, not valid for inverse queries :param wildcard_left: Boolean (optional: default=None) wildcard search to the left of a dot in a domain name :param wildcard_right: Boolean (optional: default=None) wildcard search to the right of a dot in a domain name :param inverse: boolean (optional: default=False) search for names resolving to names (e.g. MX, NS, CNAME, etc) only valid when used with name :param sort: boolean (optional: default=True) :param return_limit: integer (optional: default=10000) :param remote_limit: integer (optional: default=50000) :param epoch: boolean (optional: default=False) :param time_first_before: :param time_first_after: :param time_last_before: :param time_last_after: :return: Object """ options = dict() options["name"] = name options["ip"] = ip options["hex"] = hexadecimal options["type"] = type options["bailiwick"] = bailiwick options["wildcard_left"] = wildcard_left options["wildcard_right"] = wildcard_right options["inverse"] = inverse options["sort"] = sort options["return_limit"] = return_limit options["remote_limit"] = remote_limit options["epoch"] = epoch options["time_first_before"] = time_first_before options["time_first_after"] = time_first_after options["time_last_before"] = time_last_before options["time_last_after"] = time_last_after options["api_key"] = self.api_key options["server"] = self.server options["cache"] = self.cache options["cache_location"] = self.cache_location options["cache_timeout"] = self.cache_timeout options = utils.pre_process(options) uri = utils.build_uri(options) if options["cache"] is True: cache = Cache(options["cache_location"]) cached_result = cache.get(uri) if cached_result: data = json.loads( gzip.decompress(cached_result).decode("utf-8")) results = Result( records=data["records"], status_code=data["status_code"], error=data["error"], quota=data["quota"], cached=True, ) else: results = _query(options, uri) if results.status_code == 200 or results.status_code == 404: compressed = Result.to_compressed(results) cache.set(uri, compressed, expire=options["cache_timeout"]) else: results = _query(options, uri) if results.status_code == 200: results = utils.post_process(options, results) return results return results
class CloudFS(Operations): '''Baidu netdisk filesystem''' def __init__(self, mainArgs, *args, **kw): logger.info(colored("- fuse 4 cloud driver -", 'red')) self.buffer = Cache('./cache/buffer-batchmeta') self.dir_buffer = Cache('./cache/dir_buffer-buffer-batchmeta') self.attr_requesting = Cache('./cache/attr-requesting') self.mainArgs = mainArgs self.traversed_folder = Cache('./cache/traversed-folder') self.disk = PCS(self.mainArgs) self.createLock = Lock() self.attrLock = Lock() self.writing_files = {} self.downloading_files = {} logger.info(f'mainArgs:{mainArgs}') q = json.loads(self.disk.quota()) # only request once try: self.total_size = q['quota'] self.used = q['used'] except Exception as e: self.total_size = 100000000000 self.used = 0 logger.exception(e) logger.debug(f'con`t load quota api, fall back to default') self.avail = self.total_size - self.used if mainArgs.debug: logger.setLevel(logging.DEBUG) logger.debug(colored("- debug mode -", 'red')) logger.debug( colored("- cach would not be the same after restart -", 'red')) self.buffer = Cache('./cache/buffer-batchmeta' + str(time.time())) self.dir_buffer = Cache('./cache/dir_buffer-buffer-batchmeta' + str(time.time())) self.traversed_folder = Cache('./cache/traversed-folder' + str(time.time())) self._readDirAsync("/", 2, dirReaderDaemon) else: logger.setLevel(logging.INFO) # update all folder in other thread self._readDirAsync("/", mainArgs.preload_level, dirReaderDaemon) @staticmethod def add_write_permission(st, permission='u'): """Add `w` (`write`) permission to specified targets.""" mode_map = { 'u': stat.S_IWUSR, 'g': stat.S_IWGRP, 'o': stat.S_IWOTH, 'a': stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH, } logger.info( f'-------------------{type(stat.S_IWUSR)} ,{type(st["st_mode"])}') for t in permission: st['st_mode'] |= mode_map[t] return st def _baidu_file_attr_convert(self, path, file_info): foo = fileAttr.copy() try: foo['st_ctime'] = file_info[ 'local_ctime'] if 'local_ctime' in file_info else file_info[ 'ctime'] foo['st_mtime'] = file_info[ 'local_mtime'] if 'local_mtime' in file_info else file_info[ 'mtime'] foo['st_mode'] = 16877 if file_info['isdir'] else 36279 foo['st_nlink'] = 2 if file_info['isdir'] else 1 foo['st_size'] = int( file_info['size']) if 'size' in file_info else 0 self.buffer[path] = foo except Exception as e: logger.debug(f'======================') logger.debug(f'add buffer error {e},{path}:{file_info}') def _del_file_from_buffer(self, path): self.buffer.pop(path) def _getRootAttr(self): path = "/" if path in self.buffer: return self.buffer[path] logger.debug(f'net root: {path}') jdata = json.loads(self.disk.meta([path])) f = fileAttr.copy() f["st_mode"] = 16877 f["st_nlink"] = 2 if 'error_code' in jdata and jdata["error_code"] != 0: logger.debug(f"error_code:{jdata}") # logger.info(f'{error_map[str(jdata["error_code"])]} args: {path}') self.buffer.set(path, f, expire=60) return f if "list" not in jdata or len(jdata["list"]) == 0: logger.debug(f"{path} not list :{jdata}") self.buffer.set(path, f, expire=60) return f file_info = jdata["list"][0] self._baidu_file_attr_convert(path, file_info) return file_info @funcLog def getattr(self, path, fh=None): ''' Returns a dictionary with keys identical to the stat C structure of stat(2). st_atime, st_mtime and st_ctime should be floats. NOTE: There is an incompatibility between Linux and Mac OS X concerning st_nlink of directories. Mac OS X counts all files inside the directory, while Linux counts only the subdirectories. ''' if path in self.writing_files: return self.writing_files[path] if path.split("/")[-1].startswith("."): raise FuseOSError(errno.ENOENT) # special handle root Attr if path == "/": return self._getRootAttr() parentDir = os.path.dirname(path) if parentDir not in self.dir_buffer: self._readDir(parentDir, 1) if path in self.buffer: return self.buffer[path] raise FuseOSError(errno.ENOENT) @funcLog def truncate(self, path, length, fh=None): self.unlink(path) self.create(path, None) self.writing_files[path]["uploading_tmp"].truncate(length) def _readDirAsync(self, path, depth, p): p.submit(self._readDir, path, depth, p) def _readDir(self, path, depth=2, threadPool=pool): if path not in self.traversed_folder: self.traversed_folder.set(path, b'1', expire=self.mainArgs.cache_timeout) logger.debug(f'net dir {depth} - {path} ') try: foo = json.loads(self.disk.list_files(path)) files = ['.', '..'] if 'error_code' in foo and foo["error_code"] != 0: logger.info( f'{error_map[str(foo["error_code"])]} args: {path}') if "list" not in foo: return depth -= 1 for file in foo['list']: if file['server_filename'].startswith("."): continue files.append(file['server_filename']) # logger.debug(f'{file}') self._baidu_file_attr_convert(file['path'], file) if depth > 0: if file['isdir']: self._readDirAsync(file['path'], depth, threadPool) # self._readDir(file['path'],depth,threadPool) self.dir_buffer[path] = files except Exception as s: logger.exception(s) @funcLog def readdir(self, path, offset): self._readDirAsync(path, 1, pool) if path in self.dir_buffer: for r in self.dir_buffer[path]: yield r else: files = ['.', '..'] for r in files: yield r # @funcLog def open(self, path, flags): if path in self.writing_files: return 0 # method does not have thread race problem, open by one thread only try: if path not in self.downloading_files: url = self.disk.getRestUrl(path) x = Task(url, mainArgs, path, self.disk) x.start() self.downloading_files[path] = x except Baidu8Secs as e: logger.exception(e) except Exception as e: logger.exception(e) return 0 def read(self, path, size, offset, fh): x = self.downloading_files[path] if x: data = x.get_cache(offset, size) filename = path[path.rfind("/") + 1:] if filename.startswith("enc."): if offset == 0: if data and len(data) > encrpted_length: data = bytes( cipher(data, 0, encrpted_length, self.mainArgs.key)) else: print("decrpt failed!") return data raise FuseOSError(errno.EIO) def updateCache(self, path, newValue): ''' add updateCache(path,value) delete updateCache(path,None) udpate updateCache(path,value) ''' pass def updateCacheKeyOnly(self, old, new): ''' delete updateCacheKeyOnly(old,None) add/update updateCacheKeyOnly(old,new) ''' try: old_parent_dir = os.path.dirname(old) old_name = os.path.basename(old) if not new: oldCache = self.dir_buffer.get(old_parent_dir) # remove if oldCache: if old_name in oldCache: oldCache.remove(old_name) self.dir_buffer[old_parent_dir] = oldCache if old in self.buffer: self.buffer.pop(old) else: pass else: print("updateCache", old, new) oldCache = self.dir_buffer[old_parent_dir] new_parent_dir = os.path.dirname(new) if old_name in oldCache: # dir old remove oldCache.remove(old_name) self.dir_buffer[old_parent_dir] = oldCache # dir new add newfilename = new[new.rfind("/") + 1:] newCache = self.dir_buffer.get(new_parent_dir, []) newCache.append(newfilename) self.dir_buffer[new_parent_dir] = newCache if old in self.buffer: old_info = self.buffer.pop(old) self.buffer[new] = old_info except Exception as e: logger.info(e) def updateDir(self, old, new): pass def unlink(self, path): ''' will only delete file ''' print("unlink .....................") self.disk.delete([path]) self.updateCacheKeyOnly(path, None) def rmdir(self, path): ''' will only delete directory ''' self.disk.delete([path]) self.updateCacheKeyOnly(path, None) def access(self, path, amode): return 0 def rename(self, old, new): ''' will effect dir and file ''' logger.info(f'rename {old}, {new}') self.disk.rename(old, new) self.updateCacheKeyOnly(old, new) @funcLog def mkdir(self, path, mode): logger.info(f'making dir {path}') r = json.loads(self.disk.mkdir(path)) if 'error_code' in r: logger.info(f'{r}') logger.info( f'{error_map[str(r["error_code"])]} args: {path}, response:{r}' ) return directory = path[:path.rfind("/")] filename = path[path.rfind("/") + 1:] cache = None if directory in self.dir_buffer: cache = self.dir_buffer[directory] cache.append(filename) self.dir_buffer[directory] = cache self._baidu_file_attr_convert(path, r) @funcLog def create(self, path, mode, fh=None): logger.debug(f'create {path}') with self.createLock: if path not in self.writing_files: attr = fileAttr.copy() t = time.time() attr['uploading_tmp'] = tempfile.NamedTemporaryFile('wb') attr['st_mode'] = attr[ 'st_mode'] | stat.S_IFREG | stat.S_ISUID | stat.S_ISGID self.writing_files[path] = attr else: logger.debug(f'{path} is writing on, wait another turn..') return 0 def flush(self, path, fh): with self.createLock: if path in self.writing_files: self.writing_files[path]["uploading_tmp"].flush() return 0 def release(self, path, fh): with self.createLock: if path in self.writing_files: uploading_tmp = self.writing_files[path]['uploading_tmp'] r = json.loads(self.disk.upload(uploading_tmp.name, path)) logger.info(f'================================={r}') self.writing_files[path]['uploading_tmp'].close() # if path in self.buffer: # del self.buffer[path] if path in self.writing_files: del self.writing_files[path] # why ? prevent accidently read file when uploading still in progress if path in self.downloading_files: del self.downloading_files[path] # update file self._baidu_file_attr_convert(path, r) # update parent dir parentDir = os.path.dirname(path) filename = path[path.rfind("/") + 1:] if parentDir in self.dir_buffer: parentDirCache = self.dir_buffer[parentDir] parentDirCache.append(filename) self.dir_buffer[parentDir] = parentDirCache logger.info(f'{self.dir_buffer[parentDir]}') print("released", path) return # method does not have thread race problem, release by one thread only if path in self.downloading_files: # self.downloading_files[path].terminate() # del self.downloading_files[path] # uploading_tmp = "./uploading_tmp"+path # logger.info("delete uploading_tmp:", uploading_tmp) # os.remove(uploading_tmp) pass def write(self, path, data, offset, fp): filename = path[path.rfind("/") + 1:] if filename.startswith("enc."): if offset == 0 and data and len(data) > encrpted_length: data = bytes( cipher(data, 0, encrpted_length, self.mainArgs.key)) length = len(data) self.writing_files[path]["st_size"] += length self.writing_files[path]["uploading_tmp"].write(data) return length def chmod(self, path, mode): pass def statfs(self, path): # TODO read from cloud disk return { 'f_bavail': int((self.avail) / 4096), 'f_bfree': int((self.avail) / 4096), # 相同的值 block 'f_favail': 4290675908, 'f_ffree': 4290675908, # 相同的值 node 'f_bsize': 104857, # perferd value 'f_blocks': int(self.total_size / 8), 'f_files': 4294967279, 'f_flag': 0, 'f_frsize': 4096, 'f_namemax': 255 }
class ReadCacheDataBackend(DataBackend): def __init__(self, config): read_cache_directory = config.get('dataBackend.readCache.directory', None, types=str) read_cache_maximum_size = config.get( 'dataBackend.readCache.maximumSize', None, types=int) if read_cache_directory and not read_cache_maximum_size or not read_cache_directory and read_cache_maximum_size: raise ConfigurationError( 'Both dataBackend.readCache.directory and dataBackend.readCache.maximumSize need to be set ' + 'to enable disk based caching.') if read_cache_directory and read_cache_maximum_size: os.makedirs(read_cache_directory, exist_ok=True) try: self._read_cache = Cache( read_cache_directory, size_limit=read_cache_maximum_size, eviction_policy='least-frequently-used', statistics=1, ) except Exception: logger.warning( 'Unable to enable disk based read caching. Continuing without it.' ) self._read_cache = None else: logger.debug( 'Disk based read caching instantiated (cache size {}).'. format(read_cache_maximum_size)) else: self._read_cache = None self._use_read_cache = True # Start reader and write threads after the disk cached is created, so that they see it. super().__init__(config) def _read(self, block, metadata_only): key = self._block_uid_to_key(block.uid) metadata_key = key + self._META_SUFFIX if self._read_cache is not None and self._use_read_cache: metadata = self._read_cache.get(metadata_key) if metadata and metadata_only: return block, None, metadata elif metadata: data = self._read_cache.get(key) if data: return block, data, metadata block, data, metadata = super()._read(block, metadata_only) # We always put blocks into the cache even when self._use_read_cache is False if self._read_cache is not None: self._read_cache.set(metadata_key, metadata) if not metadata_only: self._read_cache.set(key, data) return block, data, metadata def use_read_cache(self, enable): old_value = self._use_read_cache self._use_read_cache = enable return old_value def close(self): super().close() if self._read_cache is not None: (cache_hits, cache_misses) = self._read_cache.stats() logger.debug( 'Disk based cache statistics (since cache creation): {} hits, {} misses.' .format(cache_hits, cache_misses)) self._read_cache.close()
#!/usr/bin/env python # coding: utf8 from diskcache import Cache cache = Cache('/tmp/fzquant') cache.set('abc','def')
class TorProjectClient: """Simple client to query torproject.org for exit nodes. The client will download https://check.torproject.org/exit-addresses and check if a specified IP address is present in it. If that IP address is found it will check for its last update time and return a description of the node if its last update time is less than `ttl` seconds ago. :param ttl: Tor node will be kept only if its last update was less than `ttl` seconds ago. Ignored if `ttl` is 0 :param cache_duration: Duration before refreshing the cache (in seconds). Ignored if `cache_duration` is 0. :param cache_root: Path where to store the cached file downloaded from torproject.org :type ttl: int :type cache_duration: int :type cache_root: str """ def __init__(self, ttl=86400, cache_duration=3600, cache_root='/tmp/cortex/tor_project'): self.session = requests.Session() self.delta = None self.cache = None if ttl > 0: self.delta = timedelta(seconds=ttl) if cache_duration > 0: self.cache = Cache(cache_root) self.cache_duration = cache_duration self.url = 'https://check.torproject.org/exit-addresses' __cache_key = __name__ + ':raw_data' def _get_raw_data(self): try: return self.cache['raw_data'] except(AttributeError, TypeError): return self.session.get(self.url).text except KeyError: self.cache.set( 'raw_data', self.session.get(self.url).text, expire=self.cache_duration) return self.cache['raw_data'] def search_tor_node(self, ip): """Lookup an IP address to check if it is a known tor exit node. :param ip: The IP address to lookup :type ip: str :return: Data relative to the tor node. If `ip`is a tor exit node it will contain a `node` key with the hash of the node and a `last_status` key with the last update time of the node. If `ip` is not a tor exit node, the function will return an empty dictionary. :rtype: dict """ data = {} tmp = {} present = datetime.utcnow().replace(tzinfo=pytz.utc) for line in self._get_raw_data().splitlines(): params = line.split(' ') if params[0] == 'ExitNode': tmp['node'] = params[1] elif params[0] == 'ExitAddress': tmp['last_status'] = params[2] + 'T' + params[3] + '+0000' last_status = parse(tmp['last_status']) if (self.delta is None or (present - last_status) < self.delta): data[params[1]] = tmp tmp = {} else: pass return data.get(ip, {})
class TorBlutmagieClient: """Simple client to query torstatus.blutmagie.de for exit nodes. The client will download http://torstatus.blutmagie.de/query_export.php/Tor_query_EXPORT.csv and check if a specified IP address, FQDN or domain is present in it. It will cache the response for `cache_duration` seconds to avoid too much latency. :param cache_duration: Duration before refreshing the cache (in seconds). Ignored if `cache_duration` is 0. :param cache_root: Path where to store the cached file downloaded from torstatus.blutmagie.de :type cache_duration: int :type cache_root: str """ def __init__(self, cache_duration=3600, cache_root='/tmp/cortex/tor_project'): self.session = requests.Session() self.cache_duration = cache_duration if self.cache_duration > 0: self.cache = Cache(cache_root) self.url = 'http://torstatus.blutmagie.de/query_export.php/Tor_query_EXPORT.csv' __cache_key = __name__ + ':raw_data' def _get_raw_data(self): try: return self.cache[self.__cache_key] except (AttributeError, TypeError): return self.session.get(self.url).text.encode('utf-8') except KeyError: self.cache.set( self.__cache_key, self.session.get(self.url).text.encode('utf-8'), expire=self.cache_duration, read=True) return self.cache[self.__cache_key] def _get_data(self): return csv.DictReader( self._get_raw_data().decode('utf-8').splitlines(), delimiter=',') def _extract_fields(self, line): return { 'hostname': line['Hostname'], 'name': line['Router Name'], 'country_code': line['Country Code'], 'ip': line['IP Address'], 'as_name': line['ASName'], 'as_number': line['ASNumber'] } def _get_node_from_domain(self, domain): results = [] for line in self._get_data(): if domain.lower() in line['Hostname'].lower(): results.append(self._extract_fields(line)) return results def _get_node_from_fqdn(self, fqdn): results = [] for line in self._get_data(): if fqdn.lower() == line['Hostname'].lower(): results.append(self._extract_fields(line)) break return results def _get_node_from_ip(self, ip): results = [] for line in self._get_data(): if ip == line['IP Address']: results.append(self._extract_fields(line)) break return results def search_tor_node(self, data_type, data): """Lookup an artifact to check if it is a known tor exit node. :param data_type: The artifact type. Must be one of 'ip', 'fqdn' or 'domain' :param data: The artifact to lookup :type data_type: str :type data: str :return: Data relative to the tor node. If the looked-up artifact is related to a tor exit node it will contain a `nodes` array. That array will contains a list of nodes containing the following keys: - name: name given to the router - ip: their IP address - hostname: Hostname of the router - country_code: ISO2 code of the country hosting the router - as_name: ASName registering the router - as_number: ASNumber registering the router Otherwise, `nodes` will be empty. :rtype: list """ results = [] if data_type == 'ip': results = self._get_node_from_ip(data) elif data_type == 'fqdn': results = self._get_node_from_fqdn(data) elif data_type == 'domain': results = self._get_node_from_domain(data) else: pass return {"nodes": results}
class AzureBlobTickstore(Tickstore): """ Tickstore meant to run against Microsoft's Azure Blob Storage backend, e.g. for archiving purposes. Note this is not suitable for concurrent access to the blob because the index is loaded into memory on the local node and only written back to the blob on close. We may want to implement blob locking to at least prevent accidents. """ logger = logging.getLogger(__name__) def __init__(self, connect_str: str, db_name: str, cache_dir: Path = Path('/var/tmp/abs_lru_cache'), timestamp_column: str = 'date'): self.storage = BlobServiceClient.from_connection_string(connect_str) self.container_name = db_name.replace('_', '-').lower() self.container_client = self.storage.get_container_client( self.container_name) self.cache_dir = cache_dir self.cache = Cache(str(self.cache_dir)) self.index_path = self.cache_dir.joinpath(Path('index.h5')) self.timestamp_column = timestamp_column self.closed = False # try and create the container try: self.storage.create_container(self.container_name) except ResourceExistsError: self.logger.info(f'container {self.container_name} already exists') try: # fetch index from Azure blob_client = self.storage.get_blob_client( container=self.container_name, blob='index') index_blob = blob_client.download_blob() # write index into local cache dir with open(self.index_path, 'wb') as index_file: index_file.write(index_blob.readall()) self.index = DataFrameIndex(Path(f'azure:{db_name}/'), Path(self.index_path), db_name) except ResourceNotFoundError: # clear out any previously-downloaded index first if self.index_path.exists(): self.index_path.unlink() # create new, empty index on local disk (we'll upload this to Azure when we flush()) self.index = DataFrameIndex(Path(f'azure:{db_name}/'), Path(self.index_path), db_name) # noinspection DuplicatedCode def select( self, symbol: str, start: datetime.datetime, end: datetime.datetime, as_of_time: datetime.datetime = BiTimestamp.latest_as_of ) -> pd.DataFrame: self._check_closed('select') # pass 1: grab the list of splays matching the start / end range that are valid for as_of_time selected = self.index.select(symbol, start.date(), end.date(), as_of_time) if selected.empty: return selected # load all ticks in range into memory loaded_dfs = [] for index, row in selected.iterrows(): logical_path = row['path'] loaded_dfs.append(self.read(logical_path)) # pass 2: select ticks matching the exact start/end timestamps # noinspection PyTypeChecker all_ticks = pd.concat(loaded_dfs) time_mask = (all_ticks.index.get_level_values(self.timestamp_column) >= start) \ & (all_ticks.index.get_level_values(self.timestamp_column) <= end) # sort the ticks -- probably need to optimize this to sort on paths and sort ticks on ingest selected_ticks = all_ticks.loc[time_mask] selected_ticks.sort_index(inplace=True) return selected_ticks # noinspection PyTypeChecker def read(self, logical_path: str) -> pd.DataFrame: logical_prefix = f'azure:{self.container_name}' if logical_path.startswith(logical_prefix): blob_path = logical_path[len(logical_prefix) + 1:] else: # correction for bad data in index history blob_path = logical_path logical_path = f'{logical_prefix}/{logical_path}' cached_data = self.cache.get(logical_path) if cached_data is not None: self.logger.info(f'reading ticks from cache: {logical_path}') return pd.read_parquet(io.BytesIO(cached_data)) else: blob_client = self.storage.get_blob_client( container=self.container_name, blob=blob_path) tick_blob = blob_client.download_blob() tick_data = tick_blob.readall() self.cache.set(logical_path, tick_data) return pd.read_parquet(io.BytesIO(tick_data)) # noinspection PyArgumentList def insert(self, symbol: str, ts: BiTimestamp, ticks: pd.DataFrame): self._check_closed('insert') # encode DataFrame as parquet byte stream table = pyarrow.Table.from_pandas(df=ticks) buf = pyarrow.BufferOutputStream() pyarrow.parquet.write_table(table, buf) # determine latest version number # compose a splay path based on YYYY/MM/DD, symbol and version and pass in as a functor # so it can be populated with the bitemporal version as_at_date = ts.as_at() logical_prefix = f'azure:{self.container_name}' def create_write_path(version: int): path = f'{as_at_date.year}/{as_at_date.month:02d}/{as_at_date.day:02d}/{symbol}_{version:04d}.h5' return f'{logical_prefix}/{path}' # insert into local copy of index data_path = self.index.insert(symbol, as_at_date, create_write_path) base_path = data_path[len(logical_prefix) + 1:] # update cache data_bytes = bytes(buf.getvalue()) self.cache.set(data_path, data_bytes) # upload to Azure self.logger.info(f'uploading to Azure: {base_path}') blob_client = self.storage.get_blob_client( container=self.container_name, blob=str(base_path)) try: blob_client.upload_blob(io.BytesIO(data_bytes), overwrite=False) except ResourceExistsError: self.logger.info(f'skipping upload of existing blob: {data_path}') def delete(self, symbol: str, ts: BiTimestamp): self._check_closed('delete') self.index.delete(symbol, ts.as_at()) def flush(self): # sync index to disk self.index.flush() # upload index to Azure blob storage blob_client = self.storage.get_blob_client( container=self.container_name, blob='index') with open(self.index_path, 'rb') as index_data: blob_client.upload_blob(index_data, overwrite=True) def close(self): if not self.closed: self.flush() self.closed = True def _check_closed(self, operation): if self.closed: raise Exception('unable to perform operation while closed: ' + operation) def destroy(self): pass
class TorBlutmagieClient: """Simple client to query torstatus.blutmagie.de for exit nodes. The client will download http://torstatus.blutmagie.de/query_export.php/Tor_query_EXPORT.csv and check if a specified IP address, FQDN or domain is present in it. It will cache the response for `cache_duration` seconds to avoid too much latency. :param cache_duration: Duration before refreshing the cache (in seconds). Ignored if `cache_duration` is 0. :param cache_root: Path where to store the cached file downloaded from torstatus.blutmagie.de :type cache_duration: int :type cache_root: str """ def __init__(self, cache_duration=3600, cache_root='/tmp/cortex/tor_project'): self.session = requests.Session() self.cache_duration = cache_duration if self.cache_duration > 0: self.cache = Cache(cache_root) self.url = 'http://torstatus.blutmagie.de/query_export.php/Tor_query_EXPORT.csv' __cache_key = __name__ + ':raw_data' def _get_raw_data(self): try: return self.cache[self.__cache_key] except (AttributeError, TypeError): return self.session.get(self.url).text.encode('utf-8') except KeyError: self.cache.set(self.__cache_key, self.session.get(self.url).text.encode('utf-8'), expire=self.cache_duration, read=True) return self.cache[self.__cache_key] def _get_data(self): return csv.DictReader( self._get_raw_data().decode('utf-8').splitlines(), delimiter=',') def _extract_fields(self, line): return { 'hostname': line['Hostname'], 'name': line['Router Name'], 'country_code': line['Country Code'], 'ip': line['IP Address'], 'as_name': line['ASName'], 'as_number': line['ASNumber'] } def _get_node_from_domain(self, domain): results = [] for line in self._get_data(): if domain.lower() in line['Hostname'].lower(): results.append(self._extract_fields(line)) return results def _get_node_from_fqdn(self, fqdn): results = [] for line in self._get_data(): if fqdn.lower() == line['Hostname'].lower(): results.append(self._extract_fields(line)) break return results def _get_node_from_ip(self, ip): results = [] for line in self._get_data(): if ip == line['IP Address']: results.append(self._extract_fields(line)) break return results def search_tor_node(self, data_type, data): """Lookup an artifact to check if it is a known tor exit node. :param data_type: The artifact type. Must be one of 'ip', 'fqdn' or 'domain' :param data: The artifact to lookup :type data_type: str :type data: str :return: Data relative to the tor node. If the looked-up artifact is related to a tor exit node it will contain a `nodes` array. That array will contains a list of nodes containing the following keys: - name: name given to the router - ip: their IP address - hostname: Hostname of the router - country_code: ISO2 code of the country hosting the router - as_name: ASName registering the router - as_number: ASNumber registering the router Otherwise, `nodes` will be empty. :rtype: list """ results = [] if data_type == 'ip': results = self._get_node_from_ip(data) elif data_type == 'fqdn': results = self._get_node_from_fqdn(data) elif data_type == 'domain': results = self._get_node_from_domain(data) else: pass return {"nodes": results}
class LookupService(ABC): """ Abstract model for a lookup service. """ def __init__(self, config: LookupServiceConfig): self._config = config self._cache = Cache( os.path.join(os.path.dirname(__file__), '.cache', self.__class__.__name__, self._config.cache_dir())) @property def max_hits(self): return self._config.max_hits def _update_cache(self, results: List[LookupResult]): """ Update cache entries with new results :param results: a list of LookupResults :return: """ for lookup_result in results: self._cache.set(lookup_result.label, lookup_result) # ALWAYS override! def _get_cached_entries( self, labels: List[str]) -> Tuple[List[LookupResult], List[str]]: """ Return a tuple (<cached results>, <labels to lookup>) :param labels: a list of strings to lookup :return: """ to_compute = [] cached_entries = [] for label in labels: entry = self._cache.get(label) if entry is None: to_compute.append(label) else: cached_entries.append(entry) return cached_entries, to_compute def _lookup(self, labels: List[str]) -> List[LookupResult]: """ Actual lookup function. To be implemented by all the subclasses. :param labels: :return: """ raise NotImplementedError def lookup(self, labels: List[str]) -> List[LookupResult]: """ Returns a list of LookupResult, based on the given labels. Results are fetched from a local cache, if enabled. If not enabled, already cached results (if any) will be updated with the new results. :param labels: a list of strings to lookup :return: a list of LookupResult """ cached_entries, to_compute = [], labels if self._config.enable_cache: cached_entries, to_compute = self._get_cached_entries(to_compute) new_results = self._lookup(to_compute) self._update_cache(new_results) # write new entries to cache return cached_entries + new_results def lookup_subsequences(self, labels: List[str], max_subseq_len) -> List[LookupResult]: """ Given a label, the method searches for all its subsequences (from length max_subseq_len to 1) Results are then aggregated, keeping the original rank. :param labels: a list of labels :param max_subseq_len: length of the longest subsequence to compute :return: a list of LookupResult """ subsequences_dict, subsequences_set = strings_subsequences( labels, max_subseq_len) lookup_results = dict(self.lookup(list(subsequences_set))) results = [] for label in labels: # aggregate subsequences results for each label label_results = [] for subsequence in subsequences_dict[label]: label_results += lookup_results[subsequence] results.append(LookupResult(label, label_results)) return results
class URLhaus: """Simple client to query URLhaus by abuse.ch. :param query: domain, url or hash. :param cache_duration: Duration before refreshing the cache (in seconds). Ignored if `cache_duration` is 0. :param cache_root: Path where to store the cached file. :type query: string :type cache_duration: int :type cache_root: str """ def __init__(self, query, cache_duration=300, cache_root="/tmp/cortex/URLhaus"): self.URL = "https://urlhaus.abuse.ch/browse.php" self.query = query self.cache = None if cache_duration > 0: self.cache = Cache(cache_root) self.cache_duration = cache_duration def _get_raw_data(self): try: return self.cache[self.query.encode('utf-8')] except (AttributeError, TypeError): return self.fetch() except KeyError: self.cache.set(self.query.encode('utf-8'), self.fetch(), expire=self.cache_duration) return self.cache[self.query.encode('utf-8')] def search(self): res = self._get_raw_data() return self.parse(res) def fetch(self): payload = {"search": self.query} return requests.get(self.URL, params=payload).text def parse(self, doc): results = [] soup = BeautifulSoup(doc, "html.parser") table = soup.find("table", class_="table") rows = table.find_all("tr")[1:] for row in rows: cols = row.find_all("td") results.append({ "dateadded": cols[0].text, "malware_url": cols[1].text, "link": cols[1].find("a").attrs.get("href"), "status": cols[2].text, "tags": cols[3].text.split(), "gsb": cols[4].text if len(cols) > 5 else None, "reporter": cols[5].text if len(cols) > 5 else cols[4].text }) return results
class TorProjectClient: """Simple client to query torproject.org for exit nodes. The client will download https://check.torproject.org/exit-addresses and check if a specified IP address is present in it. If that IP address is found it will check for its last update time and return a description of the node if its last update time is less than `ttl` seconds ago. :param ttl: Tor node will be kept only if its last update was less than `ttl` seconds ago. Ignored if `ttl` is 0 :param cache_duration: Duration before refreshing the cache (in seconds). Ignored if `cache_duration` is 0. :param cache_root: Path where to store the cached file downloaded from torproject.org :param proxies: Proxies to be using during requests session :type ttl: int :type cache_duration: int :type cache_root: str """ def __init__( self, ttl=86400, cache_duration=3600, cache_root="/tmp/cortex/tor_project", proxies=None, ): self.session = requests.Session() if proxies: self.session.proxies.update(proxies) self.delta = None self.cache = None if ttl > 0: self.delta = timedelta(seconds=ttl) if cache_duration > 0: self.cache = Cache(cache_root) self.cache_duration = cache_duration self.url = "https://check.torproject.org/exit-addresses" __cache_key = __name__ + ":raw_data" def _get_raw_data(self): try: return self.cache["raw_data"] except (AttributeError, TypeError): return self.session.get(self.url).text except KeyError: self.cache.set( "raw_data", self.session.get(self.url).text, expire=self.cache_duration, ) return self.cache["raw_data"] def search_tor_node(self, ip): """Lookup an IP address to check if it is a known tor exit node. :param ip: The IP address to lookup :type ip: str :return: Data relative to the tor node. If `ip`is a tor exit node it will contain a `node` key with the hash of the node and a `last_status` key with the last update time of the node. If `ip` is not a tor exit node, the function will return an empty dictionary. :rtype: dict """ data = {} tmp = {} present = datetime.utcnow().replace(tzinfo=pytz.utc) for line in self._get_raw_data().splitlines(): params = line.split(" ") if params[0] == "ExitNode": tmp["node"] = params[1] elif params[0] == "ExitAddress": tmp["last_status"] = params[2] + "T" + params[3] + "+0000" last_status = parse(tmp["last_status"]) if self.delta is None or (present - last_status) < self.delta: data[params[1]] = tmp tmp = {} else: pass return data.get(ip, {})
default_flask_logger = logging.getLogger('werkzeug') default_flask_logger.setLevel(logging.CRITICAL) from publicsuffixlist import PublicSuffixList from domain_stats.config import Config from domain_stats.network_io import IscConnection import domain_stats.rdap_query as rdap from domain_stats.freq import FreqCounter from domain_stats.expiring_diskcache import ExpiringCache from diskcache import Cache #current directory must be set by launcher to the location of the config and database cache = ExpiringCache(os.getcwd()) memocache = Cache(pathlib.Path().cwd() / "memocache") if not memocache.get("rdap_good"): memocache.set("rdap_good", 0) memocache.set("rdap_fail", 0) config = Config(os.getcwd() + "/domain_stats.yaml") if config.get("enable_freq_scores"): freq = FreqCounter() freq.load(config.get("freq_table")) logfile = logging.FileHandler(str(pathlib.Path.cwd() / 'domain_stats.log')) logformat = logging.Formatter( '%(asctime)s : %(levelname)s : %(module)s : %(process)d : %(message)s') logfile.setFormatter(logformat) if config['log_detail'] == 0: log.setLevel(level=logging.CRITICAL) elif config['log_detail'] == 1: log.addHandler(logfile)
class CacheInteraction: def __init__(self, dimension=DimensionType.DIM_2D): self._cache = Cache(settings.CACHE_ROOT) self._dimension = dimension def __del__(self): self._cache.close() def get_buff_mime(self, chunk_number, quality, db_data): chunk, tag = self._cache.get('{}_{}_{}'.format(db_data.id, chunk_number, quality), tag=True) if not chunk: chunk, tag = self.prepare_chunk_buff(db_data, quality, chunk_number) self.save_chunk(db_data.id, chunk_number, quality, chunk, tag) return chunk, tag def prepare_chunk_buff(self, db_data, quality, chunk_number): from cvat.apps.engine.frame_provider import FrameProvider # TODO: remove circular dependency writer_classes = { FrameProvider.Quality.COMPRESSED: Mpeg4CompressedChunkWriter if db_data.compressed_chunk_type == DataChoice.VIDEO else ZipCompressedChunkWriter, FrameProvider.Quality.ORIGINAL: Mpeg4ChunkWriter if db_data.original_chunk_type == DataChoice.VIDEO else ZipChunkWriter, } image_quality = 100 if writer_classes[quality] in [ Mpeg4ChunkWriter, ZipChunkWriter ] else db_data.image_quality mime_type = 'video/mp4' if writer_classes[quality] in [ Mpeg4ChunkWriter, Mpeg4CompressedChunkWriter ] else 'application/zip' kwargs = {} if self._dimension == DimensionType.DIM_3D: kwargs["dimension"] = DimensionType.DIM_3D writer = writer_classes[quality](image_quality, **kwargs) images = [] buff = BytesIO() upload_dir = { StorageChoice.LOCAL: db_data.get_upload_dirname(), StorageChoice.SHARE: settings.SHARE_ROOT, StorageChoice.CLOUD_STORAGE: db_data.get_upload_dirname(), }[db_data.storage] if hasattr(db_data, 'video'): source_path = os.path.join(upload_dir, db_data.video.path) reader = VideoDatasetManifestReader( manifest_path=db_data.get_manifest_path(), source_path=source_path, chunk_number=chunk_number, chunk_size=db_data.chunk_size, start=db_data.start_frame, stop=db_data.stop_frame, step=db_data.get_frame_step()) for frame in reader: images.append((frame, source_path, None)) else: reader = ImageDatasetManifestReader( manifest_path=db_data.get_manifest_path(), chunk_number=chunk_number, chunk_size=db_data.chunk_size, start=db_data.start_frame, stop=db_data.stop_frame, step=db_data.get_frame_step()) if db_data.storage == StorageChoice.CLOUD_STORAGE: db_cloud_storage = db_data.cloud_storage assert db_cloud_storage, 'Cloud storage instance was deleted' credentials = Credentials() credentials.convert_from_db({ 'type': db_cloud_storage.credentials_type, 'value': db_cloud_storage.credentials, }) details = { 'resource': db_cloud_storage.resource, 'credentials': credentials, 'specific_attributes': db_cloud_storage.get_specific_attributes() } try: cloud_storage_instance = get_cloud_storage_instance( cloud_provider=db_cloud_storage.provider_type, **details) cloud_storage_instance.initialize_content() for item in reader: file_name = f"{item['name']}{item['extension']}" if file_name not in cloud_storage_instance: raise Exception( '{} file was not found on a {} storage'.format( file_name, cloud_storage_instance.name)) with NamedTemporaryFile(mode='w+b', prefix='cvat', suffix=file_name.replace( os.path.sep, '#'), delete=False) as temp_file: source_path = temp_file.name buf = cloud_storage_instance.download_fileobj( file_name) temp_file.write(buf.getvalue()) checksum = item.get('checksum', None) if not checksum: slogger.cloud_storage[ db_cloud_storage.id].warning( 'A manifest file does not contain checksum for image {}' .format(item.get('name'))) if checksum and not md5_hash( source_path) == checksum: slogger.cloud_storage[ db_cloud_storage.id].warning( 'Hash sums of files {} do not match'. format(file_name)) images.append((source_path, source_path, None)) except Exception as ex: storage_status = cloud_storage_instance.get_status() if storage_status == Status.FORBIDDEN: msg = 'The resource {} is no longer available. Access forbidden.'.format( cloud_storage_instance.name) elif storage_status == Status.NOT_FOUND: msg = 'The resource {} not found. It may have been deleted.'.format( cloud_storage_instance.name) else: # check status of last file file_status = cloud_storage_instance.get_file_status( file_name) if file_status == Status.NOT_FOUND: raise Exception( "'{}' not found on the cloud storage '{}'". format(file_name, cloud_storage_instance.name)) elif file_status == Status.FORBIDDEN: raise Exception( "Access to the file '{}' on the '{}' cloud storage is denied" .format(file_name, cloud_storage_instance.name)) msg = str(ex) raise Exception(msg) else: for item in reader: source_path = os.path.join( upload_dir, f"{item['name']}{item['extension']}") images.append((source_path, source_path, None)) writer.save_as_chunk(images, buff) buff.seek(0) if db_data.storage == StorageChoice.CLOUD_STORAGE: images = [image[0] for image in images if os.path.exists(image[0])] for image_path in images: os.remove(image_path) return buff, mime_type def save_chunk(self, db_data_id, chunk_number, quality, buff, mime_type): self._cache.set('{}_{}_{}'.format(db_data_id, chunk_number, quality), buff, tag=mime_type)