class URLhaus: """Simple client to query URLhaus by abuse.ch. :param query: domain, url or hash. :param cache_duration: Duration before refreshing the cache (in seconds). Ignored if `cache_duration` is 0. :param cache_root: Path where to store the cached file. :type query: string :type cache_duration: int :type cache_root: str """ def __init__(self, query, cache_duration=300, cache_root="/tmp/cortex/URLhaus"): self.URL = "https://urlhaus.abuse.ch/browse.php" self.query = query self.cache = None if cache_duration > 0: self.cache = Cache(cache_root) self.cache_duration = cache_duration def _get_raw_data(self): try: return self.cache[self.query.encode('utf-8')] except(AttributeError, TypeError): return self.fetch() except KeyError: self.cache.set( self.query.encode('utf-8'), self.fetch(), expire=self.cache_duration) return self.cache[self.query.encode('utf-8')] def search(self): res = self._get_raw_data() return self.parse(res) def fetch(self): payload = {"search": self.query} return requests.get(self.URL, params=payload).text def parse(self, doc): results = [] soup = BeautifulSoup(doc, "html.parser") table = soup.find("table", class_="table") rows = table.find_all("tr")[1:] for row in rows: cols = row.find_all("td") results.append({ "dateadded": cols[0].text, "malware_url": cols[1].text, "link": cols[1].find("a").attrs.get("href"), "status": cols[2].text, "tags": cols[3].text.split(), "gsb": cols[4].text, "reporter": cols[5].text }) return results
def _set_cached(path, content): # 1/ memory cache JUMBO_FIELDS_MEMORY_CACHE[path] = content # 2/ disk cache if SIMPLEFLOW_ENABLE_DISK_CACHE: try: cache = Cache(constants.CACHE_DIR) cache_key = "jumbo_fields/" + path.split("/")[-1] logger.debug("diskcache: setting key={} on cache_dir={}".format(cache_key, constants.CACHE_DIR)) cache.set(cache_key, content, expire=3 * constants.HOUR) except OperationalError: logger.warning("diskcache: got an OperationalError on write, skipping cache write")
def worker(queue, eviction_policy): timings = {'get': [], 'set': [], 'delete': []} cache = Cache('tmp', eviction_policy=eviction_policy) for index, (action, key, value) in enumerate(iter(queue.get, None)): start = time.time() if action == 'set': cache.set(key, value, expire=EXPIRE) elif action == 'get': result = cache.get(key) else: assert action == 'delete' cache.delete(key) stop = time.time() if action == 'get' and PROCESSES == 1 and THREADS == 1 and EXPIRE is None: assert result == value if index > WARMUP: timings[action].append(stop - start) queue.put(timings) cache.close()
def __init__(self, query, cache_duration=300, cache_root="/tmp/cortex/URLhaus"): self.URL = "https://urlhaus.abuse.ch/browse.php" self.query = query self.cache = None if cache_duration > 0: self.cache = Cache(cache_root) self.cache_duration = cache_duration
def __init__(self, ttl=86400, cache_duration=3600, cache_root='/tmp/cortex/tor_project'): self.session = requests.Session() self.delta = None self.cache = None if ttl > 0: self.delta = timedelta(seconds=ttl) if cache_duration > 0: self.cache = Cache(cache_root) self.cache_duration = cache_duration self.url = 'https://check.torproject.org/exit-addresses'
def key_in_cache(key: tuple[str, bool]) -> bool: """Checks if key exists in cache.""" with Cache(CACHE_FILE) as cache: return key in cache
mongo = PyMongo(application) mongo_server = "localhost" mongo_port = "27017" AUTH_KEY = "14314314314314314314314314314314" # 32 char's Authentication key str = "mongodb://" + mongo_server + ":" + mongo_port connection = MongoClient(str) # connection to DB db = connection.DFS servers = db.servers SERVER_HOST = None SERVER_PORT = None cache = Cache('/tmp/mycachedir') # directory path is used to store cache def decrypt_data(key, hashed_val): decrypted_data = AES.new(key, AES.MODE_ECB).decrypt(base64.b64decode(hashed_val)) return decrypted_data def server_inst(): with application.app_context(): return db.servers.find_one({"host": SERVER_HOST, "port": SERVER_PORT}) # to upload @application.route('/upload_f', methods=['POST'])
#!/usr/bin/env python3 """ Clears cache from the default tmp directory """ from diskcache import Cache import tempfile cache = Cache(tempfile.gettempdir()) cache.clear()
class Webserver: def __init__( self, host="localhost", port=8080, hostname='hostname', workers=os.cpu_count() - 1, max_connections=socket.SOMAXCONN, max_cache_size=4e9, proto='http', ): self._max_connections = max_connections self.host = host self.port = port self.address = f'{proto}://{self.host}:{self.port}' self.hostname = hostname self.max_workers = workers self.routes = {} self.vservers_routes = {} # роуты для вирт. сервера self.regular_routes = {} self._response = None self._current_clients = [] self.data_end = b'\r\n\r\n' self.connect_refresh_timeout = 10 self.client_request_timeout = 0.1 self._max_keep_alive = timedelta(seconds=10) self.cache = Cache(size_limit=int(max_cache_size)) self.cache_expire = 100 self._running = True def set_routes(self, routes): self.routes = {**routes, **self.routes} def get_routes(self): return self.routes def handle_file(self, request, filename, root=os.getcwd(), content_type=None): path = os.path.join(root, filename) # Пробуем взять отдаваемый стат. файл из cache res = self.cache.get(path) if res: return res if not os.path.exists(path): self.cache.set(path, Errors.NOT_FOUND_PAGE, expire=self.cache_expire, tag='data') self.cache.cull() return Errors.NOT_FOUND_PAGE if content_type is None: content_type, _ = mimetypes.guess_type(path) logger.verbose('Content type for %s, was set automatically to %s', path, content_type) response = Response.response_file(request, path, content_type) if response.status == 200: # Если код 200 OK, то добавим response в cache self.cache.set(path, response, expire=self.cache_expire, tag='data') self.cache.cull() return response def handle_dir(self, request, dirname=os.getcwd()): path = os.path.abspath(dirname) if not os.path.exists(path): return Errors.NOT_FOUND_PAGE response = Response.response_dir(request, path) return response def get_routes_for_virtual_server(self, request): vserver_name = request.headers['Host'] routes = self.vservers_routes.get(vserver_name) if routes: logger.debug('Find routes for vservers %s', vserver_name) return routes return self.regular_routes def find_handler(self, request: Request): logger.debug('%s', self.regular_routes) try: response = Errors.NOT_FOUND_PAGE for path_info, handler in self.get_routes_for_virtual_server( request).items(): logger.debug('checking %s %s | %s', path_info.path, path_info.method, handler) path_info: PathInfo = path_info reg = path_info.path match = re.fullmatch(reg, request.url.path) if match and request.method == path_info.method: logger.debug('handler found %s for %s', handler, request.url) if len(match.groupdict().items()) == 0: response = handler(request) else: response = handler(request, match.group(1)) break logger.debug('handler returned %s', response) return response except Exception: logger.exception("Internal Error") return Errors.INTERNAL_SERVER_ERROR def _read_data(self, client, address): _keep_alive_deadline = datetime.now() + self._max_keep_alive request = Request() filled = False while not filled: try: line = client.recv(64 * 1024) if not line: raise ReceiveDone('Client done') split = Request.split_keep_sep(line, bytes(os.linesep, 'utf-8')) for s in split: if request.dynamic_fill(s): filled = True except BlockingIOError: # читали, а нам никто не написал, подождем и может напишут time.sleep(0.5) _current_time = datetime.now() if _current_time > _keep_alive_deadline: # ждали больше чем держим keep-alive соединение raise ConnectionError('Client dead') return request def _start_session(self, client, address): while True: request = self._read_data(client, address) request.print_headers() request.print_body() response = self.find_handler(request) logger.access(method=request.method, url=request.target, protocol=request.version, status=response.status) Response.response(client, response) # закончили обрабатывать запрос от клиента # (получили, ответили, пришли сюда) if request.headers.get('Connection') == 'keep-alive': logger.verbose("Client %s asked for keep-alive connection", address) # если браузер послал keep-alive (что он почти всегда # делает дабы избавиться от поспоянных созданий # подключений и ускорить общение с сервером) client.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) # и продолжаем крутиться в while True: else: # иначе выходим из while True: (клиент не хочет больше # с нами общаться) break def handle(self, client, address): init_thread_local(THREAD_LOCAL_DATA) THREAD_LOCAL_DATA.client = address with client: try: self._start_session(client, address) except ConnectionError: logger.verbose('Client was not alive for %s seconds', self._max_keep_alive) except OSError: logger.exception('Client had an issue') except ReceiveDone: logger.verbose('Client received the content') except Exception: logger.exception('Client had an error') Response.response(client, Errors.INTERNAL_SERVER_ERROR) logger.verbose("Disconnected") def run(self, stop: Event): self._running = True server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.make_regular_routes() with server_socket: server_socket.bind((self.host, self.port)) server_socket.listen(self._max_connections) server_socket.setblocking(False) logger.verbose(f'Start server on {self.address}') logger.debug('T %s', server_socket.gettimeout()) with ThreadPoolExecutor(max_workers=self.max_workers) as executor: logger.verbose('ThreadPoolExecutor started') while not stop.is_set(): try: self._check_is_running() try: client, address = server_socket.accept() THREAD_LOCAL_DATA.client = address logger.verbose(f'Got client') future = executor.submit(self.handle, client, address) self._current_clients.append(future) except (socket.timeout, BlockingIOError): # никто не подключился, можно подождать # logger.info(f'No client') time.sleep(1) pass except KeyboardInterrupt: logger.verbose('Server shutdown started') break logger.verbose(('ThreadPoolExecutor ' 'waiting for alive connections')) for future in self._current_clients: result = future.result() logger.verbose('ThreadPoolExecutor finished') def _check_is_running(self): if not self._running: raise KeyboardInterrupt def stop(self): self._running = False def route(self, path, method='GET', vserver='localhost'): logger.debug('Adding %s [%s]', path, method) if path in self.routes: raise AssertionError("Such route already exists.") def wrapper(handler): path_info = PathInfo(path, method) if vserver == 'localhost': self.set_routes({path_info: handler}) if vserver in self.vservers_routes: self.vservers_routes[vserver][path_info] = handler logger.debug('Add route %s for vserver %s', path_info, vserver) else: self.vservers_routes[vserver] = {path_info: handler} logger.debug('INIT route %s for vserver %s', path_info, vserver) return handler return wrapper def make_regular_routes(self): logger.debug('Processing routes %s', self.routes) for path_info, handler in self.routes.items(): reg = re.compile(path_info.path) path_info.path = reg self.regular_routes[path_info] = handler
from diskcache import Cache with Cache('../db') as cache: print('Heartrate: ', cache[b'heartrate']) print('Speed: ', cache[b'speed']) print('Cadence: ', cache[b'cadence'])
class TorBlutmagieClient: """Simple client to query torstatus.blutmagie.de for exit nodes. The client will download http://torstatus.blutmagie.de/query_export.php/Tor_query_EXPORT.csv and check if a specified IP address, FQDN or domain is present in it. It will cache the response for `cache_duration` seconds to avoid too much latency. :param cache_duration: Duration before refreshing the cache (in seconds). Ignored if `cache_duration` is 0. :param cache_root: Path where to store the cached file downloaded from torstatus.blutmagie.de :type cache_duration: int :type cache_root: str """ def __init__(self, cache_duration=3600, cache_root='/tmp/cortex/tor_project'): self.session = requests.Session() self.cache_duration = cache_duration if self.cache_duration > 0: self.cache = Cache(cache_root) self.url = 'http://torstatus.blutmagie.de/query_export.php/Tor_query_EXPORT.csv' __cache_key = __name__ + ':raw_data' def _get_raw_data(self): try: return self.cache[self.__cache_key] except (AttributeError, TypeError): return self.session.get(self.url).text.encode('utf-8') except KeyError: self.cache.set( self.__cache_key, self.session.get(self.url).text.encode('utf-8'), expire=self.cache_duration, read=True) return self.cache[self.__cache_key] def _get_data(self): return csv.DictReader( self._get_raw_data().decode('utf-8').splitlines(), delimiter=',') def _extract_fields(self, line): return { 'hostname': line['Hostname'], 'name': line['Router Name'], 'country_code': line['Country Code'], 'ip': line['IP Address'], 'as_name': line['ASName'], 'as_number': line['ASNumber'] } def _get_node_from_domain(self, domain): results = [] for line in self._get_data(): if domain.lower() in line['Hostname'].lower(): results.append(self._extract_fields(line)) return results def _get_node_from_fqdn(self, fqdn): results = [] for line in self._get_data(): if fqdn.lower() == line['Hostname'].lower(): results.append(self._extract_fields(line)) break return results def _get_node_from_ip(self, ip): results = [] for line in self._get_data(): if ip == line['IP Address']: results.append(self._extract_fields(line)) break return results def search_tor_node(self, data_type, data): """Lookup an artifact to check if it is a known tor exit node. :param data_type: The artifact type. Must be one of 'ip', 'fqdn' or 'domain' :param data: The artifact to lookup :type data_type: str :type data: str :return: Data relative to the tor node. If the looked-up artifact is related to a tor exit node it will contain a `nodes` array. That array will contains a list of nodes containing the following keys: - name: name given to the router - ip: their IP address - hostname: Hostname of the router - country_code: ISO2 code of the country hosting the router - as_name: ASName registering the router - as_number: ASNumber registering the router Otherwise, `nodes` will be empty. :rtype: list """ results = [] if data_type == 'ip': results = self._get_node_from_ip(data) elif data_type == 'fqdn': results = self._get_node_from_fqdn(data) elif data_type == 'domain': results = self._get_node_from_domain(data) else: pass return {"nodes": results}
#!/usr/bin/env python import requests import re import sys import dlrnapi_client import influxdb_utils from promoter_utils import get_dlrn_instance_for_release from diskcache import Cache cache = Cache('/tmp/skipped_promotions_cache') cache.expire() promoter_skipping_regex = re.compile( ('.*promoter Skipping promotion of (.*) from (.*) to (.*), ' 'missing successful jobs: (.*)')) html_link = "<a href='{}' target='_blank' >{}</a>" def get_failing_jobs_html(dlrn_hashes, release_name): failing_jobs_html = "" # If any of the jobs is still in progress in_progress = False try: dlrn = get_dlrn_instance_for_release(release_name) if dlrn: params = dlrnapi_client.Params2() params.commit_hash = dlrn_hashes['commit_hash'] params.distro_hash = dlrn_hashes['distro_hash']
import time import tempfile import asyncio import re import os import sys # local import from getmyancestors import Session, Tree, Indi, Fam from mergemyancestors import Gedcom from translation import translations tmp_dir = os.path.join(tempfile.gettempdir(), 'fstogedcom') global cache cache = Cache(tmp_dir) lang = cache.get('lang') def _(string): if string in translations and lang in translations[string]: return translations[string][lang] return string # Entry widget with right-clic menu to copy/cut/paste class EntryWithMenu(Entry): def __init__(self, master, **kw): super(EntryWithMenu, self).__init__(master, **kw) self.bind('<Button-3>', self.click_right)
from diskcache import Cache task_queue = Cache('/tmp/ad-poster') # print dir(task_queue) print "Count:", task_queue.count print "Cleared:", task_queue.clear()
class TorProjectClient: """Simple client to query torproject.org for exit nodes. The client will download https://check.torproject.org/exit-addresses and check if a specified IP address is present in it. If that IP address is found it will check for its last update time and return a description of the node if its last update time is less than `ttl` seconds ago. :param ttl: Tor node will be kept only if its last update was less than `ttl` seconds ago. Ignored if `ttl` is 0 :param cache_duration: Duration before refreshing the cache (in seconds). Ignored if `cache_duration` is 0. :param cache_root: Path where to store the cached file downloaded from torproject.org :type ttl: int :type cache_duration: int :type cache_root: str """ def __init__(self, ttl=86400, cache_duration=3600, cache_root='/tmp/cortex/tor_project'): self.session = requests.Session() self.delta = None self.cache = None if ttl > 0: self.delta = timedelta(seconds=ttl) if cache_duration > 0: self.cache = Cache(cache_root) self.cache_duration = cache_duration self.url = 'https://check.torproject.org/exit-addresses' __cache_key = __name__ + ':raw_data' def _get_raw_data(self): try: return self.cache['raw_data'] except(AttributeError, TypeError): return self.session.get(self.url).text except KeyError: self.cache.set( 'raw_data', self.session.get(self.url).text, expire=self.cache_duration) return self.cache['raw_data'] def search_tor_node(self, ip): """Lookup an IP address to check if it is a known tor exit node. :param ip: The IP address to lookup :type ip: str :return: Data relative to the tor node. If `ip`is a tor exit node it will contain a `node` key with the hash of the node and a `last_status` key with the last update time of the node. If `ip` is not a tor exit node, the function will return an empty dictionary. :rtype: dict """ data = {} tmp = {} present = datetime.utcnow().replace(tzinfo=pytz.utc) for line in self._get_raw_data().splitlines(): params = line.split(' ') if params[0] == 'ExitNode': tmp['node'] = params[1] elif params[0] == 'ExitAddress': tmp['last_status'] = params[2] + 'T' + params[3] + '+0000' last_status = parse(tmp['last_status']) if (self.delta is None or (present - last_status) < self.delta): data[params[1]] = tmp tmp = {} else: pass return data.get(ip, {})
def clear_cache() -> None: """Clear everything from cache.""" with Cache(CACHE_FILE) as cache: cache.clear()
import random import re import ssl import threading import time # Load configuration from settings.json with open("settings.json") as file: configuration = json.loads(file.read()) PING_SLEEP = 45 ## Cache GIBIBYTE = 2**30 MEBIBYTE = 2**20 cache = Cache("./cache", size_limit=configuration["max_cache_size_in_mebibytes"] * MEBIBYTE) ## # Sanic configuration ## LOGGING = log.LOGGING_CONFIG_DEFAULTS LOGGING["handlers"]["file"] = { 'class': 'logging.FileHandler', 'formatter': 'generic', 'filename': "log/latest.log", 'mode': 'a' } LOGGING["handlers"]["error_file"] = { 'class': 'logging.FileHandler', 'formatter': 'access',
from diskcache import Cache from pathlib import Path from config import BASE_DIR queue_cache = Cache(Path.joinpath(BASE_DIR, '.cache/queue')) widgets_cache = Cache(Path.joinpath(BASE_DIR, '.cache/widgets')) api_cache = Cache(Path.joinpath(BASE_DIR, '.cache/api')) notice_cache = Cache(Path.joinpath(BASE_DIR, '.cache/notice'))
import user_preference.user_preference as up from diskcache import Cache app = Flask(__name__) nav = Nav(app) nav.register_element( 'my_navbar', Navbar('thenav', View('Home', 'home'), View('Add', 'add'), View('Remove', 'remove'), View('Return', 'ret'), View('Take', 'take'))) serialcomm = serial.Serial('/dev/cu.usbmodem101', 9600) database = [] angle_storage = Cache("angle_storage") angle_key = "cur_angle" servo_angle = angle_storage.get(angle_key) if servo_angle == None: servo_angle = 0 cur_type_of_clothes = "" cur_color = "" class_lookup = [ "Anorak", "Blazer", "Blouse", "Bomber", "Button-Down", "Caftan", "Capris", "Cardigan", "Chinos", "Coat, Coverup", "Culottes", "Cutoffs", "Dress", "Flannel", "Gauchos", "Halter", "Henley", "Hoodie", "Jacket", "Jeans", "Jeggings", "Jersey", "Jodhurs", "Joggers", "Jumpsuit", "Kaftan", "Kimono", "Leggings", "Onesie", "Parka", "Peacoat", "Poncho", "Robe", "Romper", "Sarong", "Shorts", "Skirt", "Sweater", "Sweatpants", "Sweatshorts", "Tank", "Tee", "Top", "Trunks", "Turtleneck" ]
from tkinter import Variable import os import appdirs from diskcache import Cache __temp_dir = appdirs.user_data_dir(appname='mrplayer_live', appauthor='gzlock') print('cache dir', __temp_dir) cache = Cache(__temp_dir) log_file = os.path.join(__temp_dir, 'log.txt') def tkVariable(var: Variable, key: str, default=None): """ 将tkinter的var绑定到缓存 :param var: :param key: :param default: :return: """ _cache = cache.get(key, default=None) if _cache is None: if default is not None: var.set(default) else: var.set(_cache) var.trace('w', lambda a, b, c: cache.set(key, var.get()))
from diskcache import Cache import pypi_client pypi_client.cache = Cache()
def persistent_caching_fn(fn, name, check_cache_exists=False, cache_dir=None, cache_dirs=None, cache_allow_writes=True, retries=2) -> Callable: wait_time = 0.25 random_time = 0.25 try: cache_dirs = get_global( "cache_dirs") if cache_dirs is None else cache_dirs except: if cache_dirs is None: cache_dir = get_global( "cache_dir") if cache_dir is None else cache_dir cache_dirs = [cache_dir] try: cache_allow_writes = get_global("cache_allow_writes") except: pass try: cache_stats = get_global("cache_stats") except: cache_stats = defaultdict(lambda: defaultdict(float)) set_global("cache_stats", cache_stats) cache_stats["count_cache_dirs"] = len(set(cache_dirs)) cache_stats["cache_dirs"] = set(cache_dirs) from diskcache import Cache import joblib if check_cache_exists: assert os.path.exists(cache_dir) and os.path.isdir(cache_dir) cache_file = os.path.join(cache_dir, "cache.db") assert os.path.exists(cache_file) and os.path.isfile(cache_file) else: if os.path.exists(cache_dir): assert os.path.isdir(cache_dir) else: os.mkdir(cache_dir) args = dict(eviction_policy='none', sqlite_cache_size=2**16, sqlite_mmap_size=2**28, disk_min_file_size=2**18) caches = [Cache(cd, **args) for cd in cache_dirs] shuffle(caches) try: import inspect fnh = joblib.hashing.hash(name, 'sha1') + joblib.hashing.hash( inspect.getsourcelines(fn)[0], 'sha1') + joblib.hashing.hash( fn.__name__, 'sha1') except Exception as e: try: fnh = joblib.hashing.hash(name, 'sha1') + joblib.hashing.hash( fn.__name__, 'sha1') except Exception as e: fnh = joblib.hashing.hash(name, 'sha1') def build_hash(*args, **kwargs): hsh = fnh + joblib.hashing.hash(args, 'sha1') if len(kwargs) > 0: hsh = hsh + joblib.hashing.hash(kwargs, 'sha1') return hsh def read_hash(hsh): ts = time.time() kes = [0] * len(caches) for retry in range(retries): for cidx, cache in enumerate(caches): if kes[cidx] == 1: continue try: r = cache[hsh] cache_stats[name]["hit"] += 1 te = time.time() - ts cache_stats[name]["read_time"] = 0.9 * cache_stats[name][ "read_time"] + 0.1 * te return r except KeyError: cache_stats[name]["key_error"] += 1 kes[cidx] = 1 except Exception as e: cache_stats[name]["read_exception"] += 1 cache_stats[name]["read_retries"] += 1 sleep(wait_time * (retry + 1) + random() * random_time) sleep(wait_time * (retry + 1) + random() * random_time) cache_stats[name]["read-return-none"] += 1 if sum(kes) >= 1: return "ke", kes return None def write_hsh(hsh, r, kes): ts = time.time() if cache_allow_writes: for retry in range(retries): for cidx, cache in enumerate(caches): if kes[cidx] != 1: continue try: cache[hsh] = r cache_stats[name]["writes"] += 1 te = time.time() - ts cache_stats[name]["write_time"] = 0.9 * cache_stats[ name]["write_time"] + 0.1 * te return r except: cache_stats[name]["write_exception"] += 1 sleep(wait_time * (retry + 1) + random() * random_time) cache_stats[name]["write_retries"] += 1 sleep(wait_time * (retry + 1) + random() * random_time) def cfn(*args, **kwargs): ignore_cache = kwargs.pop("ignore_cache", False) if ignore_cache: r = fn(*args, **kwargs) return r hsh = build_hash(*args, **kwargs) cache_stats[name]["called"] += 1 r = read_hash(hsh) if r is not None: if not isinstance(r, tuple) or (isinstance(r, tuple) and r[0] != "ke"): return r if r is None: r = fn(*args, **kwargs) cache_stats[name]["re-compute"] += 1 cache_stats[name]["re-compute-cache-busy-no-write"] += 1 return r kes = r[1] r = fn( *args, **kwargs ) # r is not None and there was key-error so we need to calculate the key and put in cache cache_stats[name]["compute"] += 1 write_hsh(hsh, r, kes) return r return cfn
from ontobio.util.scigraph_util import get_nodes_from_ids, get_id_type_map, get_taxon from typing import List, Optional, Dict, Tuple, Union, FrozenSet from json.decoder import JSONDecodeError from diskcache import Cache import tempfile import logging import requests """ Functions that directly access the owlsim rest API were kept outside the class to utilize caching TODO: test if this is still the case with diskcache """ TIMEOUT = get_config().owlsim2.timeout cache = Cache(tempfile.gettempdir()) @cache.memoize() def search_by_attribute_set(url: str, profile: FrozenSet[str], limit: Optional[int] = 100, namespace_filter: Optional[str] = None) -> Dict: """ Given a list of phenotypes, returns a ranked list of individuals individuals can be filtered by namespace, eg MONDO, MGI, HGNC :returns Dict with the structure: { 'unresolved' : [...] 'query_IRIs' : [...] 'results': {...} }
import re import unicodedata import urllib.request import ssl from diskcache import Cache c = Cache(directory='/tmp') class TtsScraper(object): ssl._create_default_https_context = ssl._create_unverified_context MANUAL_ADDITIONS = [ 'arko', 'Siliski Soaps Unscented', 'Zingari The Duo', 'Zingari The Watchman', ] @classmethod @c.memoize() def get_tts_soaps(cls): def _fixup_soap_name(name): replacements = [ ('-', ' '), ('|', ''), ('!', ''), (''', "'"), ('"', '"'), ('&', '&'),
return any(c in '1234567890,.-' for c in lval) if lkey == 'crs' and lval[:5] == 'epsg:' and lval[5:].isnumeric(): return True if lkey in ['height', 'width'] and lval.isnumeric(): return True return False ''' ' INITIALISATION - Executed upon startup only. ' Loads all the model parameters and WFS services from cache or creates them ''' PARAM_CACHE_KEY = 'model_parameters' WFS_CACHE_KEY = 'wfs_dict' try: with Cache(CACHE_DIR) as cache: G_PARAM_DICT = cache.get(PARAM_CACHE_KEY) G_WFS_DICT = cache.get(WFS_CACHE_KEY) if G_PARAM_DICT is None or G_WFS_DICT is None: G_PARAM_DICT, G_WFS_DICT = get_cached_parameters() cache.add(PARAM_CACHE_KEY, G_PARAM_DICT) cache.add(WFS_CACHE_KEY, G_WFS_DICT) except OSError as os_exc: LOGGER.error("Cannot fetch parameters & wfs from cache: %s", str(os_exc)) def application(environ, start_response): ''' MAIN - This is called whenever an HTTP request arrives ''' doc_root = os.path.normcase(environ['DOCUMENT_ROOT'])
import bonobo import pickle import hashlib from diskcache import Cache from time import sleep from tqdm import tqdm def hash(data): m = hashlib.sha256() m.update(pickle.dumps(data)) return m.hexdigest() cache = Cache("pipeline") def cached(func): def wrapper_cached(*args, **kwargs): key = hash(hash(func.__name__) + hash(args) + hash(kwargs)) if key in cache: return cache[key] else: value = func(*args, **kwargs) cache[key] = value return value return wrapper_cached def generate_data():
import os CACHE_REQUESTS = "CACHE_REQUESTS" in os.environ print('CACHE_REQUESTS', CACHE_REQUESTS) cache = None if CACHE_REQUESTS: from diskcache import Cache cache = Cache(directory='/tmp/.cache') CACHE_VALIDITY = 3600 * 6 # 6 hours import requests import urllib def FetchPage(source): if CACHE_REQUESTS: with Cache(cache.directory) as store: html = store.get(source) if not html: f = urllib.request.urlopen(source) html = f.read() store.set(source, html, expire=CACHE_VALIDITY) return html else: f = urllib.request.urlopen(source) return f.read()
import pickle from concurrent.futures import ThreadPoolExecutor from pprint import pprint import threading import time import requests from cytoolz import mapcat from cytoolz import merge from cytoolz import partial from cytoolz import partition_all from cytoolz import sliding_window from diskcache import Cache from retrying import retry cache = Cache("edsm-cache") def batched(num): """Run a function over its list argument in batches of size `num`.""" def _batched(func): def _wrapped(total): batches = partition_all(num, total) combined = mapcat(func, batches) return list(combined) return _wrapped return _batched
def add_to_cache(key: tuple[str, bool], value: str) -> None: """Add key, value pair to cache.""" with Cache(CACHE_FILE) as cache: cache.add(key, value)
def __init__(self): self.cache = Cache( directory=os.path.dirname(os.path.abspath(__file__)) + "/../../assets/.cache/" )
def get_cache_value(key: tuple[str, bool]) -> str: """Get cache value, given its key.""" with Cache(CACHE_FILE) as cache: return cache.get(key)
import re import subprocess import sys import threading import time import webbrowser # to open link on browser from collections import namedtuple from typing import Tuple from urllib import request import requests from diskcache import Cache import services as s cache = Cache(os.path.join(s.SETTINGS_DIR, 'cache')) if sys.platform == "win32": import win32process import psutil import win32gui elif sys.platform == "linux": import dbus elif sys.platform == "darwin": import applescript class Song: name = "" artist = "" album = "UNKNOWN"
def get_cache_keys() -> list[Cache.iterkeys]: """Get every key from cache.""" with Cache(CACHE_FILE) as cache: return list(cache.iterkeys())
class FileDirCache(MutableMapping): def __init__( self, use_listings_cache=True, listings_expiry_time=None, listings_cache_location=None, **kwargs, ): """ Parameters ---------- use_listings_cache: bool If False, this cache never returns items, but always reports KeyError, and setting items has no effect listings_expiry_time: int or float (optional) Time in seconds that a listing is considered valid. If None, listings do not expire. listings_cache_location: str (optional) Directory path at which the listings cache file is stored. If None, an autogenerated path at the user folder is created. """ import appdirs from diskcache import Cache listings_expiry_time = listings_expiry_time and float(listings_expiry_time) if listings_cache_location: listings_cache_location = Path(listings_cache_location) / str(listings_expiry_time) listings_cache_location.mkdir(exist_ok=True, parents=True) else: listings_cache_location = Path(appdirs.user_cache_dir(appname="fsspec_dircache")) / str( listings_expiry_time ) try: listings_cache_location.mkdir(exist_ok=True, parents=True) except Exception: logger.error(f"folder for dircache could not be created at {listings_cache_location}") self.cache_location = listings_cache_location self._cache = Cache(directory=listings_cache_location) self.use_listings_cache = use_listings_cache self.listings_expiry_time = listings_expiry_time def __getitem__(self, item): """Draw item as fileobject from cache, retry if timeout occurs""" return self._cache.get(key=item, read=True, retry=True) def clear(self): self._cache.clear() def __len__(self): return len(list(self._cache.iterkeys())) def __contains__(self, item): value = self._cache.get(item, retry=True) # None, if expired if value: return True return False def __setitem__(self, key, value): if not self.use_listings_cache: return self._cache.set(key=key, value=value, expire=self.listings_expiry_time, retry=True) def __delitem__(self, key): del self._cache[key] def __iter__(self): return (k for k in self._cache.iterkeys() if k in self) def __reduce__(self): return ( FileDirCache, (self.use_listings_cache, self.listings_expiry_time, self.cache_location), )
application = Flask(__name__) mongo = PyMongo(application) mongo_server = "localhost" mongo_port = "27017" str = "mongodb://" + mongo_server + ":" + mongo_port connection = MongoClient(str) db = connection.filesystem servers = db.filesystem_servers server_transactions = transaction_sv() AUTH_KEY = "qwerty123456789asdfghjkladitya12" SERVER_HOST = None SERVER_PORT = None # Setting the cache location cache = Cache('/mycachedir') def file_upload(file, directory, headers): print "\n Uploading file ...\n" transaction_sv.uploadT(file, directory, headers) def decrypt_data(key, hashed_val): #decrypting the key decrypted_data = AES.new(key, AES.MODE_ECB).decrypt( base64.b64decode(hashed_val)) return decrypted_data def server_instance(): with application.app_context():
# memory speed benchmark storage = dict() start = time.time() for i in range(N): storage[i] = i sum = 0 for i in range(N): sum += storage[random.randrange(0, N)] end = time.time() mem_time = end - start print("memory time:", mem_time, sum) # diskcache speed benchmark storage = Cache("test") start = time.time() for i in range(N): storage[i] = i sum = 0 for i in range(N): sum += storage[random.randrange(0, N)] end = time.time() dc_time = end - start print("diskcache time:", dc_time, sum)
def _get_cache(): from os.path import expanduser, join home = expanduser("~") cache = Cache(join(home, ".aws-qc-cahce")) return cache
class PandaXT: """A "ccxt" exchanges wrapper class over Pandas lib.""" def __init__(self, exchange, api_key=None, secret=None, user_agent=None, clear_cache=False) -> NoReturn: """Class constructor. >>> markets = PandaXT('binance').markets >>> isinstance(markets, Markets) True >>> market = markets.get('BTC/USDT') # type: Market >>> isinstance(market, Market) True >>> isinstance(market.precision.price, int) True :param Text exchange: a ccxt lib supported exchange :param Text api_key: API key :param Text secret: API secret :param U[Bool, Text] user_agent: if True, exchange API keys will be load from "$HOME/.env" file. :param Bool clear_cache: if True, current cache will be ignored and overwrite. """ # for cache purposes self._basemarkets = None self._currencies = None self._symbols = None self._markets = None if clear_cache: self._cache.clear() exchange = _get_version(exchange) # assert exchange is not None, f'"{exchange}" not supported' api = getattr(ccxt, exchange) settings = dict(timeout=25000, enableRateLimit=True, fetchTickersErrors=False) if user_agent or False: if isinstance(user_agent, bool): settings.update(userAgent=api.userAgents[1]) else: settings.update(userAgent=str(user_agent)) if not all((api_key, secret)): load_dotenv() field_name = exchange.upper().strip(' _012345') key_field, secret_field = f'{field_name}_KEY', f'{field_name}_SECRET' api_key = os.environ.get(key_field) secret = os.environ.get(secret_field) if api_key and secret: settings.update(apiKey=api_key, secret=secret) if exchange == 'binance': # noinspection PyUnresolvedReferences settings.update(adjustForTimeDifference=True, parseOrderToPrecision=True) self._api = api(settings) def _load_markets(self) -> Markets: """Markets metadata cache handler. >>> isinstance(PandaXT('binance')._load_markets(), Markets) True :return: markets metadata as "Markets" instance. """ if self._markets is not None: data = self._markets else: self._cache_dir = _DATA_DIR / 'pandaxt' / self.id self._cache_dir.mkdir(exist_ok=True, parents=True) self._cache = Cache(str(self._cache_dir)) data = self._cache.get('markets', dict()) if len(data) == 0: data = self._api.load_markets() data = {k: {x: y for x, y in v.items() if y} for k, v in data.items() if v} self._cache.set('markets', data, (60.0 ** 2.0) * 6.0) return Markets(**data) @property def id(self) -> Text: """Exchange unique reference (also know as ID). >>> PandaXT('binance').id 'binance' :return: exchange unique reference. """ return self._api.id @property def timeframes(self) -> List[Text]: """Return valid exchange timeframes as list. >>> '15m' in PandaXT('binance').timeframes True :return: valid exchange timeframes. """ items = self._api.timeframes.items() od = OrderedDict(sorted(items, key=lambda x: x[1])) return list(od.keys()) @property def delisted(self) -> Symbols: """Returns delisted symbols (active -> False) >>> binance = PandaXT('binance') >>> delisted = binance.delisted >>> isinstance(delisted, list) True :return: return delisted symbols """ return Symbols(*[m for m in self.markets if hasattr(m, 'active') and not m.active]) @property def name(self) -> Text: """Exchange long name. :return: exchange long name. """ return getattr(self._api, 'name') @property def symbols(self) -> Symbols: """Get all supported symbols by exchange as "Symbol" list.""" if self._symbols is None: self._symbols = sorted([m for m in self.markets if m not in self.delisted]) return Symbols(self._symbols) @property def base_markets(self) -> Currencies: """Get exchange base markets currencies as "Currency" list.""" if self._basemarkets is None: self._basemarkets = sorted(list({s.quote for s in self.symbols})) return Currencies(self._basemarkets) @property def currencies(self) -> Currencies: """Get supported currencies by exchange as "Currency" list. >>> exchange = PandaXT('binance') >>> len(exchange.currencies) > 0 True :return: all active currencies supported by exchange. """ # Initialize markets, symbols, currencies and basemarkets if self._currencies is None: self._currencies = [s.base for s in self.symbols] self._currencies = self._currencies + list(self.base_markets) self._currencies = sorted(list(set(self._currencies))) return Currencies(self._currencies) def get_market_precision(self, symbol, precision_type=None) -> Int: """Get precision set by exchange for a market. >>> PandaXT('binance').get_market_precision('MATIC/USDT') :param symbol: symbol of the market where precision will be return. :param precision_type: accepted types: "price", "amount", "cost", "base", "quote" (default "price") :return: precision length for the supplied market symbol. """ market: Market = self.markets.get(Symbol(symbol)) precision = market.precision return getattr(precision, precision or 'price') @property def markets(self) -> Markets: """Get all exchange markets metadata. >>> exchange = PandaXT('binance') >>> markets = exchange.markets >>> isinstance(markets, Markets) and len(markets) > 1 True :return Dict: all exchange markets metadata. """ if self._markets is None: self._markets = self._load_markets() return self._markets def parse_timeframe(self, timeframe) -> U[Int, NoReturn]: """Get amount of seconds for a supplied string formatted timeframe. :param timeframe: a supported and valid string formatted timeframe (example: "15m") :return: None if timeframe is not supported otherwise, amount of seconds for supplied timeframe. """ if hasattr(self._api, 'parse_timeframe'): return self._api.parse_timeframe(timeframe) def get_timeframe(self, timeframe) -> Text: """Timeframe sanitizer. >>> PandaXT('binance').get_timeframe(15) '15m' :param timeframe: timeframe to sanitize. :type timeframe: Text or Int :return str: sanitize timeframe. """ timeframe = str(timeframe) if timeframe.isdigit() or not timeframe[-1].isalpha(): timeframe = f'{timeframe}m' if timeframe not in self.timeframes: raise TimeframeError(timeframe, exchange=self.name) else: return timeframe # @check_symbol def get_ohlc(self, symbol, timeframe='15m', limit=25) -> pd.DataFrame: """Get OHLC data for specific symbol as pandas DataFrame type. :param Text symbol: symbol name use at ohlc data request. :param Text timeframe: an exchange supported timeframe. :param int limit: max rows limit. :return pd.DataFrame: data-frame with: open, high, low, close, volume, qvolume columns and 'date' as index. """ if Symbol(symbol) not in self.symbols: # print(symbol, symbol in self.symbols, len(self.symbols)) raise SymbolError(symbol, exchange=self.name) data = self._api.fetch_ohlcv(symbol, timeframe=timeframe, limit=limit) df = pd.DataFrame(data, columns=_OHLC_FIELDS) df['qvolume'] = df['volume'] * df['close'] df.index = pd.to_datetime(df.pop('date') // 1000, unit='s') df.name = 'date' return df def get_currency(self, currency) -> Currency: """Currency name sanitizer.""" if currency: if currency in self._api.commonCurrencies: currency = self._api.commonCurrencies.get(currency) if currency not in self.currencies: log.debug(f'Currency {str(currency)} is not supported by exchange.') return Currency(currency) def altname(self, name) -> U[Currency, Symbol]: """Retrieve alternative currency or symbol name used in a specific exchange. >>> PandaXT('binance').altname('YOYOW') 'YOYOW' >>> PandaXT('binance').altname('YOYOW/BTC') (Symbol:YOYOW/BTC) >>> PandaXT('binance').altname('YOYOW/BTC') (Symbol:YOYOW/BTC) :param name: currency or symbol name to check for alternative name. :type name: Text or Currency or Symbol :return: currency alternative name as Currency or Symbol instance. :rtype: Currency or Symbol """ _symbol = str(name).upper() if '/' in _symbol.strip('/'): # print(_symbol) base, quote = _symbol.split('/') assert quote in self.base_markets, f'{quote} is not a valid base market.' base = self.get_currency(base) s = self.get_currency(base) + Currency(quote) s.price2precision = functools.partial(self.price2precision, s) s.cost2precision = functools.partial(self.cost2precision, s) s.amount2precision = functools.partial(self.amount2precision, s) return s else: return self.get_currency(_symbol) def cost2precision(self, symbol, cost, to_str=True) -> U[Float, Text]: """Return cost rounded to symbol precision exchange specifications. :param symbol: a valid exchange symbol. :type symbol: Text or Symbol :param Float cost: cost to be rounded. :param Bool to_str: True to return result as str, otherwise result will be returned as float :return float: cost rounded to specific symbol exchange specifications. """ result = self._api.cost_to_precision(symbol, cost) return result if to_str else float(result) def amount2precision(self, symbol, amount, to_str=True) -> U[Float, Text]: """Return amount rounded to symbol precision exchange specifications. :param symbol: a valid exchange symbol. :type symbol: Text or Symbol :param Float amount: amount to be rounded. :param Bool to_str: True to return result as str, otherwise result will be returned as float :return: amount rounded to specific symbol exchange specifications. :rtype: float or Text """ result = self._api.amount_to_precision(symbol, amount) return result if to_str else float(result) def price2precision(self, symbol, price, to_str=True) -> U[Float, Text]: """Return price rounded to symbol precision exchange specifications. :param symbol: a valid exchange symbol. :type symbol: Text or Symbol :param price: price to be rounded. :param Bool to_str: True to return result as str, otherwise result will be returned as float :return float: price rounded to specific symbol exchange specifications. """ result = self._api.price_to_precision(symbol, price) return result if to_str else float(result) # m = self.markets.get(symbol) # type: Market # t = str if to_str else float # template = '{:.@f}'.replace('@', str(m.precision.price)) # return t(template.format(float(price))) # @check_symbol def get_orderbook(self, symbol, limit=5) -> pd.DataFrame: """Get order book data for a symbol. >>> book_entries = PandaXT('binance').get_orderbook('MDA/BTC', 5) >>> isinstance(book_entries['ask'].values[-1], float) True :param symbol: a valid exchange symbol. :type symbol: Text or Symbol :param int limit: limit entries returned to "limit" value. :return: DataFrame type with order book data for "symbol". :rtype: pd.DataFrame """ raw = self._api.fetch_order_book(symbol, limit=int(limit)) columns = ['ask', 'ask_vol', 'bid', 'bid_vol'] data = [ask + bid for ask, bid in zip(raw['asks'], raw['bids'])] df = pd.DataFrame(data, columns=columns) return df.round({'ask': 8, 'bid': 8, 'ask_vol': 0, 'bid_vol': 0}) def get_tickers(self, symbols, sorted_by=None) -> U[Ticker, Tickers]: """Get tickers dict with symbol name as keys for all symbols specified as positional args. >>> exchange = PandaXT('binance') >>> ticker = exchange.get_tickers('ADA/BTC', sorted_by='percentage') >>> isinstance(ticker, Ticker) True :param symbols: list of valid exchange symbols. :param sorted_by: a valid ticker field like percentage, last, ask, bid, quoteVolume, ... :return: Ticker or Tickers instance with returned data. :rtype: Ticker or Tickers """ result = list() if isinstance(symbols, (Symbols, List, Tuple, Set)) and len(symbols): symbols = [symbols] if isinstance(symbols, str) else list(symbols) for s in map(self.altname, symbols): if s not in self.markets: log.debug(f'Symbol {s or "NULL"} is not listed in {self.name} exchange.') continue try: raw = self._api.fetch_tickers(symbols) if len(raw) > 0: if str(sorted_by) in list(raw.values())[0].keys(): raw = OrderedDict(sorted(raw.items(), key=lambda k: k[1][sorted_by], reverse=True)) result = Tickers(**raw) result = result[symbols[0]] if len(symbols) == 1 else result except ccxt.ExchangeError as err: print(str(err)) return result def get_indicators(self, indicators, symbol, timeframe='15m', limit=25, **kwargs) -> Dict[Text, pd.Series]: """Get technical indicators value for a symbol. :param Dict indicators: indicators and their params as dict (params ara mandatory, there is no default values). :param Text symbol: a valid exchange symbol. :param Text timeframe: an exchange valid timeframe (default 15m). :param Int limit: a valid exchange limit for returned rows (check exchange official API) supplied as a param / value dict instance also. Example: "{'roc': {'period': 9}}" :param kwargs: if "ohlc" is set with OHLC data (DataFrame) it will be use for value calculations. :return: dict type with indicators name/value pairs. """ indicators = {k.lower(): v for k, v in indicators.items()} symbol = Symbol(symbol) return_value = OrderedDict.fromkeys(indicators.keys()) supported_ti = [_a for _a in dir(tulipy.lib) if _a[0].islower()] functions = OrderedDict({i: getattr(tulipy, i) for i in indicators if i in supported_ti}) data = kwargs.get('ohlc', self.get_ohlc(symbol, timeframe=timeframe, limit=limit)) for n, fn in functions.items(): inputs = ['close' if i in 'real' else i for i in fn.inputs] indicator_params = dict() if len(fn.options): options = [opt.replace(' ', '_') for opt in fn.options] indicator_params = indicators.get(n) indicator_params = {k: v for k, v in indicator_params.items() if k in options} try: raw = fn(*data[inputs].T.values, **indicator_params) di = data.index if n in 'roc': raw = raw * 100.0 sr = pd.Series(raw, name=n.upper()) sr.index = di.values[-len(sr):] return_value[n] = sr.copy(True) except InvalidOptionError as err: print(str(err)) return dict(ohlc=data, **{k: return_value[k.lower()] for k in indicators}) def create_market_order(self, symbol, side, amount=None) -> Dict: """Create a market order order. :param Text symbol: a valid exchange symbol. :param Text side: accepted values: "buy", "sell" :param float amount: amount used in order creation. :return: order creation result data as dict. """ return self.create_order(symbol=symbol, side=side, order_type='market', amount=amount) def buy(self, symbol, amount=None, price=None) -> Dict: """Create buy order. :param Text symbol: a valid exchange symbol. :param float amount: amount to buy or None to auto-fill :param float price: buy price or None to auto-fill :return Dict: order creation result data as dict. """ return self.create_order(symbol=symbol, order_type='limit', side='buy', amount=amount, price=price) def create_order(self, symbol, side, order_type=None, amount=None, price=None) -> Dict: """Create a new order. :param Text symbol: symbol to be use for order creation. :param Text side: order side: 'sell' or 'buy' :param Text order_type: order type (default 'limit') :param float amount: amount used in order creation. :param float price: price used in order creation. :return: order result info as dict. """ symbol = Symbol(symbol) response = dict() if symbol not in self.symbols: raise SymbolError(f'Invalid symbol: {symbol}') if side not in ['buy', 'sell']: raise SideError(side) currency = symbol.quote if side in 'buy' else symbol.base balance_field = 'free' if side in 'buy' else 'total' ticker_field = 'ask' if side in 'buy' else 'bid' amount = magic2num( amount or self.get_balances(balance_field).get( currency, 0.0)) if amount > 0.0: price = magic2num(price or self.get_tickers(symbol).get(ticker_field)) if side in 'buy': amount = amount / price try: response = self._api.create_order(symbol=symbol, type=order_type or 'limit', side=side, amount=amount, price=price) except ccxt.InvalidOrder as err: print(f' - [ERROR] {str(err)}', file=sys.stderr) response = dict() return response def sell(self, symbol, amount=None, price=None) -> Dict: """Create sell order. :param Text symbol: a valid exchange symbol. :param Float amount: amount to sell or None to auto-fill :param Float price: sell price or None to auto-fill :return dict: order creation result data as dict. """ return self.create_order(symbol=symbol, order_type='limit', side='sell', amount=amount, price=price) def get_balances(self, field=None, tradeables_only=True, fiat=None) -> Wallet: """Get balances. >>> balances = PandaXT('binance').get_balances('total') >>> isinstance(balances, Wallet) True :param Text field: accepted values: if None all balances will be loaded, (values; "total", "used", "free") :param Bool tradeables_only: :return: positive balances. """ def is_zero(v) -> U[Float, Dict]: if isinstance(v, float): return v <= 0.0 elif isinstance(v, dict): return v.get('total', 0.0) <= 0.0 else: return 0.0 def is_tradeable(currency, _tickers, balance, base_market=None) -> Bool: """Check if a currency balance is over minimum tradeable amount for a base market. :param Text currency: :param Tickers _tickers: :param balance: :type balance: Dict or Float :param Text base_market: :return: True if currency is tradeable """ if currency in self.base_markets: return True base_market = Currency(base_market or 'BTC') symbol: Symbol = self.altname(currency) + base_market market: Market = self.markets.get(symbol, False) if not market: return False # min amount in quote currency limits: Limit = market.limits quote_min_amount = limits.amount ticker: Ticker = _tickers[symbol] if currency == 'BTC' and 'USD' in base_market: last = 1.0 / ticker.last else: last = ticker.last # converting min_amount to base currency base_min_amount = last * quote_min_amount # subtracting a 0.01% fee base_min_amount = base_min_amount * 0.999 if isinstance(balance or [], dict): balance = balance.get('total', 0.0) else: balance = balance or 0.0 return balance > base_min_amount try: data = self._api.fetch_balance() if 'info' in data: del data['info'] data: dict = data.pop(field) if field else data data = {str(k): v for k, v in data.items() if not is_zero(v)} if tradeables_only: symbols = {Currency(c) + Currency('BTC') for c in data} tickers = self.get_tickers(symbols) data = {str(k): v for k, v in data.items() if k in self._basemarkets or is_tradeable(k, tickers, v)} wallet = Wallet(**data) if str(fiat or '').lower() in ['EUR', 'USD']: acum = 0 for k, v in Wallet(**data).items(): if fiat.lower() == 'eur': acum += v.to_eur elif fiat.lower() == 'usd': acum += v.to_usd wallet.update({fiat.upper(): acum}) return wallet except ccxt.NetworkError as err: print('PandaXT::get_ohlc', str(err)) def get_balance(self, currency, field=None) -> Balance: """Get balance for a currency. >>> PandaXT('binance').get_balance('STORJ', 'total') :param Text currency: a valid exchange currency. :param Text field: accepted values: total, used, free :return: currency with balance amount as float. """ currency = self.altname(str(currency)) if currency not in self.currencies: raise CurrencyError(currency) if field and field in ['total', 'free', 'used']: field = field else: field = None balance_data = self.get_balances(field=field) or Balance( **{'currency': currency, field: 0.0}) if currency not in balance_data: raise ccxt.InsufficientFunds() else: return balance_data[currency] def get_user_trades(self, symbol, side=None, limit=25) -> pd.DataFrame: """Get user trades filter by symbol. :param Text symbol: a valid exchange symbol :param Text side: "buy" or "sell" :param int limit: a valid limit for rows return (please, refer to official exchange API manual for details) :return pd.DataFrame: user trades as pandas DataFrame type. """ symbol = str(symbol).upper() if symbol not in (self.symbols, self.altname(symbol) or ''): raise SymbolError(symbol) else: symbol = Symbol( symbol) if symbol in self.symbols else self.altname(symbol) trades = self._api.fetch_my_trades(symbol, limit=limit) if trades: trades = [{k: v for k, v in t.items() if k not in 'info'} for t in trades] for idx, t in enumerate(trades.copy()): trades[idx].update(total_cost=trades[idx]['fee']['cost']) del trades[idx]['fee'] trades = pd.DataFrame(trades) trades['real_cost'] = trades['cost'] + \ (trades['cost'] * trades['price']) # TODO: not totally true so revise it trades['real_price'] = trades['price'] * 1.001 trades['real_amount'] = trades['real_cost'] * trades['price'] if str(side).lower() in ['buy', 'sell']: trades = trades.query(f'side == "{str(side).lower()}"') return trades.sort_index(ascending=False) def get_trades(self, symbol, side=None, limit=25) -> pd.DataFrame: """Get latest user trades for a symbol. :param Text symbol: a valid exchange symbol :param Text side: "buy" or "sell" :param int limit: a valid limit for rows return (please, refer to official exchange API manual for details) :return: latest trades for a symbol. """ symbol = self.altname(symbol) trades = self._api.fetch_trades(symbol, limit=limit) if trades: trades = [{k: v for k, v in t.items() if k not in 'info' and v} for t in trades] trades = pd.DataFrame(trades).set_index('timestamp') if str(side).lower() in ['buy', 'sell']: trades = trades.query(f'side == "{str(side).lower()}"') return trades.sort_index(ascending=False) def get_cost(self, symbol, **kwargs) -> Float: """FIXME do not work well sometimes giving a bad result by unknown reason. Get weighted average (from buy trades data) cost for a symbol. >>> api = PandaXT('binance') >>> symbol_cost = api.get_cost('AGI/BTC') >>> symbol_cost True :param U[Symbol,Text] symbol: a valid exchange symbol. :param kwargs: accepted keys are: balance (Balance) and trades (pd.DataFrame) :return: cost calculation result as float type. """ symbol = self.altname(symbol) if isinstance(symbol or 0, Currency): symbol = symbol + Currency('BTC') base, quote = symbol.parts # reuse provided balance and trades data (for rate limit save) if 'balance' in kwargs: cached = kwargs.get('balance') # type: Balance balance = cached[base] if isinstance(cached, Wallet) else cached else: balance = self.get_balance(base, field='total') # type: Balance total_balance = balance.total if balance and hasattr( balance, 'total') else balance if total_balance > 0.0: trades = kwargs.get('trades', []) if not trades: trades = self.get_user_trades(symbol, side='buy') elif not isinstance(trades, pd.DataFrame): trades = pd.DataFrame(trades) if trades['order'].isna().all(): trades['order'].update(trades['id']) # group-by operations per column columns_op = {'amount': 'mean', 'price': 'mean', 'cost': 'mean', 'timestamp': 'mean'} trades = trades.groupby('order').agg(columns_op).sort_index( ascending=False) # type: pd.DataFrame trades = trades[['price', 'amount']].reset_index(drop=True) for index, price, amount in trades.itertuples(): if round(total_balance - amount, 8) <= 0.0: if round(total_balance - amount, 8) != 0.0: prices = trades[:index + 1]['price'].values amounts = trades[:index + 1]['amount'].values amounts[-1] = total_balance else: prices, amounts = trades[:index + 1].T.values return round(pd.np.average(prices, weights=amounts), 10) else: total_balance -= amount def get_order_status(self, order_id, symbol=None) -> Text: """Get order status by order_id. :param Text order_id: a valid order id. :param Text symbol: a valid exchange market :return: order status as str. Possible values are: "closed", "canceled", "open" """ return self._api.fetch_order_status(order_id, symbol=symbol) def get_open_orders(self, symbol=None, order_id=None) -> List[Dict[Symbol, Dict]]: """Get open orders.md for a symbol. :param Text symbol: symbol used in opened orders.md server query. :param order_id: just return data for a specific order. :type order_id: Text or int :return List: list of open orders.md for specific symbol. """ raw = self._api.fetch_open_orders(symbol or order_id) if isinstance(raw or 0, list) and len([r for r in raw if r]): return [{Symbol(k): v for k, v in r.items() if k not in 'info'} for r in raw] else: return list() def get_profit(self, currency) -> U[Tuple, Float]: """Returns current profit for a currency and its weighted average buy cost. :param Text currency: a valid currency to use at profit and cost calc :return: current profit and weighted average buy cost as a tuple """ currency = Currency(str(currency)) btc_symbol = currency + Currency('BTC') # type: Symbol balance = self.get_balance(currency) if balance.used > 0.0: cost = self.get_cost(symbol=btc_symbol) return balance.to_btc - (cost * balance.total) else: return 0.0, 0.0 def get_withdrawals(self): """Get withdrawals info. :return: """ if self._api.has['fetchWithdrawals']: result = self._api.fetch_withdrawals() return [{k: v for k, v in r.items() if k != 'info'} for r in result] else: raise ccxt.NotSupported('Withdrawals not supported by this exchange.') def get_transactions(self): """Get transactions info. :return: """ if self._api.has['fetchTransactions']: result = self._api.fetch_transactions() return [{k: v for k, v in r.items() if k != 'info'} for r in result] else: raise ccxt.NotSupported('Transactions not supported by this exchange.') def get_deposits(self): """Get deposits info. :return: """ if self._api.has['fetchDeposits']: resultb = self._api.fetch_withdrawals() return [{k: v for k, v in r.items() if k != 'info'} for r in result] else: raise ccxt.NotSupported('Deposits not supported by this exchange.') def cancel_order(self, symbol, last_only=False) -> List: """Cancel symbols open orders.md for a symbol. :param symbol: the symbol with open orders.md. :type symbol: Bool or Symbol :param Bool last_only: if True, only last order sent will be cancelled. :return: list of dict with data about cancellations. """ symbol = Symbol(symbol) pending_orders = self.get_open_orders(symbol) if len(pending_orders): if last_only: return self._api.cancel_order(pending_orders[-1]['id'], symbol) else: canceled_orders = list() for p in pending_orders: return_value = self._api.cancel_order(p['id'], symbol) if return_value and return_value.get( 'status', '') in 'cancel': canceled_orders.append( {k: v for k, v in return_value.items() if v}) else: self._api.cancel_order(p['id'], symbol) return canceled_orders def __str__(self) -> Text: """PandaXT instance as "str" type representation. >>> str(PandaXT('binance')) 'binance' :return Text: PandaXT instance as "str" type representation. """ return self.id def __repr__(self): """PandaXT instance as "str" type representation. >>> PandaXT('binance') binance :return str: PandaXT instance as "str" type representation. """ return self.id def __contains__(self, item) -> Bool: """Check if a symbol or currency is supported by exchange. >>> exchange = PandaXT('binance') >>> Currency('ETH') in exchange True >>> Currency('ETH/BTC') in exchange True >>> Currency('MyCOIN') in exchange False :param item: currency or symbol for supports checking on exchange. :type item: Text or Currency or Symbol :return bool: True is item is supported, otherwise False. """ return str(item) in self.markets or str(item) in map(str, self.currencies)
class State(StateBase): # pylint: disable=too-many-instance-attributes def __init__(self, root_dir=None, tmp_dir=None, dvcignore=None): from diskcache import Cache super().__init__() self.tmp_dir = tmp_dir self.root_dir = root_dir self.dvcignore = dvcignore if not tmp_dir: return config = { "eviction_policy": "least-recently-used", "disk_pickle_protocol": 4, } self.links = Cache(directory=os.path.join(tmp_dir, "links"), **config) self.md5s = Cache(directory=os.path.join(tmp_dir, "md5s"), **config) def close(self): self.md5s.close() self.links.close() def save(self, path_info, fs, hash_info): """Save hash for the specified path info. Args: path_info (dict): path_info to save hash for. hash_info (HashInfo): hash to save. """ if not isinstance(fs, LocalFileSystem): return mtime, size = get_mtime_and_size(path_info, fs, self.dvcignore) inode = get_inode(path_info) logger.debug( "state save (%s, %s, %s) %s", inode, mtime, str(size), hash_info.value, ) self.md5s[inode] = (mtime, str(size), hash_info.value) def get(self, path_info, fs): """Gets the hash for the specified path info. Hash will be retrieved from the state database if available. Args: path_info (dict): path info to get the hash for. Returns: HashInfo or None: hash for the specified path info or None if it doesn't exist in the state database. """ from .objects.meta import Meta if not isinstance(fs, LocalFileSystem): return None, None try: mtime, size = get_mtime_and_size(path_info, fs, self.dvcignore) except FileNotFoundError: return None, None inode = get_inode(path_info) value = self.md5s.get(inode) if not value or value[0] != mtime or value[1] != str(size): return None, None return Meta(size=size), HashInfo("md5", value[2]) def save_link(self, path_info, fs): """Adds the specified path to the list of links created by dvc. This list is later used on `dvc checkout` to cleanup old links. Args: path_info (dict): path info to add to the list of links. """ if not isinstance(fs, LocalFileSystem): return try: mtime, _ = get_mtime_and_size(path_info, fs, self.dvcignore) except FileNotFoundError: return inode = get_inode(path_info) relative_path = relpath(path_info, self.root_dir) with self.links as ref: ref[relative_path] = (inode, mtime) def get_unused_links(self, used, fs): """Removes all saved links except the ones that are used. Args: used (list): list of used links that should not be removed. """ if not isinstance(fs, LocalFileSystem): return unused = [] with self.links as ref: for relative_path in ref: path = os.path.join(self.root_dir, relative_path) if path in used or not fs.exists(path): continue inode = get_inode(path) mtime, _ = get_mtime_and_size(path, fs, self.dvcignore) if ref[relative_path] == (inode, mtime): logger.debug("Removing '%s' as unused link.", path) unused.append(relative_path) return unused def remove_links(self, unused, fs): if not isinstance(fs, LocalFileSystem): return for path in unused: remove(os.path.join(self.root_dir, path)) with self.links as ref: for path in unused: del ref[path]
def __init__(self, cache_duration=3600, cache_root='/tmp/cortex/tor_project'): self.session = requests.Session() self.cache_duration = cache_duration if self.cache_duration > 0: self.cache = Cache(cache_root) self.url = 'http://torstatus.blutmagie.de/query_export.php/Tor_query_EXPORT.csv'