class URLhaus:
"""Simple client to query URLhaus by abuse.ch.
:param query: domain, url or hash.
:param cache_duration: Duration before refreshing the cache (in seconds).
Ignored if `cache_duration` is 0.
:param cache_root: Path where to store the cached file.
:type query: string
:type cache_duration: int
:type cache_root: str
"""
def __init__(self,
query,
cache_duration=300,
cache_root="/tmp/cortex/URLhaus"):
self.URL = "https://urlhaus.abuse.ch/browse.php"
self.query = query
self.cache = None
if cache_duration > 0:
self.cache = Cache(cache_root)
self.cache_duration = cache_duration
def _get_raw_data(self):
try:
return self.cache[self.query.encode('utf-8')]
except(AttributeError, TypeError):
return self.fetch()
except KeyError:
self.cache.set(
self.query.encode('utf-8'),
self.fetch(),
expire=self.cache_duration)
return self.cache[self.query.encode('utf-8')]
def search(self):
res = self._get_raw_data()
return self.parse(res)
def fetch(self):
payload = {"search": self.query}
return requests.get(self.URL, params=payload).text
def parse(self, doc):
results = []
soup = BeautifulSoup(doc, "html.parser")
table = soup.find("table", class_="table")
rows = table.find_all("tr")[1:]
for row in rows:
cols = row.find_all("td")
results.append({
"dateadded": cols[0].text,
"malware_url": cols[1].text,
"link": cols[1].find("a").attrs.get("href"),
"status": cols[2].text,
"tags": cols[3].text.split(),
"gsb": cols[4].text,
"reporter": cols[5].text
})
return results
def _set_cached(path, content):
# 1/ memory cache
JUMBO_FIELDS_MEMORY_CACHE[path] = content
# 2/ disk cache
if SIMPLEFLOW_ENABLE_DISK_CACHE:
try:
cache = Cache(constants.CACHE_DIR)
cache_key = "jumbo_fields/" + path.split("/")[-1]
logger.debug("diskcache: setting key={} on cache_dir={}".format(cache_key, constants.CACHE_DIR))
cache.set(cache_key, content, expire=3 * constants.HOUR)
except OperationalError:
logger.warning("diskcache: got an OperationalError on write, skipping cache write")
def worker(queue, eviction_policy):
timings = {'get': [], 'set': [], 'delete': []}
cache = Cache('tmp', eviction_policy=eviction_policy)
for index, (action, key, value) in enumerate(iter(queue.get, None)):
start = time.time()
if action == 'set':
cache.set(key, value, expire=EXPIRE)
elif action == 'get':
result = cache.get(key)
else:
assert action == 'delete'
cache.delete(key)
stop = time.time()
if action == 'get' and PROCESSES == 1 and THREADS == 1 and EXPIRE is None:
assert result == value
if index > WARMUP:
timings[action].append(stop - start)
queue.put(timings)
cache.close()
def __init__(self,
query,
cache_duration=300,
cache_root="/tmp/cortex/URLhaus"):
self.URL = "https://urlhaus.abuse.ch/browse.php"
self.query = query
self.cache = None
if cache_duration > 0:
self.cache = Cache(cache_root)
self.cache_duration = cache_duration
def __init__(self, ttl=86400, cache_duration=3600,
cache_root='/tmp/cortex/tor_project'):
self.session = requests.Session()
self.delta = None
self.cache = None
if ttl > 0:
self.delta = timedelta(seconds=ttl)
if cache_duration > 0:
self.cache = Cache(cache_root)
self.cache_duration = cache_duration
self.url = 'https://check.torproject.org/exit-addresses'
def key_in_cache(key: tuple[str, bool]) -> bool:
"""Checks if key exists in cache."""
with Cache(CACHE_FILE) as cache:
return key in cache
mongo = PyMongo(application)
mongo_server = "localhost"
mongo_port = "27017"
AUTH_KEY = "14314314314314314314314314314314" # 32 char's Authentication key
str = "mongodb://" + mongo_server + ":" + mongo_port
connection = MongoClient(str) # connection to DB
db = connection.DFS
servers = db.servers
SERVER_HOST = None
SERVER_PORT = None
cache = Cache('/tmp/mycachedir') # directory path is used to store cache
def decrypt_data(key, hashed_val):
decrypted_data = AES.new(key, AES.MODE_ECB).decrypt(base64.b64decode(hashed_val))
return decrypted_data
def server_inst():
with application.app_context():
return db.servers.find_one({"host": SERVER_HOST, "port": SERVER_PORT})
# to upload
@application.route('/upload_f', methods=['POST'])
#!/usr/bin/env python3 """ Clears cache from the default tmp directory """ from diskcache import Cache import tempfile cache = Cache(tempfile.gettempdir()) cache.clear()
class Webserver:
def __init__(
self,
host="localhost",
port=8080,
hostname='hostname',
workers=os.cpu_count() - 1,
max_connections=socket.SOMAXCONN,
max_cache_size=4e9,
proto='http',
):
self._max_connections = max_connections
self.host = host
self.port = port
self.address = f'{proto}://{self.host}:{self.port}'
self.hostname = hostname
self.max_workers = workers
self.routes = {}
self.vservers_routes = {} # роуты для вирт. сервера
self.regular_routes = {}
self._response = None
self._current_clients = []
self.data_end = b'\r\n\r\n'
self.connect_refresh_timeout = 10
self.client_request_timeout = 0.1
self._max_keep_alive = timedelta(seconds=10)
self.cache = Cache(size_limit=int(max_cache_size))
self.cache_expire = 100
self._running = True
def set_routes(self, routes):
self.routes = {**routes, **self.routes}
def get_routes(self):
return self.routes
def handle_file(self,
request,
filename,
root=os.getcwd(),
content_type=None):
path = os.path.join(root, filename)
# Пробуем взять отдаваемый стат. файл из cache
res = self.cache.get(path)
if res:
return res
if not os.path.exists(path):
self.cache.set(path,
Errors.NOT_FOUND_PAGE,
expire=self.cache_expire,
tag='data')
self.cache.cull()
return Errors.NOT_FOUND_PAGE
if content_type is None:
content_type, _ = mimetypes.guess_type(path)
logger.verbose('Content type for %s, was set automatically to %s',
path, content_type)
response = Response.response_file(request, path, content_type)
if response.status == 200:
# Если код 200 OK, то добавим response в cache
self.cache.set(path,
response,
expire=self.cache_expire,
tag='data')
self.cache.cull()
return response
def handle_dir(self, request, dirname=os.getcwd()):
path = os.path.abspath(dirname)
if not os.path.exists(path):
return Errors.NOT_FOUND_PAGE
response = Response.response_dir(request, path)
return response
def get_routes_for_virtual_server(self, request):
vserver_name = request.headers['Host']
routes = self.vservers_routes.get(vserver_name)
if routes:
logger.debug('Find routes for vservers %s', vserver_name)
return routes
return self.regular_routes
def find_handler(self, request: Request):
logger.debug('%s', self.regular_routes)
try:
response = Errors.NOT_FOUND_PAGE
for path_info, handler in self.get_routes_for_virtual_server(
request).items():
logger.debug('checking %s %s | %s', path_info.path,
path_info.method, handler)
path_info: PathInfo = path_info
reg = path_info.path
match = re.fullmatch(reg, request.url.path)
if match and request.method == path_info.method:
logger.debug('handler found %s for %s', handler,
request.url)
if len(match.groupdict().items()) == 0:
response = handler(request)
else:
response = handler(request, match.group(1))
break
logger.debug('handler returned %s', response)
return response
except Exception:
logger.exception("Internal Error")
return Errors.INTERNAL_SERVER_ERROR
def _read_data(self, client, address):
_keep_alive_deadline = datetime.now() + self._max_keep_alive
request = Request()
filled = False
while not filled:
try:
line = client.recv(64 * 1024)
if not line:
raise ReceiveDone('Client done')
split = Request.split_keep_sep(line,
bytes(os.linesep, 'utf-8'))
for s in split:
if request.dynamic_fill(s):
filled = True
except BlockingIOError:
# читали, а нам никто не написал, подождем и может напишут
time.sleep(0.5)
_current_time = datetime.now()
if _current_time > _keep_alive_deadline:
# ждали больше чем держим keep-alive соединение
raise ConnectionError('Client dead')
return request
def _start_session(self, client, address):
while True:
request = self._read_data(client, address)
request.print_headers()
request.print_body()
response = self.find_handler(request)
logger.access(method=request.method,
url=request.target,
protocol=request.version,
status=response.status)
Response.response(client, response)
# закончили обрабатывать запрос от клиента
# (получили, ответили, пришли сюда)
if request.headers.get('Connection') == 'keep-alive':
logger.verbose("Client %s asked for keep-alive connection",
address)
# если браузер послал keep-alive (что он почти всегда
# делает дабы избавиться от поспоянных созданий
# подключений и ускорить общение с сервером)
client.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
# и продолжаем крутиться в while True:
else:
# иначе выходим из while True: (клиент не хочет больше
# с нами общаться)
break
def handle(self, client, address):
init_thread_local(THREAD_LOCAL_DATA)
THREAD_LOCAL_DATA.client = address
with client:
try:
self._start_session(client, address)
except ConnectionError:
logger.verbose('Client was not alive for %s seconds',
self._max_keep_alive)
except OSError:
logger.exception('Client had an issue')
except ReceiveDone:
logger.verbose('Client received the content')
except Exception:
logger.exception('Client had an error')
Response.response(client, Errors.INTERNAL_SERVER_ERROR)
logger.verbose("Disconnected")
def run(self, stop: Event):
self._running = True
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.make_regular_routes()
with server_socket:
server_socket.bind((self.host, self.port))
server_socket.listen(self._max_connections)
server_socket.setblocking(False)
logger.verbose(f'Start server on {self.address}')
logger.debug('T %s', server_socket.gettimeout())
with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
logger.verbose('ThreadPoolExecutor started')
while not stop.is_set():
try:
self._check_is_running()
try:
client, address = server_socket.accept()
THREAD_LOCAL_DATA.client = address
logger.verbose(f'Got client')
future = executor.submit(self.handle, client,
address)
self._current_clients.append(future)
except (socket.timeout, BlockingIOError):
# никто не подключился, можно подождать
# logger.info(f'No client')
time.sleep(1)
pass
except KeyboardInterrupt:
logger.verbose('Server shutdown started')
break
logger.verbose(('ThreadPoolExecutor '
'waiting for alive connections'))
for future in self._current_clients:
result = future.result()
logger.verbose('ThreadPoolExecutor finished')
def _check_is_running(self):
if not self._running:
raise KeyboardInterrupt
def stop(self):
self._running = False
def route(self, path, method='GET', vserver='localhost'):
logger.debug('Adding %s [%s]', path, method)
if path in self.routes:
raise AssertionError("Such route already exists.")
def wrapper(handler):
path_info = PathInfo(path, method)
if vserver == 'localhost':
self.set_routes({path_info: handler})
if vserver in self.vservers_routes:
self.vservers_routes[vserver][path_info] = handler
logger.debug('Add route %s for vserver %s', path_info, vserver)
else:
self.vservers_routes[vserver] = {path_info: handler}
logger.debug('INIT route %s for vserver %s', path_info,
vserver)
return handler
return wrapper
def make_regular_routes(self):
logger.debug('Processing routes %s', self.routes)
for path_info, handler in self.routes.items():
reg = re.compile(path_info.path)
path_info.path = reg
self.regular_routes[path_info] = handler
from diskcache import Cache
with Cache('../db') as cache:
print('Heartrate: ', cache[b'heartrate'])
print('Speed: ', cache[b'speed'])
print('Cadence: ', cache[b'cadence'])
class TorBlutmagieClient:
"""Simple client to query torstatus.blutmagie.de for exit nodes.
The client will download http://torstatus.blutmagie.de/query_export.php/Tor_query_EXPORT.csv
and check if a specified IP address, FQDN or domain is present in it.
It will cache the response for `cache_duration` seconds to avoid
too much latency.
:param cache_duration: Duration before refreshing the cache (in seconds).
Ignored if `cache_duration` is 0.
:param cache_root: Path where to store the cached file
downloaded from torstatus.blutmagie.de
:type cache_duration: int
:type cache_root: str
"""
def __init__(self, cache_duration=3600, cache_root='/tmp/cortex/tor_project'):
self.session = requests.Session()
self.cache_duration = cache_duration
if self.cache_duration > 0:
self.cache = Cache(cache_root)
self.url = 'http://torstatus.blutmagie.de/query_export.php/Tor_query_EXPORT.csv'
__cache_key = __name__ + ':raw_data'
def _get_raw_data(self):
try:
return self.cache[self.__cache_key]
except (AttributeError, TypeError):
return self.session.get(self.url).text.encode('utf-8')
except KeyError:
self.cache.set(
self.__cache_key,
self.session.get(self.url).text.encode('utf-8'),
expire=self.cache_duration, read=True)
return self.cache[self.__cache_key]
def _get_data(self):
return csv.DictReader(
self._get_raw_data().decode('utf-8').splitlines(),
delimiter=',')
def _extract_fields(self, line):
return {
'hostname': line['Hostname'],
'name': line['Router Name'],
'country_code': line['Country Code'],
'ip': line['IP Address'],
'as_name': line['ASName'],
'as_number': line['ASNumber']
}
def _get_node_from_domain(self, domain):
results = []
for line in self._get_data():
if domain.lower() in line['Hostname'].lower():
results.append(self._extract_fields(line))
return results
def _get_node_from_fqdn(self, fqdn):
results = []
for line in self._get_data():
if fqdn.lower() == line['Hostname'].lower():
results.append(self._extract_fields(line))
break
return results
def _get_node_from_ip(self, ip):
results = []
for line in self._get_data():
if ip == line['IP Address']:
results.append(self._extract_fields(line))
break
return results
def search_tor_node(self, data_type, data):
"""Lookup an artifact to check if it is a known tor exit node.
:param data_type: The artifact type. Must be one of 'ip', 'fqdn'
or 'domain'
:param data: The artifact to lookup
:type data_type: str
:type data: str
:return: Data relative to the tor node. If the looked-up artifact is
related to a tor exit node it will contain a `nodes` array.
That array will contains a list of nodes containing the
following keys:
- name: name given to the router
- ip: their IP address
- hostname: Hostname of the router
- country_code: ISO2 code of the country hosting the router
- as_name: ASName registering the router
- as_number: ASNumber registering the router
Otherwise, `nodes` will be empty.
:rtype: list
"""
results = []
if data_type == 'ip':
results = self._get_node_from_ip(data)
elif data_type == 'fqdn':
results = self._get_node_from_fqdn(data)
elif data_type == 'domain':
results = self._get_node_from_domain(data)
else:
pass
return {"nodes": results}
#!/usr/bin/env python
import requests
import re
import sys
import dlrnapi_client
import influxdb_utils
from promoter_utils import get_dlrn_instance_for_release
from diskcache import Cache
cache = Cache('/tmp/skipped_promotions_cache')
cache.expire()
promoter_skipping_regex = re.compile(
('.*promoter Skipping promotion of (.*) from (.*) to (.*), '
'missing successful jobs: (.*)'))
html_link = "<a href='{}' target='_blank' >{}</a>"
def get_failing_jobs_html(dlrn_hashes, release_name):
failing_jobs_html = ""
# If any of the jobs is still in progress
in_progress = False
try:
dlrn = get_dlrn_instance_for_release(release_name)
if dlrn:
params = dlrnapi_client.Params2()
params.commit_hash = dlrn_hashes['commit_hash']
params.distro_hash = dlrn_hashes['distro_hash']
import time
import tempfile
import asyncio
import re
import os
import sys
# local import
from getmyancestors import Session, Tree, Indi, Fam
from mergemyancestors import Gedcom
from translation import translations
tmp_dir = os.path.join(tempfile.gettempdir(), 'fstogedcom')
global cache
cache = Cache(tmp_dir)
lang = cache.get('lang')
def _(string):
if string in translations and lang in translations[string]:
return translations[string][lang]
return string
# Entry widget with right-clic menu to copy/cut/paste
class EntryWithMenu(Entry):
def __init__(self, master, **kw):
super(EntryWithMenu, self).__init__(master, **kw)
self.bind('<Button-3>', self.click_right)
from diskcache import Cache
task_queue = Cache('/tmp/ad-poster')
# print dir(task_queue)
print "Count:", task_queue.count
print "Cleared:", task_queue.clear()
class TorProjectClient:
"""Simple client to query torproject.org for exit nodes.
The client will download https://check.torproject.org/exit-addresses
and check if a specified IP address is present in it. If that IP address
is found it will check for its last update time and return a description
of the node if its last update time is less than `ttl` seconds ago.
:param ttl: Tor node will be kept only if its last update was
less than `ttl` seconds ago. Ignored if `ttl` is 0
:param cache_duration: Duration before refreshing the cache (in seconds).
Ignored if `cache_duration` is 0.
:param cache_root: Path where to store the cached file
downloaded from torproject.org
:type ttl: int
:type cache_duration: int
:type cache_root: str
"""
def __init__(self, ttl=86400, cache_duration=3600,
cache_root='/tmp/cortex/tor_project'):
self.session = requests.Session()
self.delta = None
self.cache = None
if ttl > 0:
self.delta = timedelta(seconds=ttl)
if cache_duration > 0:
self.cache = Cache(cache_root)
self.cache_duration = cache_duration
self.url = 'https://check.torproject.org/exit-addresses'
__cache_key = __name__ + ':raw_data'
def _get_raw_data(self):
try:
return self.cache['raw_data']
except(AttributeError, TypeError):
return self.session.get(self.url).text
except KeyError:
self.cache.set(
'raw_data',
self.session.get(self.url).text,
expire=self.cache_duration)
return self.cache['raw_data']
def search_tor_node(self, ip):
"""Lookup an IP address to check if it is a known tor exit node.
:param ip: The IP address to lookup
:type ip: str
:return: Data relative to the tor node. If `ip`is a tor exit node
it will contain a `node` key with the hash of the node and
a `last_status` key with the last update time of the node.
If `ip` is not a tor exit node, the function will return an
empty dictionary.
:rtype: dict
"""
data = {}
tmp = {}
present = datetime.utcnow().replace(tzinfo=pytz.utc)
for line in self._get_raw_data().splitlines():
params = line.split(' ')
if params[0] == 'ExitNode':
tmp['node'] = params[1]
elif params[0] == 'ExitAddress':
tmp['last_status'] = params[2] + 'T' + params[3] + '+0000'
last_status = parse(tmp['last_status'])
if (self.delta is None or
(present - last_status) < self.delta):
data[params[1]] = tmp
tmp = {}
else:
pass
return data.get(ip, {})
def clear_cache() -> None:
"""Clear everything from cache."""
with Cache(CACHE_FILE) as cache:
cache.clear()
import random
import re
import ssl
import threading
import time
# Load configuration from settings.json
with open("settings.json") as file:
configuration = json.loads(file.read())
PING_SLEEP = 45
## Cache
GIBIBYTE = 2**30
MEBIBYTE = 2**20
cache = Cache("./cache",
size_limit=configuration["max_cache_size_in_mebibytes"] *
MEBIBYTE)
##
# Sanic configuration
##
LOGGING = log.LOGGING_CONFIG_DEFAULTS
LOGGING["handlers"]["file"] = {
'class': 'logging.FileHandler',
'formatter': 'generic',
'filename': "log/latest.log",
'mode': 'a'
}
LOGGING["handlers"]["error_file"] = {
'class': 'logging.FileHandler',
'formatter': 'access',
from diskcache import Cache from pathlib import Path from config import BASE_DIR queue_cache = Cache(Path.joinpath(BASE_DIR, '.cache/queue')) widgets_cache = Cache(Path.joinpath(BASE_DIR, '.cache/widgets')) api_cache = Cache(Path.joinpath(BASE_DIR, '.cache/api')) notice_cache = Cache(Path.joinpath(BASE_DIR, '.cache/notice'))
import user_preference.user_preference as up
from diskcache import Cache
app = Flask(__name__)
nav = Nav(app)
nav.register_element(
'my_navbar',
Navbar('thenav', View('Home', 'home'), View('Add', 'add'),
View('Remove', 'remove'), View('Return', 'ret'),
View('Take', 'take')))
serialcomm = serial.Serial('/dev/cu.usbmodem101', 9600)
database = []
angle_storage = Cache("angle_storage")
angle_key = "cur_angle"
servo_angle = angle_storage.get(angle_key)
if servo_angle == None:
servo_angle = 0
cur_type_of_clothes = ""
cur_color = ""
class_lookup = [
"Anorak", "Blazer", "Blouse", "Bomber", "Button-Down", "Caftan", "Capris",
"Cardigan", "Chinos", "Coat, Coverup", "Culottes", "Cutoffs", "Dress",
"Flannel", "Gauchos", "Halter", "Henley", "Hoodie", "Jacket", "Jeans",
"Jeggings", "Jersey", "Jodhurs", "Joggers", "Jumpsuit", "Kaftan", "Kimono",
"Leggings", "Onesie", "Parka", "Peacoat", "Poncho", "Robe", "Romper",
"Sarong", "Shorts", "Skirt", "Sweater", "Sweatpants", "Sweatshorts",
"Tank", "Tee", "Top", "Trunks", "Turtleneck"
]
from tkinter import Variable
import os
import appdirs
from diskcache import Cache
__temp_dir = appdirs.user_data_dir(appname='mrplayer_live', appauthor='gzlock')
print('cache dir', __temp_dir)
cache = Cache(__temp_dir)
log_file = os.path.join(__temp_dir, 'log.txt')
def tkVariable(var: Variable, key: str, default=None):
"""
将tkinter的var绑定到缓存
:param var:
:param key:
:param default:
:return:
"""
_cache = cache.get(key, default=None)
if _cache is None:
if default is not None:
var.set(default)
else:
var.set(_cache)
var.trace('w', lambda a, b, c: cache.set(key, var.get()))
from diskcache import Cache import pypi_client pypi_client.cache = Cache()
def persistent_caching_fn(fn,
name,
check_cache_exists=False,
cache_dir=None,
cache_dirs=None,
cache_allow_writes=True,
retries=2) -> Callable:
wait_time = 0.25
random_time = 0.25
try:
cache_dirs = get_global(
"cache_dirs") if cache_dirs is None else cache_dirs
except:
if cache_dirs is None:
cache_dir = get_global(
"cache_dir") if cache_dir is None else cache_dir
cache_dirs = [cache_dir]
try:
cache_allow_writes = get_global("cache_allow_writes")
except:
pass
try:
cache_stats = get_global("cache_stats")
except:
cache_stats = defaultdict(lambda: defaultdict(float))
set_global("cache_stats", cache_stats)
cache_stats["count_cache_dirs"] = len(set(cache_dirs))
cache_stats["cache_dirs"] = set(cache_dirs)
from diskcache import Cache
import joblib
if check_cache_exists:
assert os.path.exists(cache_dir) and os.path.isdir(cache_dir)
cache_file = os.path.join(cache_dir, "cache.db")
assert os.path.exists(cache_file) and os.path.isfile(cache_file)
else:
if os.path.exists(cache_dir):
assert os.path.isdir(cache_dir)
else:
os.mkdir(cache_dir)
args = dict(eviction_policy='none',
sqlite_cache_size=2**16,
sqlite_mmap_size=2**28,
disk_min_file_size=2**18)
caches = [Cache(cd, **args) for cd in cache_dirs]
shuffle(caches)
try:
import inspect
fnh = joblib.hashing.hash(name, 'sha1') + joblib.hashing.hash(
inspect.getsourcelines(fn)[0], 'sha1') + joblib.hashing.hash(
fn.__name__, 'sha1')
except Exception as e:
try:
fnh = joblib.hashing.hash(name, 'sha1') + joblib.hashing.hash(
fn.__name__, 'sha1')
except Exception as e:
fnh = joblib.hashing.hash(name, 'sha1')
def build_hash(*args, **kwargs):
hsh = fnh + joblib.hashing.hash(args, 'sha1')
if len(kwargs) > 0:
hsh = hsh + joblib.hashing.hash(kwargs, 'sha1')
return hsh
def read_hash(hsh):
ts = time.time()
kes = [0] * len(caches)
for retry in range(retries):
for cidx, cache in enumerate(caches):
if kes[cidx] == 1:
continue
try:
r = cache[hsh]
cache_stats[name]["hit"] += 1
te = time.time() - ts
cache_stats[name]["read_time"] = 0.9 * cache_stats[name][
"read_time"] + 0.1 * te
return r
except KeyError:
cache_stats[name]["key_error"] += 1
kes[cidx] = 1
except Exception as e:
cache_stats[name]["read_exception"] += 1
cache_stats[name]["read_retries"] += 1
sleep(wait_time * (retry + 1) + random() * random_time)
sleep(wait_time * (retry + 1) + random() * random_time)
cache_stats[name]["read-return-none"] += 1
if sum(kes) >= 1:
return "ke", kes
return None
def write_hsh(hsh, r, kes):
ts = time.time()
if cache_allow_writes:
for retry in range(retries):
for cidx, cache in enumerate(caches):
if kes[cidx] != 1:
continue
try:
cache[hsh] = r
cache_stats[name]["writes"] += 1
te = time.time() - ts
cache_stats[name]["write_time"] = 0.9 * cache_stats[
name]["write_time"] + 0.1 * te
return r
except:
cache_stats[name]["write_exception"] += 1
sleep(wait_time * (retry + 1) + random() * random_time)
cache_stats[name]["write_retries"] += 1
sleep(wait_time * (retry + 1) + random() * random_time)
def cfn(*args, **kwargs):
ignore_cache = kwargs.pop("ignore_cache", False)
if ignore_cache:
r = fn(*args, **kwargs)
return r
hsh = build_hash(*args, **kwargs)
cache_stats[name]["called"] += 1
r = read_hash(hsh)
if r is not None:
if not isinstance(r, tuple) or (isinstance(r, tuple)
and r[0] != "ke"):
return r
if r is None:
r = fn(*args, **kwargs)
cache_stats[name]["re-compute"] += 1
cache_stats[name]["re-compute-cache-busy-no-write"] += 1
return r
kes = r[1]
r = fn(
*args, **kwargs
) # r is not None and there was key-error so we need to calculate the key and put in cache
cache_stats[name]["compute"] += 1
write_hsh(hsh, r, kes)
return r
return cfn
from ontobio.util.scigraph_util import get_nodes_from_ids, get_id_type_map, get_taxon
from typing import List, Optional, Dict, Tuple, Union, FrozenSet
from json.decoder import JSONDecodeError
from diskcache import Cache
import tempfile
import logging
import requests
"""
Functions that directly access the owlsim rest API were kept
outside the class to utilize caching
TODO: test if this is still the case with diskcache
"""
TIMEOUT = get_config().owlsim2.timeout
cache = Cache(tempfile.gettempdir())
@cache.memoize()
def search_by_attribute_set(url: str,
profile: FrozenSet[str],
limit: Optional[int] = 100,
namespace_filter: Optional[str] = None) -> Dict:
"""
Given a list of phenotypes, returns a ranked list of individuals
individuals can be filtered by namespace, eg MONDO, MGI, HGNC
:returns Dict with the structure: {
'unresolved' : [...]
'query_IRIs' : [...]
'results': {...}
}
import re
import unicodedata
import urllib.request
import ssl
from diskcache import Cache
c = Cache(directory='/tmp')
class TtsScraper(object):
ssl._create_default_https_context = ssl._create_unverified_context
MANUAL_ADDITIONS = [
'arko',
'Siliski Soaps Unscented',
'Zingari The Duo',
'Zingari The Watchman',
]
@classmethod
@c.memoize()
def get_tts_soaps(cls):
def _fixup_soap_name(name):
replacements = [
('-', ' '),
('|', ''),
('!', ''),
(''', "'"),
('"', '"'),
('&', '&'),
return any(c in '1234567890,.-' for c in lval)
if lkey == 'crs' and lval[:5] == 'epsg:' and lval[5:].isnumeric():
return True
if lkey in ['height', 'width'] and lval.isnumeric():
return True
return False
'''
' INITIALISATION - Executed upon startup only.
' Loads all the model parameters and WFS services from cache or creates them
'''
PARAM_CACHE_KEY = 'model_parameters'
WFS_CACHE_KEY = 'wfs_dict'
try:
with Cache(CACHE_DIR) as cache:
G_PARAM_DICT = cache.get(PARAM_CACHE_KEY)
G_WFS_DICT = cache.get(WFS_CACHE_KEY)
if G_PARAM_DICT is None or G_WFS_DICT is None:
G_PARAM_DICT, G_WFS_DICT = get_cached_parameters()
cache.add(PARAM_CACHE_KEY, G_PARAM_DICT)
cache.add(WFS_CACHE_KEY, G_WFS_DICT)
except OSError as os_exc:
LOGGER.error("Cannot fetch parameters & wfs from cache: %s", str(os_exc))
def application(environ, start_response):
'''
MAIN - This is called whenever an HTTP request arrives
'''
doc_root = os.path.normcase(environ['DOCUMENT_ROOT'])
import bonobo
import pickle
import hashlib
from diskcache import Cache
from time import sleep
from tqdm import tqdm
def hash(data):
m = hashlib.sha256()
m.update(pickle.dumps(data))
return m.hexdigest()
cache = Cache("pipeline")
def cached(func):
def wrapper_cached(*args, **kwargs):
key = hash(hash(func.__name__) + hash(args) + hash(kwargs))
if key in cache:
return cache[key]
else:
value = func(*args, **kwargs)
cache[key] = value
return value
return wrapper_cached
def generate_data():
import os
CACHE_REQUESTS = "CACHE_REQUESTS" in os.environ
print('CACHE_REQUESTS', CACHE_REQUESTS)
cache = None
if CACHE_REQUESTS:
from diskcache import Cache
cache = Cache(directory='/tmp/.cache')
CACHE_VALIDITY = 3600 * 6 # 6 hours
import requests
import urllib
def FetchPage(source):
if CACHE_REQUESTS:
with Cache(cache.directory) as store:
html = store.get(source)
if not html:
f = urllib.request.urlopen(source)
html = f.read()
store.set(source, html, expire=CACHE_VALIDITY)
return html
else:
f = urllib.request.urlopen(source)
return f.read()
import pickle
from concurrent.futures import ThreadPoolExecutor
from pprint import pprint
import threading
import time
import requests
from cytoolz import mapcat
from cytoolz import merge
from cytoolz import partial
from cytoolz import partition_all
from cytoolz import sliding_window
from diskcache import Cache
from retrying import retry
cache = Cache("edsm-cache")
def batched(num):
"""Run a function over its list argument in batches of size `num`."""
def _batched(func):
def _wrapped(total):
batches = partition_all(num, total)
combined = mapcat(func, batches)
return list(combined)
return _wrapped
return _batched
def add_to_cache(key: tuple[str, bool], value: str) -> None:
"""Add key, value pair to cache."""
with Cache(CACHE_FILE) as cache:
cache.add(key, value)
def __init__(self):
self.cache = Cache(
directory=os.path.dirname(os.path.abspath(__file__))
+ "/../../assets/.cache/"
)
def get_cache_value(key: tuple[str, bool]) -> str:
"""Get cache value, given its key."""
with Cache(CACHE_FILE) as cache:
return cache.get(key)
import re
import subprocess
import sys
import threading
import time
import webbrowser # to open link on browser
from collections import namedtuple
from typing import Tuple
from urllib import request
import requests
from diskcache import Cache
import services as s
cache = Cache(os.path.join(s.SETTINGS_DIR, 'cache'))
if sys.platform == "win32":
import win32process
import psutil
import win32gui
elif sys.platform == "linux":
import dbus
elif sys.platform == "darwin":
import applescript
class Song:
name = ""
artist = ""
album = "UNKNOWN"
def get_cache_keys() -> list[Cache.iterkeys]:
"""Get every key from cache."""
with Cache(CACHE_FILE) as cache:
return list(cache.iterkeys())
class FileDirCache(MutableMapping):
def __init__(
self,
use_listings_cache=True,
listings_expiry_time=None,
listings_cache_location=None,
**kwargs,
):
"""
Parameters
----------
use_listings_cache: bool
If False, this cache never returns items, but always reports KeyError,
and setting items has no effect
listings_expiry_time: int or float (optional)
Time in seconds that a listing is considered valid. If None,
listings do not expire.
listings_cache_location: str (optional)
Directory path at which the listings cache file is stored. If None,
an autogenerated path at the user folder is created.
"""
import appdirs
from diskcache import Cache
listings_expiry_time = listings_expiry_time and float(listings_expiry_time)
if listings_cache_location:
listings_cache_location = Path(listings_cache_location) / str(listings_expiry_time)
listings_cache_location.mkdir(exist_ok=True, parents=True)
else:
listings_cache_location = Path(appdirs.user_cache_dir(appname="fsspec_dircache")) / str(
listings_expiry_time
)
try:
listings_cache_location.mkdir(exist_ok=True, parents=True)
except Exception:
logger.error(f"folder for dircache could not be created at {listings_cache_location}")
self.cache_location = listings_cache_location
self._cache = Cache(directory=listings_cache_location)
self.use_listings_cache = use_listings_cache
self.listings_expiry_time = listings_expiry_time
def __getitem__(self, item):
"""Draw item as fileobject from cache, retry if timeout occurs"""
return self._cache.get(key=item, read=True, retry=True)
def clear(self):
self._cache.clear()
def __len__(self):
return len(list(self._cache.iterkeys()))
def __contains__(self, item):
value = self._cache.get(item, retry=True) # None, if expired
if value:
return True
return False
def __setitem__(self, key, value):
if not self.use_listings_cache:
return
self._cache.set(key=key, value=value, expire=self.listings_expiry_time, retry=True)
def __delitem__(self, key):
del self._cache[key]
def __iter__(self):
return (k for k in self._cache.iterkeys() if k in self)
def __reduce__(self):
return (
FileDirCache,
(self.use_listings_cache, self.listings_expiry_time, self.cache_location),
)
application = Flask(__name__)
mongo = PyMongo(application)
mongo_server = "localhost"
mongo_port = "27017"
str = "mongodb://" + mongo_server + ":" + mongo_port
connection = MongoClient(str)
db = connection.filesystem
servers = db.filesystem_servers
server_transactions = transaction_sv()
AUTH_KEY = "qwerty123456789asdfghjkladitya12"
SERVER_HOST = None
SERVER_PORT = None
# Setting the cache location
cache = Cache('/mycachedir')
def file_upload(file, directory, headers):
print "\n Uploading file ...\n"
transaction_sv.uploadT(file, directory, headers)
def decrypt_data(key, hashed_val): #decrypting the key
decrypted_data = AES.new(key, AES.MODE_ECB).decrypt(
base64.b64decode(hashed_val))
return decrypted_data
def server_instance():
with application.app_context():
# memory speed benchmark
storage = dict()
start = time.time()
for i in range(N):
storage[i] = i
sum = 0
for i in range(N):
sum += storage[random.randrange(0, N)]
end = time.time()
mem_time = end - start
print("memory time:", mem_time, sum)
# diskcache speed benchmark
storage = Cache("test")
start = time.time()
for i in range(N):
storage[i] = i
sum = 0
for i in range(N):
sum += storage[random.randrange(0, N)]
end = time.time()
dc_time = end - start
print("diskcache time:", dc_time, sum)
def _get_cache():
from os.path import expanduser, join
home = expanduser("~")
cache = Cache(join(home, ".aws-qc-cahce"))
return cache
class PandaXT:
"""A "ccxt" exchanges wrapper class over Pandas lib."""
def __init__(self, exchange, api_key=None, secret=None, user_agent=None, clear_cache=False) -> NoReturn:
"""Class constructor.
>>> markets = PandaXT('binance').markets
>>> isinstance(markets, Markets)
True
>>> market = markets.get('BTC/USDT') # type: Market
>>> isinstance(market, Market)
True
>>> isinstance(market.precision.price, int)
True
:param Text exchange: a ccxt lib supported exchange
:param Text api_key: API key
:param Text secret: API secret
:param U[Bool, Text] user_agent: if True, exchange API keys will be load from "$HOME/.env" file.
:param Bool clear_cache: if True, current cache will be ignored and overwrite.
"""
# for cache purposes
self._basemarkets = None
self._currencies = None
self._symbols = None
self._markets = None
if clear_cache:
self._cache.clear()
exchange = _get_version(exchange)
# assert exchange is not None, f'"{exchange}" not supported'
api = getattr(ccxt, exchange)
settings = dict(timeout=25000, enableRateLimit=True, fetchTickersErrors=False)
if user_agent or False:
if isinstance(user_agent, bool):
settings.update(userAgent=api.userAgents[1])
else:
settings.update(userAgent=str(user_agent))
if not all((api_key, secret)):
load_dotenv()
field_name = exchange.upper().strip(' _012345')
key_field, secret_field = f'{field_name}_KEY', f'{field_name}_SECRET'
api_key = os.environ.get(key_field)
secret = os.environ.get(secret_field)
if api_key and secret:
settings.update(apiKey=api_key, secret=secret)
if exchange == 'binance':
# noinspection PyUnresolvedReferences
settings.update(adjustForTimeDifference=True, parseOrderToPrecision=True)
self._api = api(settings)
def _load_markets(self) -> Markets:
"""Markets metadata cache handler.
>>> isinstance(PandaXT('binance')._load_markets(), Markets)
True
:return: markets metadata as "Markets" instance.
"""
if self._markets is not None:
data = self._markets
else:
self._cache_dir = _DATA_DIR / 'pandaxt' / self.id
self._cache_dir.mkdir(exist_ok=True, parents=True)
self._cache = Cache(str(self._cache_dir))
data = self._cache.get('markets', dict())
if len(data) == 0:
data = self._api.load_markets()
data = {k: {x: y for x, y in v.items() if y}
for k, v in data.items()
if v}
self._cache.set('markets', data, (60.0 ** 2.0) * 6.0)
return Markets(**data)
@property
def id(self) -> Text:
"""Exchange unique reference (also know as ID).
>>> PandaXT('binance').id
'binance'
:return: exchange unique reference.
"""
return self._api.id
@property
def timeframes(self) -> List[Text]:
"""Return valid exchange timeframes as list.
>>> '15m' in PandaXT('binance').timeframes
True
:return: valid exchange timeframes.
"""
items = self._api.timeframes.items()
od = OrderedDict(sorted(items, key=lambda x: x[1]))
return list(od.keys())
@property
def delisted(self) -> Symbols:
"""Returns delisted symbols (active -> False)
>>> binance = PandaXT('binance')
>>> delisted = binance.delisted
>>> isinstance(delisted, list)
True
:return: return delisted symbols
"""
return Symbols(*[m
for m in self.markets
if hasattr(m, 'active') and not m.active])
@property
def name(self) -> Text:
"""Exchange long name.
:return: exchange long name.
"""
return getattr(self._api, 'name')
@property
def symbols(self) -> Symbols:
"""Get all supported symbols by exchange as "Symbol" list."""
if self._symbols is None:
self._symbols = sorted([m for m in self.markets if m not in self.delisted])
return Symbols(self._symbols)
@property
def base_markets(self) -> Currencies:
"""Get exchange base markets currencies as "Currency" list."""
if self._basemarkets is None:
self._basemarkets = sorted(list({s.quote for s in self.symbols}))
return Currencies(self._basemarkets)
@property
def currencies(self) -> Currencies:
"""Get supported currencies by exchange as "Currency" list.
>>> exchange = PandaXT('binance')
>>> len(exchange.currencies) > 0
True
:return: all active currencies supported by exchange.
"""
# Initialize markets, symbols, currencies and basemarkets
if self._currencies is None:
self._currencies = [s.base for s in self.symbols]
self._currencies = self._currencies + list(self.base_markets)
self._currencies = sorted(list(set(self._currencies)))
return Currencies(self._currencies)
def get_market_precision(self, symbol, precision_type=None) -> Int:
"""Get precision set by exchange for a market.
>>> PandaXT('binance').get_market_precision('MATIC/USDT')
:param symbol: symbol of the market where precision will be return.
:param precision_type: accepted types: "price", "amount", "cost", "base", "quote" (default "price")
:return: precision length for the supplied market symbol.
"""
market: Market = self.markets.get(Symbol(symbol))
precision = market.precision
return getattr(precision, precision or 'price')
@property
def markets(self) -> Markets:
"""Get all exchange markets metadata.
>>> exchange = PandaXT('binance')
>>> markets = exchange.markets
>>> isinstance(markets, Markets) and len(markets) > 1
True
:return Dict: all exchange markets metadata.
"""
if self._markets is None:
self._markets = self._load_markets()
return self._markets
def parse_timeframe(self, timeframe) -> U[Int, NoReturn]:
"""Get amount of seconds for a supplied string formatted timeframe.
:param timeframe: a supported and valid string formatted timeframe (example: "15m")
:return: None if timeframe is not supported otherwise, amount of seconds for supplied timeframe.
"""
if hasattr(self._api, 'parse_timeframe'):
return self._api.parse_timeframe(timeframe)
def get_timeframe(self, timeframe) -> Text:
"""Timeframe sanitizer.
>>> PandaXT('binance').get_timeframe(15)
'15m'
:param timeframe: timeframe to sanitize.
:type timeframe: Text or Int
:return str: sanitize timeframe.
"""
timeframe = str(timeframe)
if timeframe.isdigit() or not timeframe[-1].isalpha():
timeframe = f'{timeframe}m'
if timeframe not in self.timeframes:
raise TimeframeError(timeframe, exchange=self.name)
else:
return timeframe
# @check_symbol
def get_ohlc(self, symbol, timeframe='15m', limit=25) -> pd.DataFrame:
"""Get OHLC data for specific symbol as pandas DataFrame type.
:param Text symbol: symbol name use at ohlc data request.
:param Text timeframe: an exchange supported timeframe.
:param int limit: max rows limit.
:return pd.DataFrame: data-frame with: open, high, low, close, volume, qvolume columns and 'date' as index.
"""
if Symbol(symbol) not in self.symbols:
# print(symbol, symbol in self.symbols, len(self.symbols))
raise SymbolError(symbol, exchange=self.name)
data = self._api.fetch_ohlcv(symbol, timeframe=timeframe, limit=limit)
df = pd.DataFrame(data, columns=_OHLC_FIELDS)
df['qvolume'] = df['volume'] * df['close']
df.index = pd.to_datetime(df.pop('date') // 1000, unit='s')
df.name = 'date'
return df
def get_currency(self, currency) -> Currency:
"""Currency name sanitizer."""
if currency:
if currency in self._api.commonCurrencies:
currency = self._api.commonCurrencies.get(currency)
if currency not in self.currencies:
log.debug(f'Currency {str(currency)} is not supported by exchange.')
return Currency(currency)
def altname(self, name) -> U[Currency, Symbol]:
"""Retrieve alternative currency or symbol name used in a specific exchange.
>>> PandaXT('binance').altname('YOYOW')
'YOYOW'
>>> PandaXT('binance').altname('YOYOW/BTC')
(Symbol:YOYOW/BTC)
>>> PandaXT('binance').altname('YOYOW/BTC')
(Symbol:YOYOW/BTC)
:param name: currency or symbol name to check for alternative name.
:type name: Text or Currency or Symbol
:return: currency alternative name as Currency or Symbol instance.
:rtype: Currency or Symbol
"""
_symbol = str(name).upper()
if '/' in _symbol.strip('/'):
# print(_symbol)
base, quote = _symbol.split('/')
assert quote in self.base_markets, f'{quote} is not a valid base market.'
base = self.get_currency(base)
s = self.get_currency(base) + Currency(quote)
s.price2precision = functools.partial(self.price2precision, s)
s.cost2precision = functools.partial(self.cost2precision, s)
s.amount2precision = functools.partial(self.amount2precision, s)
return s
else:
return self.get_currency(_symbol)
def cost2precision(self, symbol, cost, to_str=True) -> U[Float, Text]:
"""Return cost rounded to symbol precision exchange specifications.
:param symbol: a valid exchange symbol.
:type symbol: Text or Symbol
:param Float cost: cost to be rounded.
:param Bool to_str: True to return result as str, otherwise result will be returned as float
:return float: cost rounded to specific symbol exchange specifications.
"""
result = self._api.cost_to_precision(symbol, cost)
return result if to_str else float(result)
def amount2precision(self, symbol, amount, to_str=True) -> U[Float, Text]:
"""Return amount rounded to symbol precision exchange specifications.
:param symbol: a valid exchange symbol.
:type symbol: Text or Symbol
:param Float amount: amount to be rounded.
:param Bool to_str: True to return result as str, otherwise result will be returned as float
:return: amount rounded to specific symbol exchange specifications.
:rtype: float or Text
"""
result = self._api.amount_to_precision(symbol, amount)
return result if to_str else float(result)
def price2precision(self, symbol, price, to_str=True) -> U[Float, Text]:
"""Return price rounded to symbol precision exchange specifications.
:param symbol: a valid exchange symbol.
:type symbol: Text or Symbol
:param price: price to be rounded.
:param Bool to_str: True to return result as str, otherwise result will be returned as float
:return float: price rounded to specific symbol exchange specifications.
"""
result = self._api.price_to_precision(symbol, price)
return result if to_str else float(result)
# m = self.markets.get(symbol) # type: Market
# t = str if to_str else float
# template = '{:.@f}'.replace('@', str(m.precision.price))
# return t(template.format(float(price)))
# @check_symbol
def get_orderbook(self, symbol, limit=5) -> pd.DataFrame:
"""Get order book data for a symbol.
>>> book_entries = PandaXT('binance').get_orderbook('MDA/BTC', 5)
>>> isinstance(book_entries['ask'].values[-1], float)
True
:param symbol: a valid exchange symbol.
:type symbol: Text or Symbol
:param int limit: limit entries returned to "limit" value.
:return: DataFrame type with order book data for "symbol".
:rtype: pd.DataFrame
"""
raw = self._api.fetch_order_book(symbol, limit=int(limit))
columns = ['ask', 'ask_vol', 'bid', 'bid_vol']
data = [ask + bid for ask, bid in zip(raw['asks'], raw['bids'])]
df = pd.DataFrame(data, columns=columns)
return df.round({'ask': 8, 'bid': 8, 'ask_vol': 0, 'bid_vol': 0})
def get_tickers(self, symbols, sorted_by=None) -> U[Ticker, Tickers]:
"""Get tickers dict with symbol name as keys for all symbols specified as positional args.
>>> exchange = PandaXT('binance')
>>> ticker = exchange.get_tickers('ADA/BTC', sorted_by='percentage')
>>> isinstance(ticker, Ticker)
True
:param symbols: list of valid exchange symbols.
:param sorted_by: a valid ticker field like percentage, last, ask, bid, quoteVolume, ...
:return: Ticker or Tickers instance with returned data.
:rtype: Ticker or Tickers
"""
result = list()
if isinstance(symbols, (Symbols, List, Tuple, Set)) and len(symbols):
symbols = [symbols] if isinstance(symbols, str) else list(symbols)
for s in map(self.altname, symbols):
if s not in self.markets:
log.debug(f'Symbol {s or "NULL"} is not listed in {self.name} exchange.')
continue
try:
raw = self._api.fetch_tickers(symbols)
if len(raw) > 0:
if str(sorted_by) in list(raw.values())[0].keys():
raw = OrderedDict(sorted(raw.items(), key=lambda k: k[1][sorted_by], reverse=True))
result = Tickers(**raw)
result = result[symbols[0]] if len(symbols) == 1 else result
except ccxt.ExchangeError as err:
print(str(err))
return result
def get_indicators(self, indicators, symbol, timeframe='15m', limit=25, **kwargs) -> Dict[Text, pd.Series]:
"""Get technical indicators value for a symbol.
:param Dict indicators: indicators and their params as dict (params ara mandatory, there is no default values).
:param Text symbol: a valid exchange symbol.
:param Text timeframe: an exchange valid timeframe (default 15m).
:param Int limit: a valid exchange limit for returned rows (check exchange official API)
supplied as a param / value dict instance also. Example: "{'roc': {'period': 9}}"
:param kwargs: if "ohlc" is set with OHLC data (DataFrame) it will be use for value calculations.
:return: dict type with indicators name/value pairs.
"""
indicators = {k.lower(): v for k, v in indicators.items()}
symbol = Symbol(symbol)
return_value = OrderedDict.fromkeys(indicators.keys())
supported_ti = [_a for _a in dir(tulipy.lib) if _a[0].islower()]
functions = OrderedDict({i: getattr(tulipy, i)
for i in indicators if i in supported_ti})
data = kwargs.get('ohlc', self.get_ohlc(symbol, timeframe=timeframe, limit=limit))
for n, fn in functions.items():
inputs = ['close' if i in 'real' else i for i in fn.inputs]
indicator_params = dict()
if len(fn.options):
options = [opt.replace(' ', '_') for opt in fn.options]
indicator_params = indicators.get(n)
indicator_params = {k: v
for k, v in indicator_params.items()
if k in options}
try:
raw = fn(*data[inputs].T.values, **indicator_params)
di = data.index
if n in 'roc':
raw = raw * 100.0
sr = pd.Series(raw, name=n.upper())
sr.index = di.values[-len(sr):]
return_value[n] = sr.copy(True)
except InvalidOptionError as err:
print(str(err))
return dict(ohlc=data, **{k: return_value[k.lower()] for k in indicators})
def create_market_order(self, symbol, side, amount=None) -> Dict:
"""Create a market order order.
:param Text symbol: a valid exchange symbol.
:param Text side: accepted values: "buy", "sell"
:param float amount: amount used in order creation.
:return: order creation result data as dict.
"""
return self.create_order(symbol=symbol, side=side, order_type='market', amount=amount)
def buy(self, symbol, amount=None, price=None) -> Dict:
"""Create buy order.
:param Text symbol: a valid exchange symbol.
:param float amount: amount to buy or None to auto-fill
:param float price: buy price or None to auto-fill
:return Dict: order creation result data as dict.
"""
return self.create_order(symbol=symbol, order_type='limit', side='buy', amount=amount, price=price)
def create_order(self, symbol, side, order_type=None, amount=None, price=None) -> Dict:
"""Create a new order.
:param Text symbol: symbol to be use for order creation.
:param Text side: order side: 'sell' or 'buy'
:param Text order_type: order type (default 'limit')
:param float amount: amount used in order creation.
:param float price: price used in order creation.
:return: order result info as dict.
"""
symbol = Symbol(symbol)
response = dict()
if symbol not in self.symbols:
raise SymbolError(f'Invalid symbol: {symbol}')
if side not in ['buy', 'sell']:
raise SideError(side)
currency = symbol.quote if side in 'buy' else symbol.base
balance_field = 'free' if side in 'buy' else 'total'
ticker_field = 'ask' if side in 'buy' else 'bid'
amount = magic2num(
amount or self.get_balances(balance_field).get(
currency, 0.0))
if amount > 0.0:
price = magic2num(price or self.get_tickers(symbol).get(ticker_field))
if side in 'buy':
amount = amount / price
try:
response = self._api.create_order(symbol=symbol,
type=order_type or 'limit',
side=side,
amount=amount,
price=price)
except ccxt.InvalidOrder as err:
print(f' - [ERROR] {str(err)}', file=sys.stderr)
response = dict()
return response
def sell(self, symbol, amount=None, price=None) -> Dict:
"""Create sell order.
:param Text symbol: a valid exchange symbol.
:param Float amount: amount to sell or None to auto-fill
:param Float price: sell price or None to auto-fill
:return dict: order creation result data as dict.
"""
return self.create_order(symbol=symbol, order_type='limit', side='sell', amount=amount, price=price)
def get_balances(self, field=None, tradeables_only=True, fiat=None) -> Wallet:
"""Get balances.
>>> balances = PandaXT('binance').get_balances('total')
>>> isinstance(balances, Wallet)
True
:param Text field: accepted values: if None all balances will be loaded, (values; "total", "used", "free")
:param Bool tradeables_only:
:return: positive balances.
"""
def is_zero(v) -> U[Float, Dict]:
if isinstance(v, float):
return v <= 0.0
elif isinstance(v, dict):
return v.get('total', 0.0) <= 0.0
else:
return 0.0
def is_tradeable(currency, _tickers, balance, base_market=None) -> Bool:
"""Check if a currency balance is over minimum tradeable amount for a base market.
:param Text currency:
:param Tickers _tickers:
:param balance:
:type balance: Dict or Float
:param Text base_market:
:return: True if currency is tradeable
"""
if currency in self.base_markets:
return True
base_market = Currency(base_market or 'BTC')
symbol: Symbol = self.altname(currency) + base_market
market: Market = self.markets.get(symbol, False)
if not market:
return False
# min amount in quote currency
limits: Limit = market.limits
quote_min_amount = limits.amount
ticker: Ticker = _tickers[symbol]
if currency == 'BTC' and 'USD' in base_market:
last = 1.0 / ticker.last
else:
last = ticker.last
# converting min_amount to base currency
base_min_amount = last * quote_min_amount
# subtracting a 0.01% fee
base_min_amount = base_min_amount * 0.999
if isinstance(balance or [], dict):
balance = balance.get('total', 0.0)
else:
balance = balance or 0.0
return balance > base_min_amount
try:
data = self._api.fetch_balance()
if 'info' in data:
del data['info']
data: dict = data.pop(field) if field else data
data = {str(k): v
for k, v in data.items()
if not is_zero(v)}
if tradeables_only:
symbols = {Currency(c) + Currency('BTC') for c in data}
tickers = self.get_tickers(symbols)
data = {str(k): v
for k, v in data.items()
if k in self._basemarkets or is_tradeable(k, tickers, v)}
wallet = Wallet(**data)
if str(fiat or '').lower() in ['EUR', 'USD']:
acum = 0
for k, v in Wallet(**data).items():
if fiat.lower() == 'eur':
acum += v.to_eur
elif fiat.lower() == 'usd':
acum += v.to_usd
wallet.update({fiat.upper(): acum})
return wallet
except ccxt.NetworkError as err:
print('PandaXT::get_ohlc', str(err))
def get_balance(self, currency, field=None) -> Balance:
"""Get balance for a currency.
>>> PandaXT('binance').get_balance('STORJ', 'total')
:param Text currency: a valid exchange currency.
:param Text field: accepted values: total, used, free
:return: currency with balance amount as float.
"""
currency = self.altname(str(currency))
if currency not in self.currencies:
raise CurrencyError(currency)
if field and field in ['total', 'free', 'used']:
field = field
else:
field = None
balance_data = self.get_balances(field=field) or Balance(
**{'currency': currency, field: 0.0})
if currency not in balance_data:
raise ccxt.InsufficientFunds()
else:
return balance_data[currency]
def get_user_trades(self, symbol, side=None, limit=25) -> pd.DataFrame:
"""Get user trades filter by symbol.
:param Text symbol: a valid exchange symbol
:param Text side: "buy" or "sell"
:param int limit: a valid limit for rows return (please, refer to official exchange API manual for details)
:return pd.DataFrame: user trades as pandas DataFrame type.
"""
symbol = str(symbol).upper()
if symbol not in (self.symbols, self.altname(symbol) or ''):
raise SymbolError(symbol)
else:
symbol = Symbol(
symbol) if symbol in self.symbols else self.altname(symbol)
trades = self._api.fetch_my_trades(symbol, limit=limit)
if trades:
trades = [{k: v for k, v in t.items() if k not in 'info'}
for t in trades]
for idx, t in enumerate(trades.copy()):
trades[idx].update(total_cost=trades[idx]['fee']['cost'])
del trades[idx]['fee']
trades = pd.DataFrame(trades)
trades['real_cost'] = trades['cost'] + \
(trades['cost'] * trades['price'])
# TODO: not totally true so revise it
trades['real_price'] = trades['price'] * 1.001
trades['real_amount'] = trades['real_cost'] * trades['price']
if str(side).lower() in ['buy', 'sell']:
trades = trades.query(f'side == "{str(side).lower()}"')
return trades.sort_index(ascending=False)
def get_trades(self, symbol, side=None, limit=25) -> pd.DataFrame:
"""Get latest user trades for a symbol.
:param Text symbol: a valid exchange symbol
:param Text side: "buy" or "sell"
:param int limit: a valid limit for rows return (please, refer to official exchange API manual for details)
:return: latest trades for a symbol.
"""
symbol = self.altname(symbol)
trades = self._api.fetch_trades(symbol, limit=limit)
if trades:
trades = [{k: v for k, v in t.items() if k not in 'info' and v}
for t in trades]
trades = pd.DataFrame(trades).set_index('timestamp')
if str(side).lower() in ['buy', 'sell']:
trades = trades.query(f'side == "{str(side).lower()}"')
return trades.sort_index(ascending=False)
def get_cost(self, symbol, **kwargs) -> Float:
"""FIXME do not work well sometimes giving a bad result by unknown reason.
Get weighted average (from buy trades data) cost for a symbol.
>>> api = PandaXT('binance')
>>> symbol_cost = api.get_cost('AGI/BTC')
>>> symbol_cost
True
:param U[Symbol,Text] symbol: a valid exchange symbol.
:param kwargs: accepted keys are: balance (Balance) and trades (pd.DataFrame)
:return: cost calculation result as float type.
"""
symbol = self.altname(symbol)
if isinstance(symbol or 0, Currency):
symbol = symbol + Currency('BTC')
base, quote = symbol.parts
# reuse provided balance and trades data (for rate limit save)
if 'balance' in kwargs:
cached = kwargs.get('balance') # type: Balance
balance = cached[base] if isinstance(cached, Wallet) else cached
else:
balance = self.get_balance(base, field='total') # type: Balance
total_balance = balance.total if balance and hasattr(
balance, 'total') else balance
if total_balance > 0.0:
trades = kwargs.get('trades', [])
if not trades:
trades = self.get_user_trades(symbol, side='buy')
elif not isinstance(trades, pd.DataFrame):
trades = pd.DataFrame(trades)
if trades['order'].isna().all():
trades['order'].update(trades['id'])
# group-by operations per column
columns_op = {'amount': 'mean',
'price': 'mean',
'cost': 'mean',
'timestamp': 'mean'}
trades = trades.groupby('order').agg(columns_op).sort_index(
ascending=False) # type: pd.DataFrame
trades = trades[['price', 'amount']].reset_index(drop=True)
for index, price, amount in trades.itertuples():
if round(total_balance - amount, 8) <= 0.0:
if round(total_balance - amount, 8) != 0.0:
prices = trades[:index + 1]['price'].values
amounts = trades[:index + 1]['amount'].values
amounts[-1] = total_balance
else:
prices, amounts = trades[:index + 1].T.values
return round(pd.np.average(prices, weights=amounts), 10)
else:
total_balance -= amount
def get_order_status(self, order_id, symbol=None) -> Text:
"""Get order status by order_id.
:param Text order_id: a valid order id.
:param Text symbol: a valid exchange market
:return: order status as str. Possible values are: "closed", "canceled", "open"
"""
return self._api.fetch_order_status(order_id, symbol=symbol)
def get_open_orders(self, symbol=None, order_id=None) -> List[Dict[Symbol, Dict]]:
"""Get open orders.md for a symbol.
:param Text symbol: symbol used in opened orders.md server query.
:param order_id: just return data for a specific order.
:type order_id: Text or int
:return List: list of open orders.md for specific symbol.
"""
raw = self._api.fetch_open_orders(symbol or order_id)
if isinstance(raw or 0, list) and len([r for r in raw if r]):
return [{Symbol(k): v for k, v in r.items() if k not in 'info'} for r in raw]
else:
return list()
def get_profit(self, currency) -> U[Tuple, Float]:
"""Returns current profit for a currency and its weighted average buy cost.
:param Text currency: a valid currency to use at profit and cost calc
:return: current profit and weighted average buy cost as a tuple
"""
currency = Currency(str(currency))
btc_symbol = currency + Currency('BTC') # type: Symbol
balance = self.get_balance(currency)
if balance.used > 0.0:
cost = self.get_cost(symbol=btc_symbol)
return balance.to_btc - (cost * balance.total)
else:
return 0.0, 0.0
def get_withdrawals(self):
"""Get withdrawals info.
:return:
"""
if self._api.has['fetchWithdrawals']:
result = self._api.fetch_withdrawals()
return [{k: v for k, v in r.items() if k != 'info'} for r in result]
else:
raise ccxt.NotSupported('Withdrawals not supported by this exchange.')
def get_transactions(self):
"""Get transactions info.
:return:
"""
if self._api.has['fetchTransactions']:
result = self._api.fetch_transactions()
return [{k: v for k, v in r.items() if k != 'info'} for r in result]
else:
raise ccxt.NotSupported('Transactions not supported by this exchange.')
def get_deposits(self):
"""Get deposits info.
:return:
"""
if self._api.has['fetchDeposits']:
resultb = self._api.fetch_withdrawals()
return [{k: v for k, v in r.items() if k != 'info'} for r in result]
else:
raise ccxt.NotSupported('Deposits not supported by this exchange.')
def cancel_order(self, symbol, last_only=False) -> List:
"""Cancel symbols open orders.md for a symbol.
:param symbol: the symbol with open orders.md.
:type symbol: Bool or Symbol
:param Bool last_only: if True, only last order sent will be cancelled.
:return: list of dict with data about cancellations.
"""
symbol = Symbol(symbol)
pending_orders = self.get_open_orders(symbol)
if len(pending_orders):
if last_only:
return self._api.cancel_order(pending_orders[-1]['id'], symbol)
else:
canceled_orders = list()
for p in pending_orders:
return_value = self._api.cancel_order(p['id'], symbol)
if return_value and return_value.get(
'status', '') in 'cancel':
canceled_orders.append(
{k: v for k, v in return_value.items() if v})
else:
self._api.cancel_order(p['id'], symbol)
return canceled_orders
def __str__(self) -> Text:
"""PandaXT instance as "str" type representation.
>>> str(PandaXT('binance'))
'binance'
:return Text: PandaXT instance as "str" type representation.
"""
return self.id
def __repr__(self):
"""PandaXT instance as "str" type representation.
>>> PandaXT('binance')
binance
:return str: PandaXT instance as "str" type representation.
"""
return self.id
def __contains__(self, item) -> Bool:
"""Check if a symbol or currency is supported by exchange.
>>> exchange = PandaXT('binance')
>>> Currency('ETH') in exchange
True
>>> Currency('ETH/BTC') in exchange
True
>>> Currency('MyCOIN') in exchange
False
:param item: currency or symbol for supports checking on exchange.
:type item: Text or Currency or Symbol
:return bool: True is item is supported, otherwise False.
"""
return str(item) in self.markets or str(item) in map(str, self.currencies)
class State(StateBase): # pylint: disable=too-many-instance-attributes
def __init__(self, root_dir=None, tmp_dir=None, dvcignore=None):
from diskcache import Cache
super().__init__()
self.tmp_dir = tmp_dir
self.root_dir = root_dir
self.dvcignore = dvcignore
if not tmp_dir:
return
config = {
"eviction_policy": "least-recently-used",
"disk_pickle_protocol": 4,
}
self.links = Cache(directory=os.path.join(tmp_dir, "links"), **config)
self.md5s = Cache(directory=os.path.join(tmp_dir, "md5s"), **config)
def close(self):
self.md5s.close()
self.links.close()
def save(self, path_info, fs, hash_info):
"""Save hash for the specified path info.
Args:
path_info (dict): path_info to save hash for.
hash_info (HashInfo): hash to save.
"""
if not isinstance(fs, LocalFileSystem):
return
mtime, size = get_mtime_and_size(path_info, fs, self.dvcignore)
inode = get_inode(path_info)
logger.debug(
"state save (%s, %s, %s) %s",
inode,
mtime,
str(size),
hash_info.value,
)
self.md5s[inode] = (mtime, str(size), hash_info.value)
def get(self, path_info, fs):
"""Gets the hash for the specified path info. Hash will be
retrieved from the state database if available.
Args:
path_info (dict): path info to get the hash for.
Returns:
HashInfo or None: hash for the specified path info or None if it
doesn't exist in the state database.
"""
from .objects.meta import Meta
if not isinstance(fs, LocalFileSystem):
return None, None
try:
mtime, size = get_mtime_and_size(path_info, fs, self.dvcignore)
except FileNotFoundError:
return None, None
inode = get_inode(path_info)
value = self.md5s.get(inode)
if not value or value[0] != mtime or value[1] != str(size):
return None, None
return Meta(size=size), HashInfo("md5", value[2])
def save_link(self, path_info, fs):
"""Adds the specified path to the list of links created by dvc. This
list is later used on `dvc checkout` to cleanup old links.
Args:
path_info (dict): path info to add to the list of links.
"""
if not isinstance(fs, LocalFileSystem):
return
try:
mtime, _ = get_mtime_and_size(path_info, fs, self.dvcignore)
except FileNotFoundError:
return
inode = get_inode(path_info)
relative_path = relpath(path_info, self.root_dir)
with self.links as ref:
ref[relative_path] = (inode, mtime)
def get_unused_links(self, used, fs):
"""Removes all saved links except the ones that are used.
Args:
used (list): list of used links that should not be removed.
"""
if not isinstance(fs, LocalFileSystem):
return
unused = []
with self.links as ref:
for relative_path in ref:
path = os.path.join(self.root_dir, relative_path)
if path in used or not fs.exists(path):
continue
inode = get_inode(path)
mtime, _ = get_mtime_and_size(path, fs, self.dvcignore)
if ref[relative_path] == (inode, mtime):
logger.debug("Removing '%s' as unused link.", path)
unused.append(relative_path)
return unused
def remove_links(self, unused, fs):
if not isinstance(fs, LocalFileSystem):
return
for path in unused:
remove(os.path.join(self.root_dir, path))
with self.links as ref:
for path in unused:
del ref[path]
def __init__(self, cache_duration=3600, cache_root='/tmp/cortex/tor_project'):
self.session = requests.Session()
self.cache_duration = cache_duration
if self.cache_duration > 0:
self.cache = Cache(cache_root)
self.url = 'http://torstatus.blutmagie.de/query_export.php/Tor_query_EXPORT.csv'
