class CDNClient(object):
DepotManifestClass = CDNDepotManifest
_LOG = logging.getLogger("CDNClient")
servers = deque() #: CS Server list
_chunk_cache = LRUCache(20)
cell_id = 0 #: Cell ID to use, initialized from SteamClient instance
def __init__(self, client):
"""CDNClient allows loading and reading of manifests for Steam apps are used
to list and download content
:param client: logged in SteamClient instance
:type client: :class:`.SteamClient`
"""
self.gpool = GPool(8) #: task pool
self.steam = client #: SteamClient instance
if self.steam:
self.cell_id = self.steam.cell_id
self.web = make_requests_session()
self.depot_keys = {} #: depot decryption keys
self.manifests = {} #: CDNDepotManifest instances
self.app_depots = {} #: app depot info
self.beta_passwords = {} #: beta branch decryption keys
self.licensed_app_ids = set(
) #: app_ids that the SteamClient instance has access to
self.licensed_depot_ids = set(
) #: depot_ids that the SteamClient instance has access to
if not self.servers:
self.fetch_content_servers()
self.load_licenses()
def load_licenses(self):
"""Read licenses from SteamClient instance, required for determining accessible content"""
self.licensed_app_ids.clear()
self.licensed_depot_ids.clear()
if self.steam.steam_id.type == EType.AnonUser:
packages = [17906]
else:
if not self.steam.licenses:
self._LOG.debug(
"No steam licenses found on SteamClient instance")
return
packages = list(self.steam.licenses.keys())
for package_id, info in iteritems(
self.steam.get_product_info(packages=packages)['packages']):
self.licensed_app_ids.update(info['appids'].values())
self.licensed_depot_ids.update(info['depotids'].values())
def fetch_content_servers(self, num_servers=20):
"""Update CS server list
:param num_servers: numbers of CS server to fetch
:type num_servers: int
"""
self.servers.clear()
self._LOG.debug("Trying to fetch content servers from Steam API")
servers = get_content_servers_from_webapi(self.cell_id)
self.servers.extend(servers)
if not self.servers:
raise SteamError("Failed to fetch content servers")
def get_content_server(self, rotate=False):
"""Get a CS server for content download
:param rotate: forcefully rotate server list and get a new server
:type rotate: bool
"""
if rotate:
self.servers.rotate(-1)
return self.servers[0]
def get_depot_key(self, app_id, depot_id):
"""Get depot key, which is needed to decrypt files
:param app_id: app id
:type app_id: int
:param depot_id: depot id
:type depot_id: int
:return: returns decryption key
:rtype: bytes
:raises SteamError: error message
"""
if depot_id not in self.depot_keys:
msg = self.steam.get_depot_key(app_id, depot_id)
if msg and msg.eresult == EResult.OK:
self.depot_keys[depot_id] = msg.depot_encryption_key
else:
raise SteamError(
"Failed getting depot key",
EResult.Timeout if msg is None else EResult(msg.eresult))
return self.depot_keys[depot_id]
def cdn_cmd(self, command, args):
"""Run CDN command request
:param command: command name
:type command: str
:param args: args
:type args: str
:returns: requests response
:rtype: :class:`requests.Response`
:raises SteamError: on error
"""
server = self.get_content_server()
while True:
url = "%s://%s:%s/%s/%s" % (
'https' if server.https else 'http',
server.host,
server.port,
command,
args,
)
try:
resp = self.web.get(url, timeout=10)
except Exception as exp:
self._LOG.debug("Request error: %s", exp)
else:
if resp.ok:
return resp
elif 400 <= resp.status_code < 500:
self._LOG.debug("Got HTTP %s", resp.status_code)
raise SteamError("HTTP Error %s" % resp.status_code)
self.steam.sleep(0.5)
server = self.get_content_server(rotate=True)
def get_chunk(self, app_id, depot_id, chunk_id):
"""Download a single content chunk
:param app_id: App ID
:type app_id: int
:param depot_id: Depot ID
:type depot_id: int
:param chunk_id: Chunk ID
:type chunk_id: int
:returns: chunk data
:rtype: bytes
:raises SteamError: error message
"""
if (depot_id, chunk_id) not in self._chunk_cache:
resp = self.cdn_cmd('depot', '%s/chunk/%s' % (depot_id, chunk_id))
data = symmetric_decrypt(resp.content,
self.get_depot_key(app_id, depot_id))
if data[:2] == b'VZ':
if data[-2:] != b'zv':
raise SteamError("VZ: Invalid footer: %s" %
repr(data[-2:]))
if data[2:3] != b'a':
raise SteamError("VZ: Invalid version: %s" %
repr(data[2:3]))
vzfilter = lzma._decode_filter_properties(
lzma.FILTER_LZMA1, data[7:12])
vzdec = lzma.LZMADecompressor(lzma.FORMAT_RAW,
filters=[vzfilter])
checksum, decompressed_size = struct.unpack(
'<II', data[-10:-2])
# decompress_size is needed since lzma will sometime produce longer output
# [12:-9] is need as sometimes lzma will produce shorter output
# together they get us the right data
data = vzdec.decompress(data[12:-9])[:decompressed_size]
if crc32(data) != checksum:
raise SteamError(
"VZ: CRC32 checksum doesn't match for decompressed data"
)
else:
with ZipFile(BytesIO(data)) as zf:
data = zf.read(zf.filelist[0])
self._chunk_cache[(depot_id, chunk_id)] = data
return self._chunk_cache[(depot_id, chunk_id)]
def get_manifest(self, app_id, depot_id, manifest_gid, decrypt=True):
"""Download a manifest file
:param app_id: App ID
:type app_id: int
:param depot_id: Depot ID
:type depot_id: int
:param manifest_gid: Manifest gid
:type manifest_gid: int
:param decrypt: Decrypt manifest filenames
:type decrypt: bool
:returns: manifest instance
:rtype: :class:`.CDNDepotManifest`
"""
if (app_id, depot_id, manifest_gid) not in self.manifests:
resp = self.cdn_cmd('depot',
'%s/manifest/%s/5' % (depot_id, manifest_gid))
if resp.ok:
manifest = self.DepotManifestClass(self, app_id, resp.content)
if decrypt:
manifest.decrypt_filenames(
self.get_depot_key(app_id, depot_id))
self.manifests[(app_id, depot_id, manifest_gid)] = manifest
return self.manifests[(app_id, depot_id, manifest_gid)]
def check_beta_password(self, app_id, password):
"""Check branch beta password to unlock encrypted branches
:param app_id: App ID
:type app_id: int
:param password: beta password
:type password: str
:returns: result
:rtype: :class:`.EResult`
"""
resp = self.steam.send_job_and_wait(
MsgProto(EMsg.ClientCheckAppBetaPassword), {
'app_id': app_id,
'betapassword': password
})
if resp.eresult == EResult.OK:
self._LOG.debug(
"Unlocked following beta branches: %s",
', '.join(map(lambda x: x.betaname.lower(),
resp.betapasswords)))
for entry in resp.betapasswords:
self.beta_passwords[(app_id,
entry.betaname.lower())] = unhexlify(
entry.betapassword)
else:
self._LOG.debug("App beta password check failed. %r" %
EResult(resp.eresult))
return EResult(resp.eresult)
def get_app_depot_info(self, app_id):
if app_id not in self.app_depots:
self.app_depots[app_id] = self.steam.get_product_info(
[app_id])['apps'][app_id]['depots']
return self.app_depots[app_id]
def get_manifests(self,
app_id,
branch='public',
password=None,
filter_func=None,
decrypt=True):
"""Get a list of CDNDepotManifest for app
:param app_id: App ID
:type app_id: int
:param branch: branch name
:type branch: str
:param password: branch password for locked branches
:type password: str
:param filter_func:
Function to filter depots. ``func(depot_id, depot_info)``
:returns: list of :class:`.CDNDepotManifest`
:rtype: :class:`list` [:class:`.CDNDepotManifest`]
:raises SteamError: error message
"""
depots = self.get_app_depot_info(app_id)
is_enc_branch = False
if branch not in depots['branches']:
raise SteamError("No branch named %s for app_id %s" %
(repr(branch), app_id))
elif int(depots['branches'][branch].get('pwdrequired', 0)) > 0:
is_enc_branch = True
if (app_id, branch) not in self.beta_passwords:
if not password:
raise SteamError("Branch %r requires a password" % branch)
result = self.check_beta_password(app_id, password)
if result != EResult.OK:
raise SteamError("Branch password is not valid. %r" %
result)
if (app_id, branch) not in self.beta_passwords:
raise SteamError("Incorrect password for branch %r" %
branch)
def async_fetch_manifest(app_id, depot_id, manifest_gid, decrypt,
name):
manifest = self.get_manifest(app_id, depot_id, manifest_gid,
decrypt)
manifest.name = name
return manifest
tasks = []
shared_depots = {}
for depot_id, depot_info in iteritems(depots):
if not depot_id.isdigit():
continue
depot_id = int(depot_id)
# if filter_func set, use it to filter the list the depots
if filter_func and not filter_func(depot_id, depot_info):
continue
# if we have no license for the depot, no point trying as we won't get depot_key
if decrypt and depot_id not in self.licensed_depot_ids:
self._LOG.debug(
"No license for depot %s (%s). Skipping...",
repr(depot_info['name']),
depot_id,
)
continue
# accumulate the shared depots
if 'depotfromapp' in depot_info:
shared_depots.setdefault(int(depot_info['depotfromapp']),
set()).add(depot_id)
continue
# process depot, and get manifest for branch
if is_enc_branch:
egid = depot_info.get('encryptedmanifests',
{}).get(branch,
{}).get('encrypted_gid_2')
if egid is not None:
manifest_gid = decrypt_manifest_gid_2(
unhexlify(egid), self.beta_passwords[(app_id, branch)])
else:
manifest_gid = depot_info.get('manifests',
{}).get('public')
else:
manifest_gid = depot_info.get('manifests', {}).get(branch)
if manifest_gid is not None:
tasks.append(
self.gpool.spawn(
async_fetch_manifest,
app_id,
depot_id,
manifest_gid,
decrypt,
depot_info['name'],
))
# collect results
manifests = []
for task in tasks:
manifests.append(task.get())
# try:
# result = task.get()
# except SteamError as exp:
# self._LOG.error("Error: %s", exp)
# raise
# else:
# if isinstance(result, list):
# manifests.extend(result)
# else:
# manifests.append(result)
# load shared depot manifests
for app_id, depot_ids in iteritems(shared_depots):
def nested_ffunc(depot_id,
depot_info,
depot_ids=depot_ids,
ffunc=filter_func):
return (int(depot_id) in depot_ids
and (ffunc is None or ffunc(depot_id, depot_info)))
manifests += self.get_manifests(app_id, filter_func=nested_ffunc)
return manifests
def iter_files(self,
app_id,
filename_filter=None,
branch='public',
password=None,
filter_func=None):
"""Like :meth:`.get_manifests` but returns a iterator that goes through all the files
in all the manifest.
:param app_id: App ID
:type app_id: int
:param filename_filter: wildcard filter for file paths
:type branch: str
:param branch: branch name
:type branch: str
:param password: branch password for locked branches
:type password: str
:param filter_func:
Function to filter depots. ``func(depot_id, depot_info)``
:returns: generator of of CDN files
:rtype: [:class:`.CDNDepotFile`]
"""
for manifest in self.get_manifests(app_id, branch, password,
filter_func):
for fp in manifest.iter_files(filename_filter):
yield fp
def get_manifest_for_workshop_item(self, item_id):
"""Get the manifest file for a worshop item that is hosted on SteamPipe
:param item_id: Workshop ID
:type item_id: int
:returns: manifest instance
:rtype: :class:`.CDNDepotManifest`
:raises SteamError: error message
"""
resp = self.steam.send_um_and_wait('PublishedFile.GetDetails#1', {
'publishedfileids': [item_id],
'includetags': False,
'includeadditionalpreviews': False,
'includechildren': False,
'includekvtags': False,
'includevotes': False,
'short_description': True,
'includeforsaledata': False,
'includemetadata': False,
'language': 0
},
timeout=7)
if resp.header.eresult != EResult.OK:
raise SteamError(resp.header.error_message or 'No message',
resp.header.eresult)
wf = None if resp is None else resp.body.publishedfiledetails[0]
if wf is None or wf.result != EResult.OK:
raise SteamError(
"Failed getting workshop file info",
EResult.Timeout if resp is None else EResult(wf.result))
elif not wf.hcontent_file:
raise SteamError("Workshop file is not on SteamPipe",
EResult.FileNotFound)
app_id = ws_app_id = wf.consumer_appid
manifest = self.get_manifest(app_id, ws_app_id, wf.hcontent_file)
manifest.name = wf.title
return manifest
def _initialize_block_cache(self):
self._block_cache = LRUCache(maxsize=self._cache_size)
self._cache_hit_count = 0
self._cache_miss_count = 0
# dictionary, but they don't get added to the givens
# dictionary. Here, we try to fix that.
if param in named_nodes_ancestors:
for node in named_nodes_ancestors[param]:
if node.name not in givens and (node, size) in drawn:
givens[node.name] = (node, drawn[(node, size)])
value = _draw_value(param, point=point, givens=givens.values(), size=size)
evaluated[param_idx] = drawn[(param, size)] = value
givens[param.name] = (param, value)
except aesara.graph.fg.MissingInputError:
missing_inputs.add(param_idx)
return [evaluated[j] for j in params] # set the order back
@cached(LRUCache(128), key=hash_key)
def _compile_aesara_function(param, vars, givens=None):
"""Compile aesara function for a given parameter and input variables.
This function is memoized to avoid repeating costly aesara compilations
when repeatedly drawing values, which is done when generating posterior
predictive samples.
Parameters
----------
param: Model variable from which to draw value
vars: Children variables of `param`
givens: Variables to be replaced in the Aesara graph
Returns
-------
@cached(TTLCache(100, 600))
async def get_artist(artist_id):
if not var.spotify_token or time() > var.spotify_token_expires:
await authorize()
r = await request_get(
f'https://api.spotify.com/v1/artists/{artist_id}',
headers={'Authorization': f'Bearer {var.spotify_token}'})
print(r.url)
json = await r.json()
if not json.get('error'):
return AttrDict(json)
else:
raise ValueError('Error getting artist: ' + json.get('error'))
@cached(LRUCache(5000))
async def match_track(spotify_track_id):
sp_track = await get_track(spotify_track_id)
search_query = f'{sp_track.artists[0].name} {sp_track.name}'
search_query2 = f'{sp_track.artists[0].name} {sp_track.name}'\
.lower().split('(f')[0].split('feat.')[0].split('ft.')[0]
tracks = await deezer_api.search(search_query)
if not tracks:
tracks = await deezer_api.search(search_query2)
return tracks and tracks[0]
@cached(LRUCache(5000))
async def match_album(spotify_album_id):
sp_album = await get_album(spotify_album_id)
def __init__(self,
name,
data,
lon,
lat,
depth=None,
time=None,
transpose=False,
vmin=None,
vmax=None,
time_origin=0,
units=None,
interp_method='linear',
allow_time_extrapolation=None):
self.name = name
self.data = data
self.lon = lon
self.lat = lat
self.depth = np.zeros(1, dtype=np.float32) if depth is None else depth
self.time = np.zeros(1, dtype=np.float64) if time is None else time
self.time_origin = time_origin
self.units = units if units is not None else UnitConverter()
self.interp_method = interp_method
if allow_time_extrapolation is None:
self.allow_time_extrapolation = True if time is None else False
else:
self.allow_time_extrapolation = allow_time_extrapolation
# Ensure that field data is the right data type
if not self.data.dtype == np.float32:
print("WARNING: Casting field data to np.float32")
self.data = self.data.astype(np.float32)
if not self.lon.dtype == np.float32:
print("WARNING: Casting lon data to np.float32")
self.lon = self.lon.astype(np.float32)
if not self.lat.dtype == np.float32:
print("WARNING: Casting lat data to np.float32")
self.lat = self.lat.astype(np.float32)
if not self.time.dtype == np.float64:
print("WARNING: Casting time data to np.float64")
self.time = self.time.astype(np.float64)
if transpose:
# Make a copy of the transposed array to enforce
# C-contiguous memory layout for JIT mode.
self.data = np.transpose(self.data).copy()
self.data = self.data.reshape(
(self.time.size, self.lat.size, self.lon.size))
# Hack around the fact that NaN and ridiculously large values
# propagate in SciPy's interpolators
if vmin is not None:
self.data[self.data < vmin] = 0.
if vmax is not None:
self.data[self.data > vmax] = 0.
self.data[np.isnan(self.data)] = 0.
# Variable names in JIT code
self.ccode_data = self.name
self.ccode_lon = self.name + "_lon"
self.ccode_lat = self.name + "_lat"
self.interpolator_cache = LRUCache(maxsize=2)
self.time_index_cache = LRUCache(maxsize=2)
class MapMask:
def __init__(self, img_file: str, resolution: float = 0.1):
"""
Init a map mask object that contains the semantic prior (drivable surface and sidewalks) mask.
:param img_file: File path to map png file.
:param resolution: Map resolution in meters.
"""
assert osp.exists(img_file), 'map mask {} does not exist'.format(
img_file)
assert resolution >= 0.1, "Only supports down to 0.1 meter resolution."
self.img_file = img_file
self.resolution = resolution
self.foreground = 255
self.background = 0
@cached(cache=LRUCache(maxsize=3))
def mask(self, dilation: float = 0.0) -> np.ndarray:
"""
Returns the map mask, optionally dilated.
:param dilation: Dilation in meters.
:return: Dilated map mask.
"""
if dilation == 0:
return self._base_mask
else:
distance_mask = cv2.distanceTransform(
(self.foreground - self._base_mask).astype(np.uint8),
cv2.DIST_L2, 5)
distance_mask = (distance_mask * self.resolution).astype(
np.float32)
return (distance_mask <= dilation).astype(
np.uint8) * self.foreground
@property
def transform_matrix(self) -> np.ndarray:
"""
Generate transform matrix for this map mask.
:return: <np.array: 4, 4>. The transformation matrix.
"""
return np.array(
[[1.0 / self.resolution, 0, 0, 0],
[0, -1.0 / self.resolution, 0, self._base_mask.shape[0]],
[0, 0, 1, 0], [0, 0, 0, 1]])
def is_on_mask(self, x: Any, y: Any, dilation: float = 0) -> np.array:
"""
Determine whether the given coordinates are on the (optionally dilated) map mask.
:param x: Global x coordinates. Can be a scalar, list or a numpy array of x coordinates.
:param y: Global y coordinates. Can be a scalar, list or a numpy array of x coordinates.
:param dilation: Optional dilation of map mask.
:return: <np.bool: x.shape>. Whether the points are on the mask.
"""
px, py = self.to_pixel_coords(x, y)
on_mask = np.ones(px.size, dtype=np.bool)
this_mask = self.mask(dilation)
on_mask[px < 0] = False
on_mask[px >= this_mask.shape[1]] = False
on_mask[py < 0] = False
on_mask[py >= this_mask.shape[0]] = False
on_mask[on_mask] = this_mask[py[on_mask],
px[on_mask]] == self.foreground
return on_mask
def to_pixel_coords(self, x: Any, y: Any) -> Tuple[np.ndarray, np.ndarray]:
"""
Maps x, y location in global map coordinates to the map image coordinates.
:param x: Global x coordinates. Can be a scalar, list or a numpy array of x coordinates.
:param y: Global y coordinates. Can be a scalar, list or a numpy array of x coordinates.
:return: (px <np.uint8: x.shape>, py <np.uint8: y.shape>). Pixel coordinates in map.
"""
x = np.array(x)
y = np.array(y)
x = np.atleast_1d(x)
y = np.atleast_1d(y)
assert x.shape == y.shape
assert x.ndim == y.ndim == 1
pts = np.stack([x, y, np.zeros(x.shape), np.ones(x.shape)])
pixel_coords = np.round(np.dot(self.transform_matrix,
pts)).astype(np.int32)
return pixel_coords[0, :], pixel_coords[1, :]
@property
@cached(cache=LRUCache(maxsize=1))
def _base_mask(self) -> np.ndarray:
"""
Returns the original binary mask stored in map png file.
:return: <np.int8: image.height, image.width>. The binary mask.
"""
# Pillow allows us to specify the maximum image size above, whereas this is more difficult in OpenCV.
img = Image.open(self.img_file)
# Resize map mask to desired resolution.
native_resolution = 0.1
size_x = int(img.size[0] / self.resolution * native_resolution)
size_y = int(img.size[1] / self.resolution * native_resolution)
img = img.resize((size_x, size_y), resample=Image.NEAREST)
# Convert to numpy.
raw_mask = np.array(img)
return raw_mask
def get_project_base_directory():
global PROJECT_BASE
if PROJECT_BASE is None:
PROJECT_BASE = os.path.abspath(
os.path.join(os.path.dirname(os.path.realpath(__file__)),
os.pardir, os.pardir, os.pardir))
return PROJECT_BASE
def get_python_base_directory():
return os.path.join(get_project_base_directory(), "python")
@cached(cache=LRUCache(maxsize=10))
def load_json_conf(conf_path):
if os.path.isabs(conf_path):
json_conf_path = conf_path
else:
json_conf_path = os.path.join(get_project_base_directory(), conf_path)
try:
with open(json_conf_path) as f:
return json.load(f)
except:
raise EnvironmentError(
"loading json file config from '{}' failed!".format(
json_conf_path))
def dump_json_conf(config_data, conf_path):
from cachetools import cached, LRUCache
from smart_open import smart_open
import logging
logging.basicConfig(
format='%(asctime)s [%(process)d] [%(levelname)s] %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
level=logging.INFO)
PUNCTSYM = re.compile(r'^[\p{P}\p{S}]+$')
LIKENUM = re.compile(r'^[\p{N},\.-]+$')
LIKEUNIT = re.compile(r'^[\p{N},\.-]+(\w+)$')
CONTRACTION1 = re.compile(r"(\w+)'(s|m|re|ve|d|ll)$")
CONTRACTION2 = re.compile(r"(\w+)n't$")
cache = LRUCache(maxsize=100000)
@cached(cache)
def tokenizer(text, tokenizer_fn, to_lower=False):
text = ftfy.fix_text(text)
if to_lower:
text = text.lower()
try:
seq = Sequence(text.strip())
except ValueError:
return
tokens = tokenizer_fn.transform(seq)
new_tokens = []
for token in tokens:
if token.strip() == '':
import collections
from ufl.indexed import Indexed
from ufl.domain import join_domains
from pyop2 import READ, WRITE, RW, INC, MIN, MAX
import pyop2
import loopy
import coffee.base as ast
from firedrake.logging import warning
from firedrake import constant
from firedrake.utils import ScalarType_c
try:
from cachetools import LRUCache
kernel_cache = LRUCache(maxsize=128)
except ImportError:
warning("cachetools not available, firedrake.par_loop calls will be slowed down")
kernel_cache = None
__all__ = ['par_loop', 'direct', 'READ', 'WRITE', 'RW', 'INC', 'MIN', 'MAX']
class _DirectLoop(object):
r"""A singleton object which can be used in a :func:`par_loop` in place
of the measure in order to indicate that the loop is a direct loop
over degrees of freedom."""
def integral_type(self):
return "direct"
def __init__(self, *args):
self.weighted_servers = args
self.cache = LRUCache(1024)
<td><b style="color: {event_color}">{record.event}</b></td>
<td class="time"><span title="{time_absolute}" style="{time_color}">{time_display}</span></td>
<td>{record.logger}</td>
<td>{record.level}</td>
<td>{fields}</td>
</tr>
"""
def _colorize_cache_key(value: Any, min_luminance: float) -> Tuple[str, float]:
if isinstance(value, (list, dict)):
return repr(value), min_luminance
return value, min_luminance
@cached(LRUCache(maxsize=2**24))
def rgb_color_picker(obj,
min_luminance: float = None,
max_luminance: float = None) -> Color:
"""Modified version of colour.RGB_color_picker"""
color_value = (int.from_bytes(
hashlib.md5(str(obj).encode("utf-8")).digest(), "little") % 0xFFFFFF)
color = Color(f"#{color_value:06x}")
if min_luminance and color.get_luminance() < min_luminance:
color.set_luminance(min_luminance)
elif max_luminance and color.get_luminance() > max_luminance:
color.set_luminance(max_luminance)
return color
def nice_time_diff(time_base: datetime,
class StreetViewFSA:
""" Finite State Automaton for Street-View """
OLD_FEATURE, NEW_FEATURE, RGB = range(3)
image_feature_cache = LRUCache(maxsize=100000)
def __init__(self, node_file, link_file, image_feature_folder,
forward_setting_strict, mapping_type):
self.image_feature_folder = image_feature_folder
self.forward_setting_strict = forward_setting_strict
if mapping_type == "old":
self.mapping = StreetViewFSA.OLD_FEATURE
elif mapping_type == "new":
self.mapping = StreetViewFSA.NEW_FEATURE
elif mapping_type == "rgb":
self.mapping = StreetViewFSA.RGB
else:
raise AssertionError("Unhandled Mapping type " % mapping_type)
if self.mapping == StreetViewFSA.OLD_FEATURE:
self.image_feature_mapping = defaultdict(list)
for feature_file in os.listdir(self.image_feature_folder):
# the file name is formated as {panorama_id}.{perspective_angle}.npy
panorama_id, angle, ext = feature_file.split(".")
self.image_feature_mapping[panorama_id].append(
(int(angle),
os.path.join(self.image_feature_folder, feature_file)))
# sort angles for all perspective views in each node (panorama)
for panorama_id in self.image_feature_mapping.keys():
self.image_feature_mapping[panorama_id] = sorted(
self.image_feature_mapping[panorama_id],
key=lambda x: x[0])
else:
self.image_feature_mapping = None
self.node_to_panorama_dict = dict()
self.panorama_to_node_dict = dict()
node_lines = open(node_file).readlines()
for line in node_lines:
words = line.split(",")
assert len(words) == 5
# ignoring latitude and longititude information at 4th and 5th place
node_id = words[0]
panorama_id = words[1]
panorama_orientation = int(words[2])
self.node_to_panorama_dict[node_id] = (panorama_id,
panorama_orientation)
self.panorama_to_node_dict[panorama_id] = node_id
link_lines = open(link_file).readlines()
self.outgoing_edges = dict()
for line in link_lines:
words = line.split(",")
assert len(words) == 4
source_node_id = words[0]
orientation = int(words[1])
dest_node_id = words[2]
if source_node_id in self.outgoing_edges:
self.outgoing_edges[source_node_id].append(
(dest_node_id, orientation))
else:
self.outgoing_edges[source_node_id] = [(dest_node_id,
orientation)]
# self.fsa_states = set()
# for source_id in self.outgoing_edges:
# for dest_node_id, orientation in self.outgoing_edges[source_id]:
# self.fsa_states.add((dest_node_id, orientation)) # one can enter dest_node_id with this orientation
# self.fsa_states.add((source_id, orientation)) # one can also turn in the original node to face this orientation
def reset_to_new_state(self, datapoint):
trajectory = datapoint.get_trajectory()
start_panorama_id = trajectory[0]
goal_panorama_id = trajectory[-1]
start_node_id = self.panorama_to_node_dict[start_panorama_id]
goal_node_id = self.panorama_to_node_dict[goal_panorama_id]
start_orientation = datapoint.get_start_orientation()
goal_orientation = datapoint.get_end_orientation()
if self.mapping == StreetViewFSA.RGB:
# For RGBs image we align the start orientation to face the road
shortest_angle, shortest_angle_orientation = 1000, None
for (_, orientation) in self.outgoing_edges[start_node_id]:
if orientation == start_orientation:
shortest_angle_orientation = None
break
angle_diff = self.get_turn_angle(start_orientation,
orientation, "shortest")
if angle_diff < shortest_angle:
shortest_angle = angle_diff
shortest_angle_orientation = orientation
if shortest_angle_orientation is not None:
start_orientation = shortest_angle_orientation
return StreetViewState(start_node_id,
start_orientation), StreetViewState(
goal_node_id, goal_orientation)
def take_action_return_new_state(self, state, act_name):
node_id = state.get_node_id()
orientation = state.get_orientation()
outgoing_edges = self.outgoing_edges[node_id]
if act_name == "forward":
# Find the closest road where the agent can go to
smallest_angle_diff = 1000 # set to a value more than 360 degree
outgoing_edge = outgoing_edges[0]
for (new_node_id, new_orientation) in outgoing_edges:
angle_diff = self.get_turn_angle(orientation,
new_orientation,
direction="shortest")
if angle_diff < smallest_angle_diff:
smallest_angle_diff = angle_diff
outgoing_edge = (new_node_id, new_orientation)
new_node_id, new_orientation = outgoing_edge
next_smallest_angle_diff = 1000
closest_next_orientation = new_orientation
new_outgoing_edges = self.outgoing_edges[new_node_id]
for (_, next_orientation) in new_outgoing_edges:
angle_diff = self.get_turn_angle(next_orientation,
new_orientation,
direction="shortest")
if angle_diff < next_smallest_angle_diff:
next_smallest_angle_diff = angle_diff
closest_next_orientation = next_orientation
if self.forward_setting_strict:
if smallest_angle_diff == 0.0:
return StreetViewState(new_node_id,
closest_next_orientation)
else:
return StreetViewState(node_id, orientation)
else:
return StreetViewState(new_node_id, closest_next_orientation)
elif act_name == "turnleft":
smallest_angle_diff = 1000 # set to a value more than 360 degree
outgoing_edge = outgoing_edges[0]
for (new_node_id, new_orientation) in outgoing_edges:
if new_orientation == orientation:
continue
angle_diff = self.get_turn_angle(orientation,
new_orientation,
direction="left")
if 0 < angle_diff < smallest_angle_diff:
smallest_angle_diff = angle_diff
outgoing_edge = (new_node_id, new_orientation)
new_node_id, new_orientation = outgoing_edge
return StreetViewState(node_id, new_orientation)
elif act_name == "turnright":
smallest_angle_diff = 1000 # set to a value more than 360 degree
outgoing_edge = outgoing_edges[0]
for (new_node_id, new_orientation) in outgoing_edges:
if new_orientation == orientation:
continue
angle_diff = self.get_turn_angle(orientation,
new_orientation,
direction="right")
if 0 < angle_diff < smallest_angle_diff:
smallest_angle_diff = angle_diff
outgoing_edge = (new_node_id, new_orientation)
new_node_id, new_orientation = outgoing_edge
return StreetViewState(node_id, new_orientation)
elif act_name == "stop":
return StreetViewState(state.get_node_id(),
state.get_orientation())
else:
raise AssertionError("Unhandled action name %s " % act_name)
def get_image_from_state(self, state):
if self.mapping == StreetViewFSA.OLD_FEATURE:
return self.get_image_from_state_old(state)
elif self.mapping == StreetViewFSA.NEW_FEATURE:
return self.get_image_from_state_condensed_features(state)
elif self.mapping == StreetViewFSA.RGB:
return self.get_image_rgb(state)
else:
raise NotImplementedError()
def get_image_from_state_condensed_features(self, state):
panorama_id, _ = self.node_to_panorama_dict[state.get_node_id()]
state_orientation = state.get_orientation()
key = (panorama_id, state_orientation)
if key in self.image_feature_cache:
return self.image_feature_cache[key]
else:
_, panorama_orientation = self.node_to_panorama_dict[
state.get_node_id()]
path = os.path.join(self.image_feature_folder,
'{}.npy'.format(panorama_id))
image = np.load(path)
image = np.expand_dims(image, axis=0)
pano_width = image.shape[1]
shift_angle = 157.5 + panorama_orientation - state_orientation
shift = int(pano_width * shift_angle / 360)
image = np.roll(image, shift, axis=1)
image = image.swapaxes(1, 2).swapaxes(0, 1)
self.image_feature_cache[key] = image
return image
def get_image_from_state_old(self, state):
panorama_id, _ = self.node_to_panorama_dict[state.get_node_id()]
state_orientation = state.get_orientation()
key = (panorama_id, state_orientation)
if key in self.image_feature_cache:
return self.image_feature_cache[key]
else:
# OLD FEATURES
# find the closest angle from the agent orientation
index, _ = min(enumerate(self.image_feature_mapping[panorama_id]),
key=lambda x: (x[1][0] - state_orientation) % 360)
feature_paths = [
feature_path
for _, feature_path in self.image_feature_mapping[panorama_id]
]
# rotate the list to the right so the closest feature is in the middle
feature_paths = feature_paths[index - 3:] + feature_paths[:index -
3]
# load perspective image features and transpose so the depth is at the last dimension
perspective_features = [
np.load(feature_path).transpose(1, 2, 0)
for feature_path in feature_paths
]
# for each perspective image, the shape is (height, width, depth) = (100, 58, 128)
height, width, depth = perspective_features[0].shape
# concatenate all perspective images along the width axis
# so the full pano has shape (height, width, depth) = (100, 58 * 8, 128)
image = np.concatenate(perspective_features, axis=1)
# rotate to the right so half of the last feature is cut off and get appended to the left most
image = np.roll(image, int(width // 2), axis=1)
image = image.swapaxes(1, 2).swapaxes(0, 1)
self.image_feature_cache[key] = image
return image
def get_image_rgb(self, state):
panorama_id, _ = self.node_to_panorama_dict[state.get_node_id()]
state_orientation = int(state.get_orientation())
key = (panorama_id, state_orientation)
if key in self.image_feature_cache:
return self.image_feature_cache[key]
else:
path = os.path.join(
self.image_feature_folder,
'{}.{}.npy'.format(panorama_id, state_orientation))
image = np.load(path)
image = image / 255.0 # important to keep values in 0-1
image = image.swapaxes(1, 2).swapaxes(0, 1)
self.image_feature_cache[key] = image
return image
def get_panorama(self, state):
raise NotImplementedError()
@staticmethod
def get_turn_angle(angle1, angle2, direction="shortest"):
""" Turn angle from angle1 to angle2 """
if direction == "shortest":
diff = (angle1 - angle2) % 360
return min(360 - diff, diff)
elif direction == "left":
# turning left reduces the orientation
return (angle1 - angle2) % 360
elif direction == "right":
# turning right increases the orientation
return (angle2 - angle1) % 360
else:
raise AssertionError(
"Dir has to be either shortest, left or right.")
def cf(self):
return self.__dict__.setdefault(
"_cache", defaultdict(lambda: LRUCache(128)))[f_name]
from consumer import Aiohttp, Any, To, RouteId
from components import cache, direct, aiohttp_request, log, soup, zipper
from evaluator import header, body, set_body, get_header
import re
from cachetools import LRUCache
futaboard_cache = LRUCache(maxsize=1000)
#yapf:disable
(Aiohttp('/futaboard-images')
.validate({
'rule': lambda ex: re.match('^http://board.futakuro.com/jk2/res/\d+.htm$', ex.get_header('url','')),
'message':'urlパラメータに有効なふたボード urlをセットしてください。書式: http://board.futakuro.com/jk2/res/{id}.htm$'})
.to(cache({
'cache_object': futaboard_cache,
'keys': [header('url')],
'to': Any()
.throttle(1)
.to(cache({
'cache_object': futaboard_cache,
'keys': [header('url')],
'to': To(direct('futaboard_images_main'))
}))
}))
)
#yapf:disable
(RouteId('futaboard_images_main')
.validate({
'process_rule': To(aiohttp_request({'url': header('url'), 'isValid': True})),
'message': 'futaboardサーバーからのレスポンスに問題があるため、処理を中断しました。urlおよびfutaboardサーバーのステータスを確認してください。'
__all__ = (
'ScrambleCalculator',
'ScrambleBlockInfo',
)
from typing import List, NamedTuple
from cachetools import LRUCache
from ..constant import *
from ..definition import *
scramble_data_cache = LRUCache(maxsize=256)
class ScrambleBlockInfo(NamedTuple):
"""
Records the source and destination region of a single block in the image.
`Source` is assumed to be the original image and `Destination` is the scrambled image.
"""
src_x: int
src_y: int
dest_x: int
dest_y: int
width: int
height: int
class ScrambleCalculator:
# A constant in the JavaScript code of ComicFuz
NFBR_A0X_A3H = 4
def __init__(self, M, Q, S, overdispersion):
self._cache_SQt = LRUCache(maxsize=1)
self._cache_m = LRUCache(maxsize=1)
self._cache_K = LRUCache(maxsize=1)
self._cache_diagK = LRUCache(maxsize=1)
self._cache_update = LRUCache(maxsize=1)
self._cache_lml_components = LRUCache(maxsize=1)
self._cache_L = LRUCache(maxsize=1)
self._cache_A = LRUCache(maxsize=1)
self._cache_C = LRUCache(maxsize=1)
self._cache_BiQt = LRUCache(maxsize=1)
self._cache_QBiQtAm = LRUCache(maxsize=1)
self._cache_QBiQtCteta = LRUCache(maxsize=1)
self._logger = logging.getLogger(__name__)
if not is_all_finite(Q) or not is_all_finite(isfinite(S)):
raise ValueError("There are non-finite numbers in the provided" +
" eigen decomposition.")
if S.min() <= 0:
raise ValueError("The provided covariance matrix is not" +
" positive-definite because the minimum" +
" eigvalue is %f." % S.min())
make_sure_reasonable_conditioning(S)
self._S = S
self._Q = Q
self.__QSQt = None
nsamples = M.shape[0]
self._previous_sitelik_tau = zeros(nsamples)
self._previous_sitelik_eta = zeros(nsamples)
self._sitelik_tau = zeros(nsamples)
self._sitelik_eta = zeros(nsamples)
self._cav_tau = zeros(nsamples)
self._cav_eta = zeros(nsamples)
self._joint_tau = zeros(nsamples)
self._joint_eta = zeros(nsamples)
self._v = None
self._delta = 0
self._overdispersion = overdispersion
self._tM = None
self.__tbeta = None
self._covariate_setup(M)
self._loghz = empty(nsamples)
self._hmu = empty(nsamples)
self._hvar = empty(nsamples)
self._ep_params_initialized = False
no_mmap = os.environ.get('VAEX_NO_MMAP', False)
if no_mmap:
from ctypes import *
libc = cdll.LoadLibrary("libc.dylib")
import ctypes
import io
from cachetools import LRUCache
import threading
GB = 1024**3
def getsizeof(ar):
return ar.nbytes
cache = LRUCache(maxsize=10 * GB, getsizeof=getsizeof)
cache_lock = threading.RLock()
F_NOCACHE = 48
class ColumnReader(vaex.dataset.Column):
def __init__(self, dataset, file, byte_offset, length, dtype):
self.dataset = dataset
self.file = file
fcntl.fcntl(self.file.fileno(), F_NOCACHE, 1)
#libc.fcntl(self.file.fileno(), fcntl.F_NOCACHE, 1)
#libc.fcntl(c_int(self.file.fileno()), c_int(fcntl.F_NOCACHE), c_int(1))
self.byte_offset = byte_offset
self.length = length
self.dtype = np.dtype(dtype)
self.shape = (length, )
def _copy_to(self, ep):
ep._cache_SQt = LRUCache(maxsize=1)
ep._cache_m = LRUCache(maxsize=1)
ep._cache_K = LRUCache(maxsize=1)
ep._cache_diagK = LRUCache(maxsize=1)
ep._cache_update = LRUCache(maxsize=1)
ep._cache_lml_components = LRUCache(maxsize=1)
ep._cache_L = LRUCache(maxsize=1)
ep._cache_A = LRUCache(maxsize=1)
ep._cache_C = LRUCache(maxsize=1)
ep._cache_BiQt = LRUCache(maxsize=1)
ep._cache_QBiQtAm = LRUCache(maxsize=1)
ep._cache_QBiQtCteta = LRUCache(maxsize=1)
ep._logger = logging.getLogger(__name__)
ep._S = self._S
ep._Q = self._Q
ep.__QSQt = self.__QSQt
ep._previous_sitelik_tau = self._previous_sitelik_tau.copy()
ep._previous_sitelik_eta = self._previous_sitelik_eta.copy()
ep._sitelik_tau = self._sitelik_tau.copy()
ep._sitelik_eta = self._sitelik_eta.copy()
ep._cav_tau = self._cav_tau.copy()
ep._cav_eta = self._cav_eta.copy()
ep._joint_tau = self._joint_tau.copy()
ep._joint_eta = self._joint_eta.copy()
ep._v = self._v
ep._delta = self._delta
ep._overdispersion = self._overdispersion
ep._tM = self._tM
ep.__tbeta = self.__tbeta.copy()
ep._M = self._M
ep._svd_U = self._svd_U
ep._svd_S12 = self._svd_S12
ep._svd_V = self._svd_V
ep._loghz = self._loghz.copy()
ep._hmu = self._hmu.copy()
ep._hvar = self._hvar.copy()
ep._ep_params_initialized = self._ep_params_initialized
_logger.warning(f"invalid poll data {choice} {obj} {meta}")
return 0
@filters.app_template_filter()
def get_total_answers_count(obj, meta):
cached = meta.get("question_replies", 0)
if cached:
return cached
cnt = 0
for choice in obj.get("anyOf", obj.get("oneOf", [])):
cnt += choice.get("replies", {}).get("totalItems", 0)
return cnt
_FILE_URL_CACHE = LRUCache(4096)
def _get_file_url(url, size, kind) -> str:
k = (url, size, kind)
cached = _FILE_URL_CACHE.get(k)
if cached:
return cached
doc = MEDIA_CACHE.get_file(*k)
if doc:
out = f"/media/{str(doc._id)}"
_FILE_URL_CACHE[k] = out
return out
_logger.error(f"cache not available for {url}/{size}/{kind}")
class AirProfile(object):
"""Automatic analysis of Airbnb Host profiles.
To Use:
>>> from AirProfile import AirProfile
>>> ap = AirProfile(liwc_path='../LIWC2007/liwc_2007.trie')
>>> host_profile = "I have spent my life in the service industry." \
"I look forward to being your host and I look forward to meeting you."
>>> ap.predict_topics(input)
>>> ap.predict_trust(input)
Attributes:
Prediction: A named tuple of the prediction result and probability.
"""
__liwc_internal__ = None
__vec_internal__ = None
__classifier_cat_internal__ = None
__classifier_trust_internal__ = {}
__lemm_n = LRUCache(maxsize=256,
missing=partial(WordNetLemmatizer().lemmatize,
pos='n'))
__lemm_v = LRUCache(maxsize=256,
missing=partial(WordNetLemmatizer().lemmatize,
pos='v'))
__SentenceToken = namedtuple('SentenceToken', ['raw', 'clean'])
Prediction = namedtuple('Prediction', ['prob', 'predict'])
def __init__(self, liwc_path='', model_dir_path='./AirProfile/models'):
"""
Args:
liwc_path: The path to the LIWC 2007 trie. Trust prediction will
not work unless this is specified.
model_dir_path: The directory path for the AirProfile models.
Defaults to './AirProfile/models'.
"""
self.__liwc_path = liwc_path if isinstance(liwc_path,
Path) else Path(liwc_path)
# TODO(kenlimmj): This should perform a high-level integrity check on
# the directory structure.
self.__model_dir_path = model_dir_path if isinstance(
model_dir_path, Path) else Path(model_dir_path)
@property
def __liwc(self):
if not self.__liwc_internal__:
self.__liwc_internal__ = LiwcUtil(self.__liwc_path)
return self.__liwc_internal__
@property
def __vec(self):
"""Lazily loads the sentence vectorizer model from disk."""
if not self.__vec_internal__:
vec_path = (self.__model_dir_path / 'sentence_vectorizer' /
'vectorizer.pkl')
if not (vec_path.exists() and vec_path.is_file()):
raise ValueError
self.__vec_internal__ = joblib.load(vec_path)
return self.__vec_internal__
@property
def __classifier_cat(self):
"""Lazily loads the sentence category models from disk."""
if not self.__classifier_cat_internal__:
cat_model_dir = self.__model_dir_path / 'sentence_categories'
if not (cat_model_dir.exists() and cat_model_dir.is_dir()):
raise ValueError
cat_models = []
for cat_model_path in cat_model_dir.glob('*.pkl'):
cat_models.append(joblib.load(cat_model_path))
self.__classifier_cat_internal__ = cat_models
return self.__classifier_cat_internal__
def __get_classifier_trust(self, fname):
"""Lazily loads the trust model from disk."""
if fname not in self.__classifier_trust_internal__:
trust_model_path = self.__model_dir_path / 'trust' / fname
if not (trust_model_path.exists() and trust_model_path.is_file()):
raise ValueError
self.__classifier_trust_internal__[fname] = joblib.load(
trust_model_path)
return self.__classifier_trust_internal__[fname]
def predict_trust(self, profile, strip_html=True):
"""Predicts the trustworthiness of a profile.
Segments the input with sentence-level granularity, returning the
probability that the profile represented by the input is perceived to
be more trustworthy compared to other profiles of similar length.
Args:
profile: An Airbnb host profile, as a string.
strip_html: Whether HTML tags in the input should be stripped. True
by default, but can be disabled for speed if the input is known
to be sanitized.
Returns:
An AirProfile.Prediction object for trustworthiness of the profile.
Raises:
ValueError: If the input is an invalid or empty string.
IOError: If the LIWC trie is not available.
"""
if not (profile and profile.strip()):
raise ValueError
if not (self.__liwc_path.exists() and self.__liwc_path.is_file()):
raise IOError
sentence_tokens = self.__preprocess(profile, strip_html)
liwc_features = self.__liwc.summarize(profile, sentence_tokens)
word_count = liwc_features['WC']
liwc_features['wc_log'] = np.log(word_count)
liwc_features['readability'] = ts.flesch_kincaid_grade(
profile.decode('utf-8'))
prediction_agg = np.empty(len(self.__classifier_cat))
for sent in sentence_tokens:
prediction_agg += np.array(
[c.predict for c in self.__classify_sentence(sent)])
for idx, cat in enumerate(FEAT_WC_CATEGORIES):
liwc_features[cat] = prediction_agg[idx]
feats = [
liwc_features[f]
for f in AirProfile.__get_trust_model_feat_cols(word_count)
]
feats_shape = np.array(feats).reshape(1, -1)
model = self.__get_classifier_trust(
AirProfile.__get_trust_model_fname(word_count))
return self.Prediction(
np.round(model.predict_proba(feats_shape)[0][1], 2),
model.predict(feats_shape)[0])
def __classify_sentence(self, tokens):
"""Classifies a sentence based on the trust category models.
Args:
tokens: A list of AirProfile.SentenceToken, corresponding to a
sentence.
Returns:
A list of AirProfile.Prediction. Each entry in the list corresponds
to the prediction made by a category model.
"""
vector = self.__vec.transform([' '.join(tokens.clean)])
# TODO(kenlimmj): This should be an ordered or keyed data structure,
# since the category classifiers are reconciled later on.
output = []
for classifier in self.__classifier_cat:
prediction = self.Prediction(
np.round(classifier.predict_proba(vector)[0], 2),
classifier.predict(vector)[0])
output.append(prediction)
return output
def predict_topics(self, profile, strip_html=True):
"""Predicts the trust evaluation topics for a profile.
Args:
profile: An Airbnb host profile, as a string.
strip_html: Whether HTML tags in the input should be stripped. True
by default, but can be disabled for speed if the input is known
to be sanitized.
Returns:
A list of prediction probabilities
Raises:
ValueError: If the input is an invalid or empty string.
"""
if not (profile and profile.strip()):
raise ValueError
sentence_tokens = self.__preprocess(profile, strip_html)
results = []
for token in sentence_tokens:
probs = np.array(
[c.prob[1] for c in self.__classify_sentence(token)])
predictions = {}
for idx, cat in enumerate(FEAT_WC_CATEGORIES):
predictions[cat] = np.round(probs[idx], 2)
results.append([' '.join(token.raw), predictions])
return results
# TODO(kenlimmj): Consider making this a decorator, since it's used by both
# `predict_topics` and `predict_trust`.
def __preprocess(self, profile, strip_html=True):
"""Cleans and tokenizes an input string for trust prediction.
Performs the follow operations, in order:
1. Strips HTML tags, if requested.
2. Removes non alphanumeric characters.
3. Converts to lowercase.
4. Tokenizes on whitespace.
5. Removes stopwords (see `constants.STOPWORDS`).
6. Lemmatizes each token.
Args:
profile: The string to be pre-processed.
strip_html: Whether HTML tags in the input should be stripped. True
by default, but can be disabled for speed if the input is known
to be sanitized.
Returns:
A list of AirProfile.SentenceToken.
"""
stripped_html = BeautifulSoup(
profile, 'lxml').get_text() if strip_html else profile
sentences = [
str(s.decode('utf-8')).translate(None, NON_ALNUM_CHARS).lower()
for s in sent_tokenize(stripped_html)
]
output = []
for sent in sentences:
tokens = sent.split()
tokens_stop = [t for t in tokens if t not in STOPWORDS]
tokens_lemm_verb = [self.__lemm_v[t] for t in tokens_stop]
tokens_lemm_noun = [self.__lemm_n[t] for t in tokens_lemm_verb]
output.append(self.__SentenceToken(tokens, tokens_lemm_noun))
return output
@staticmethod
def __get_trust_model_feat_cols(word_count):
"""Gets the LIWC features to be evaluated based on word count."""
feat_cols = FEAT_WC_LING + FEAT_WC_PSYCH + FEAT_WC_CONCERN
if word_count > 19:
feat_cols += FEAT_WC_CATEGORIES
return feat_cols
@staticmethod
def __get_trust_model_fname(word_count):
"""Gets the trust model to be used based on word count."""
if word_count <= 19:
return 'trust0.pkl'
elif word_count <= 36:
return 'trust1.pkl'
elif word_count <= 58:
return 'trust2.pkl'
elif word_count <= 88:
return 'trust3.pkl'
return 'trust4.pkl'
import logging
import random
import socket
import ssl
import string
from urllib.parse import urlparse
from cachetools import LRUCache
from pyvcloud.vcd.client import BasicLoginCredentials
from pyvcloud.vcd.client import Client
from pyvcloud.vcd.platform import Platform
from pyvcloud.vcd.vapp import VApp
from pyvcloud.vcd.vm import VM
from vsphere_guest_run.vsphere import VSphere
cache = LRUCache(maxsize=1024)
LOGGER = logging.getLogger('cse.utils')
def hex_chunks(s):
return [s[i:i + 2] for i in range(0, len(s), 2)]
def get_thumbprint(host, port):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(10)
wrappedSocket = ssl.wrap_socket(sock)
wrappedSocket.connect((host, port))
der_cert_bin = wrappedSocket.getpeercert(True)
thumb_sha1 = hashlib.sha1(der_cert_bin).hexdigest()
def __init__(self, cachesize=DEFAULT_CACHE_SIZE):
self._cache = LRUCache(maxsize=cachesize)
self._components = OrderedDict()
self._loaded_modules = set()
self.ready = False
def clear_cache(self):
self._block_cache = LRUCache(maxsize=self._cache_size)
self._block_cache_hits = 0
self._block_cache_misses = 0
if '__dict__' in cls.__dict__:
d = cls.__dict__['__dict__']
if inspect.isgetsetdescriptor(d) or inspect.ismemberdescriptor(
d):
size += _get_size(obj.__dict__, seen)
break
if isinstance(obj, dict):
size += sum((_get_size(v, seen) for v in obj.values()))
size += sum((_get_size(k, seen) for k in obj.keys()))
elif hasattr(obj, '__iter__') and not isinstance(obj,
(str, bytes, bytearray)):
size += sum((_get_size(i, seen) for i in obj))
return size
LRU_CACHE = LRUCache(os.environ.get('TEXTACY_MAX_CACHE_SIZE', 2147483648),
getsizeof=_get_size)
""":class:`cachetools.LRUCache`: Least Recently Used (LRU) cache for loaded data.
The max cache size may be set by the `TEXTACY_MAX_CACHE_SIZE` environment variable,
where the value must be an integer (in bytes). Otherwise, the max size is 2GB.
"""
def clear():
"""Clear textacy's cache of loaded data."""
global LRU_CACHE
LRU_CACHE.clear()
@cached(LRU_CACHE, key=functools.partial(hashkey, 'spacy'))
def load_spacy(name, disable=None):
def __init__(self):
self._cache = LRUCache(256)
DEFAULT_ONLINE_LIMIT = 2
DEFAULT_TIME_SPAN = 'MINUTE'
DEFAULT_STAT_LIMIT = 5
HEADERS = {
'Content-Type': 'application/json',
'Accept': 'application/vnd.pagerduty+json;version=2',
'Authorization': 'Token token={token}'.format(token=PD_API_KEY)
}
COLORS = [
"#f49d41", "#d6f441", '#41f449', '#41f4d3', '#4188f4', '#be41f4', '#f4415b'
]
cache = LRUCache(maxsize=30)
app = Flask(__name__)
def headers_filter(headers_list):
return {
key: value
for key, value in HEADERS.iteritems() if key in headers_list
}
def list_oncalls():
oncall_url = 'https://api.pagerduty.com/oncalls'
response = requests.get(oncall_url,
headers=headers_filter(['Accept',
def clear_cache(self):
self._block_cache = LRUCache(maxsize=self._cache_size)
self._cache_hit_count = 0
self._cache_miss_count = 0
def __init__(self, backend, sort_order):
super(PandoraLibraryProvider, self).__init__(backend)
self.sort_order = sort_order.lower()
self.pandora_station_cache = StationCache(self, maxsize=5)
self.pandora_track_cache = LRUCache(maxsize=10)
class KeycloakAuthHandler(auth.AuthHandler):
def authenticate(self, req):
if 'X-Auth-Token' not in req.headers:
msg = _("Auth token must be provided in 'X-Auth-Token' header.")
LOG.error(msg)
raise exc.UnauthorizedException(message=msg)
access_token = req.headers.get('X-Auth-Token')
try:
decoded = jwt.decode(access_token,
algorithms=['RS256'],
verify=False)
except Exception as e:
msg = _("Token can't be decoded because of wrong format %s")\
% str(e)
LOG.error(msg)
raise exc.UnauthorizedException(message=msg)
# Get user realm from parsed token
# Format is "iss": "http://<host>:<port>/auth/realms/<realm_name>",
__, __, realm_name = decoded['iss'].strip().rpartition('/realms/')
audience = decoded.get('aud')
# Get roles from parsed token
roles = ','.join(decoded['realm_access']['roles']) \
if 'realm_access' in decoded else ''
# NOTE(rakhmerov): There's a special endpoint for introspecting
# access tokens described in OpenID Connect specification but it's
# available in KeyCloak starting only with version 1.8.Final so we have
# to use user info endpoint which also takes exactly one parameter
# (access token) and replies with error if token is invalid.
user_info_endpoint_url = CONF.keycloak_oidc.user_info_endpoint_url
if user_info_endpoint_url.startswith(('http://', 'https://')):
self.send_request_to_auth_server(url=user_info_endpoint_url,
access_token=access_token)
else:
public_key = self.get_public_key(realm_name)
keycloak_iss = None
try:
if CONF.keycloak_oidc.keycloak_iss:
keycloak_iss = CONF.keycloak_oidc.keycloak_iss % realm_name
jwt.decode(access_token,
public_key,
audience=audience,
issuer=keycloak_iss,
algorithms=['RS256'],
verify=True)
except Exception:
LOG.exception('The request access token is invalid.')
raise exc.UnauthorizedException()
req.headers["X-Identity-Status"] = "Confirmed"
req.headers["X-Project-Id"] = realm_name
req.headers["X-Roles"] = roles
@staticmethod
def get_system_ca_file():
"""Return path to system default CA file."""
# Standard CA file locations for Debian/Ubuntu, RedHat/Fedora,
# Suse, FreeBSD/OpenBSD, MacOSX, and the bundled ca.
ca_path = [
'/etc/ssl/certs/ca-certificates.crt',
'/etc/pki/tls/certs/ca-bundle.crt', '/etc/ssl/ca-bundle.pem',
'/etc/ssl/cert.pem', '/System/Library/OpenSSL/certs/cacert.pem',
requests.certs.where()
]
for ca in ca_path:
LOG.debug("Looking for ca file %s", ca)
if os.path.exists(ca):
LOG.debug("Using ca file %s", ca)
return ca
LOG.warning("System ca file could not be found.")
@cached(LRUCache(maxsize=32))
def get_public_key(self, realm_name):
keycloak_key_url = (CONF.keycloak_oidc.auth_url +
CONF.keycloak_oidc.public_cert_url % realm_name)
response_json = self.send_request_to_auth_server(keycloak_key_url)
keys = response_json.get('keys')
if not keys:
raise exc.MistralException(
'Unexpected response structure from the keycloak server.')
public_key = jwt_algos.RSAAlgorithm.from_jwk(json.dumps(keys[0]))
return public_key
def send_request_to_auth_server(self, url, access_token=None):
certfile = CONF.keycloak_oidc.certfile
keyfile = CONF.keycloak_oidc.keyfile
cafile = CONF.keycloak_oidc.cafile or self.get_system_ca_file()
insecure = CONF.keycloak_oidc.insecure
verify = None
if urllib.parse.urlparse(url).scheme == "https":
verify = False if insecure else cafile
cert = (certfile, keyfile) if certfile and keyfile else None
headers = {}
if access_token:
headers["Authorization"] = "Bearer %s" % access_token
try:
resp = requests.get(url, headers=headers, verify=verify, cert=cert)
except requests.ConnectionError:
msg = _("Can't connect to the keycloak server with address '%s'."
) % url
LOG.exception(msg)
raise exc.MistralException(message=msg)
if resp.status_code == 401:
LOG.warning(
"HTTP response from OIDC provider:"
" [%s] with WWW-Authenticate: [%s]", pprint.pformat(resp.text),
resp.headers.get("WWW-Authenticate"))
else:
LOG.debug("HTTP response from the OIDC provider: %s",
pprint.pformat(resp.json()))
resp.raise_for_status()
return resp.json()
#!/usr/bin/python
# -*- coding: utf-8 -*-
from cachetools import LRUCache
from kids.cache import cache
from http_cache import http_get
# examples
# https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=PMC3039489&resulttype=core&format=json&tool=oadoi
# https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=PMC3606428&resulttype=core&format=json&tool=oadoi
# https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=10.1093/jisesa/iex068&resulttype=core&format=json&tool=oadoi
@cache(use=LRUCache(maxsize=32))
def query_pmc(query_text):
if not query_text:
return None
url_template = u"https://www.ebi.ac.uk/europepmc/webservices/rest/search?query={}&resulttype=core&format=json&tool=oadoi"
url = url_template.format(query_text)
r = http_get(url)
data = r.json()
result_list = data["resultList"]["result"]
return result_list
