def is_token_revoked(decoded_token):
"""
Checks if the given token is revoked or not. Because we are adding all the
tokens that we create into this database, if the token is not present
in the database we are going to consider it revoked, as we don't know where
it was created.
:param decoded_token:
"""
jti = decoded_token['jti']
cache = Cache()
if cache.get('token_' + jti) is None:
return True
token = eval(cache.get('token_' + jti).decode("utf-8"))
if token is not None:
return token['revoked']
else:
db = Mysql()
token = db.execute_select(
"SELECT * FROM `user_session` WHERE `jti` = %s", (jti, ))
if len(token) > 0:
return token[0]['revoked']
else:
return True
def get_instruments(self, instrument_map, get_params):
ids_to_retrieve = set()
for url in get_params:
# Check if instrument is in the cache before querying Robinhood
data = Cache.get(url)
if data:
instrument = self.instrument_class()(**data)
self.set_instrument(instrument_map, instrument)
else:
id = get_params[url]
ids_to_retrieve.add(id)
# Perform a batch query for the rest of the ids
if ids_to_retrieve:
retrieved_instruments = self.instrument_class().search(
ids=ids_to_retrieve)
for instrument in retrieved_instruments:
self.set_instrument(instrument_map, instrument)
# Cache results of both a resource get and a search query
Cache.set(instrument.url, instrument.data)
search_url = self.build_search_url(
self.get_search_params(instrument.identifier()))
Cache.set(search_url, {'results': [instrument.data]})
def add_token_to_database(encoded_token, identity_claim):
"""
Adds a new token to the cache and database. It is not revoked when it is added.
:param encoded_token:
:param identity_claim:
"""
# Prepare
decoded_token = decode_token(encoded_token)
jti = decoded_token['jti']
token_type = decoded_token['type']
user_identity = decoded_token[identity_claim]
revoked = False
expires = _epoch_utc_to_datetime(decoded_token['exp'])
# Save
cache = Cache()
cache.set(
'token_' + jti,
str({
'jti': jti,
'token_type': token_type,
'user_identity': user_identity,
'revoked': revoked,
'expires': expires
}))
db = Mysql()
db.execute(
"INSERT INTO `user_session` (`user_id`,`jti`,`token_type`,`user_identity`,`revoked`,`expires`) VALUES (%s,%s,%s,%s,%s,%s)",
(user_identity, jti, token_type, user_identity, revoked, expires))
def test_stock_handler(self):
identifier = 'AMZN'
instruments = self.stock_handler.find_instruments(identifier)
self.assertTrue(identifier in instruments)
instrument = instruments[identifier]
self.validate_stock(identifier, instrument)
self.assertTrue(instrument.url in instruments)
instrument = instruments[instrument.url]
self.validate_stock(identifier, instrument)
# Test that the cache was set correctly
cached_data = Cache.get(instrument.url)
self.assertIsNotNone(cached_data)
instrument_from_cache = Stock(**cached_data)
self.validate_stock(identifier, instrument_from_cache)
cached_data = Cache.get(Stock.search_url(symbol=instrument.symbol))
self.assertIsNotNone(cached_data)
self.assertTrue('results' in cached_data)
self.assertTrue(len(cached_data['results']) == 1)
instrument_from_cache = Stock(**cached_data['results'][0])
self.validate_stock(identifier, instrument_from_cache)
# Test that we retrieve from the cache correctly
start_time = time()
instruments = self.stock_handler.find_instruments(identifier)
self.assertTrue(identifier in instruments)
instrument = instruments[identifier]
self.validate_stock(identifier, instrument)
self.assertTrue(time() - start_time < 0.05, "Cached entry was not used")
def search_instruments(self, instrument_map, search_params):
search_jobs = {}
# Initiate a thread pool for these requests.
# Using a shared thread pool can cause hanging
# when using a multi-process runner such as uwsgi.
with thread_pool(10) as pool:
for identifier in search_params:
instrument = None
params = search_params[identifier]
search_url = self.build_search_url(params)
data = Cache.get(search_url)
if data:
if 'results' in data:
cached_instruments = [
self.instrument_class()(**d)
for d in data['results']
]
if len(cached_instruments) > 1:
cached_instruments = self.filter_results(
cached_instruments, params)
if len(cached_instruments) == 1:
instrument = cached_instruments[0]
else:
instrument = self.instrument_class()(**data)
if instrument:
self.set_instrument(instrument_map, instrument, identifier)
else:
search_jobs[identifier] = pool.call(
self.instrument_class().search, **params)
for identifier in search_jobs:
params = search_params[identifier]
search_job = search_jobs[identifier]
retrieved_instruments = search_job.get()
search_url = self.build_search_url(params)
# Cache results for the search query
Cache.set(search_url,
{'results': [i.data for i in retrieved_instruments]})
matching_instruments = self.filter_results(retrieved_instruments,
params)
if len(matching_instruments) == 0:
raise NotFoundException("No {}s found for {}".format(
self.TYPE, identifier))
elif len(matching_instruments) > 1:
raise Exception(
"Multiple possible {}s found for {}, could not select a unique one"
.format(self.TYPE, identifier))
instrument = matching_instruments[0]
self.set_instrument(instrument_map, instrument, identifier)
# Cache results for the resource query
Cache.set(instrument.url, instrument.data)
def __init__(self, dataFolder, sessionName):
self.dataFolder = dataFolder
self.textFileReader = textFile.TextFile()
self.wordList = Wordlist()
self.sessionName = sessionName
self.logger = logger.Logger()
self.cache = Cache()
self.logger.log("Starting a new SVD Session: " + sessionName)
def start_backup(self):
"""
Start backup.
"""
self.cache = Cache(self.alias)
self.dbx = dropbox.Dropbox(self.config.get('token'))
# Get files recursively
self.get_children()
def start_backup(self):
"""Start backup."""
# Set cache
self.cache = Cache(self.alias)
# Backup
self.get_children()
# Cleanup
self.cleanup()
def test_option_handler(self):
identifier = 'AAPL250.0C@01/17/20'
instruments = self.option_handler.find_instruments(identifier)
self.assertTrue(identifier in instruments)
instrument = instruments[identifier]
self.validate_option(identifier, instrument)
self.assertTrue(instrument.url in instruments)
instrument = instruments[instrument.url]
self.validate_option(identifier, instrument)
# Test that the cache was set correctly
cached_data = Cache.get(instrument.url)
self.assertIsNotNone(cached_data)
instrument_from_cache = Option(**cached_data)
self.validate_option(identifier, instrument_from_cache)
cached_data = Cache.get(Option.search_url(
chain_symbol=instrument.chain_symbol,
strike_price=instrument.strike_price,
type=instrument.type,
expiration_date=instrument.expiration_date
))
self.assertIsNotNone(cached_data)
self.assertTrue('results' in cached_data)
self.assertTrue(len(cached_data['results']) > 0)
option_data = None
expiration_date_str = instrument.expiration_date.strftime("%Y-%m-%d")
for d in cached_data['results']:
if d['expiration_date'] == expiration_date_str:
option_data = d
self.assertIsNotNone(option_data, "No option matching expiration {} was cached".format(instrument.expiration_date))
instrument_from_cache = Option(**option_data)
self.validate_option(identifier, instrument_from_cache)
# Test that we retrieve from the cache correctly
start_time = time()
instruments = self.option_handler.find_instruments(identifier)
self.assertTrue(identifier in instruments)
instrument = instruments[identifier]
self.validate_option(identifier, instrument)
self.assertTrue(time() - start_time < 0.05, "Cached entry was not used")
def prune_database():
"""
Delete tokens that have expired from the database.
How (and if) you call this is entirely up you. You could expose it to an
endpoint that only administrators could call, you could run it as a cron,
set it up with flask cli, etc.
"""
now = datetime.now()
db = Mysql()
expired_tokens = db.execute_select(
"SELECT * FROM `user_session` WHERE `expires` < NOW()", (now, ))
for expired_token in expired_tokens:
cache = Cache()
cache.delete('token_' + expired_token['jti'])
db.execute("DELETE FROM `user_session` WHERE `expires` < %s", (now, ))
def test_clean(self):
cache = Cache()
# Fill cache
for i in range(len(cache), cache.limit):
cache[i] = 'Anything'
self.assertEqual(100, len(cache))
# Overload cache
cache['TooMany'] = 2
self.assertEqual(cache.limit - int(cache.limit * Cache.REMOVE_PCT) + 1,
len(cache))
def fetch_ornament_for_cart(cls, ornament_id):
cart_item_byte = Cache.get(key='cart_item')
cart_item = json.loads(
cart_item_byte.decode()) if cart_item_byte else {}
if cart_item.get(ornament_id):
raise ItemAlreadyExistInCart(message="Item already exist in cart")
ornament = Ornament.get_ornament_by_id(_id=ornament_id)
category = Category.get_category_by_id(
category_id=ornament.category_id)
ornament_dict = dict(id=ornament.id,
name=ornament.name,
weight=ornament.weight,
wastage=ornament.wastage,
making_charge=ornament.making_charge,
category_id=ornament.category_id,
category_name=category.name,
category_material=category.material)
cart_item[ornament_id] = ornament_dict
Cache.set(key="cart_item", value=json.dumps(cart_item))
return ornament_dict
def delete_tokens(token_ids_list, user_identity):
"""
Delete tokens.
:param tokens: List of tokens
:type tokens: list
"""
# Delete from cache
db = Mysql()
tokens = db.execute_select(
"SELECT `jti` FROM `user_session` WHERE `user_id` = %s AND `id` IN (%s)",
(user_identity, token_ids_list))
if len(tokens) > 0:
for token in tokens:
cache = Cache()
cache.delete('token_' + token['jti'])
# Delete from DB
db.execute(
"DELETE FROM `user_session` WHERE `user_id` = %s AND `id` IN (%s)",
(user_identity, token_ids_list))
def revoke_token(token_id, user_identity):
"""
Revokes the given token. Raises a TokenNotFound error if the token does not exist in the database.
:param token_id: Id of the token
:param user_identity:
:type token_id: int
"""
db = Mysql()
token = db.execute_select(
"SELECT * FROM `user_session` WHERE `user_identity` = %s AND `id` = %s",
(user_identity, token_id))
if len(token) > 0:
token = token[0]
token['revoked'] = 1
cache = Cache()
cache.set('token_' + token['jti'], str(token))
db.execute(
"UPDATE `user_session` SET `revoked` = %s WHERE `user_identity` = %s AND `id` = %s",
(1, user_identity, token_id))
else:
return False
def exec_tool(csv_file, is_preprocessed, merge_fill_blanks, delete, entity):
app_cache = Cache()
app_cache.load_cache_file()
def terminateProcess(signalNumber, frame):
print('SIG-', signalNumber,
': Dumping cache, then terminating process...')
app_cache.write_cache_file()
sys.exit()
# Catch kill signals and dump cache before exiting
signal.signal(signal.SIGTERM, terminateProcess)
signal.signal(signal.SIGINT, terminateProcess)
try:
main_tool_process(csv_file, is_preprocessed, merge_fill_blanks, delete,
entity, app_cache)
except:
app_cache.write_cache_file()
raise
else:
app_cache.write_cache_file()
class SVD:
def __init__(self, dataFolder, sessionName):
self.dataFolder = dataFolder
self.textFileReader = textFile.TextFile()
self.wordList = Wordlist()
self.sessionName = sessionName
self.logger = logger.Logger()
self.cache = Cache()
self.logger.log("Starting a new SVD Session: " + sessionName)
def getOneHotMatrix(self, data, wordlist):
nr_cols = len(wordlist)
self.logger.startLongRunningLog(len(data), 10000)
counter = 0
ts = time.time()
matrix = lil_matrix((len(data), len(wordlist)))
for session in data:
row = np.zeros(nr_cols)
for device in session:
row[wordlist[device]] += 1
matrix[counter] = row
counter += 1
self.logger.log("")
self.logger.endLongRunningLog()
print("matrix: {} r1: {} c1: {}".format(matrix.shape, matrix[0],
matrix[0, 0]))
return matrix
def getOneHotSquareMatrixSession(self, data, wordlist):
nr_cols = len(wordlist)
matrix = lil_matrix((len(wordlist), len(wordlist)))
for dataRow in data:
prev = None
row = np.zeros(nr_cols)
sessionItems = []
for item in dataRow:
for sessionItem in sessionItems:
matrix[wordlist[sessionItem], wordlist[item]] += 1
matrix[wordlist[item], wordlist[sessionItem]] += 1
sessionItems.append(item)
return matrix
def getOneHotMatrixFromGraph(self, data, keys):
matrix = []
size = len(keys)
matrix = lil_matrix((size, size))
self.logger.startLongRunningLog(size, 100)
counter = 0
for i in range(size):
row = np.zeros(size)
for j in range(size):
if keys[i] in data and keys[j] in data[keys[i]]:
row[j] = data[keys[i]][keys[j]]['weight']
## Instead of zero add avg as default value ##
'''
rowWithMean = np.full(size, np.mean(row))
for j in range(size):
if keys[i] in data and keys[j] in data[keys[i]]:
rowWithMean[j] = data[keys[i]][keys[j]]['weight']
matrix[counter] = rowWithMean
'''
matrix[counter] = row
counter += 1
self.logger.log("Building graph matrix...")
self.logger.endLongRunningLog()
return matrix
def createOneHotFromName(self, items):
row = np.zeros(len(self.wl))
for item in items:
row[self.wl[item]] += 1
return row
def getConceptFromOneHot(self, vector):
print(self.vt.shape)
print(self.v.shape)
print(vector.reshape((1, -1)).shape)
return np.inner(vector.reshape((1, -1)), self.vt)
#return np.dot(vector.reshape((1, -1)), self.v)
def cosineSimilarity(self, v1, v2, n_cols=None, applySigma=True):
if applySigma:
return cosine_similarity([np.multiply(v1, self.sigma)],
[np.multiply(v2, self.sigma)])[0][0]
else:
return cosine_similarity([v1], [v2])[0][0]
def exportSimilarityUnidirectionalSquareMatrix(self, filepath):
size = len(self.u)
self.logger("Exporting nodes...")
with open(filepath + "/nodes.csv", "w") as text_file:
text_file.write("Id,Label\n")
for i in range(size):
text_file.write("{},{}\n".format(i, self.reversedWl[i]))
self.logger.startLongRunningLog((size * (size - 1) / 2), 10000)
with open(filepath + "/edges.csv", "w") as text_file:
text_file.write("Source,Target,Weight\n")
for i in range(size):
concepts_i = self.u[i]
for j in range(i + 1, size):
concepts_j = self.u[j]
sim = self.cosineSimilarity(concepts_i, concepts_j)
if (sim > 0.5):
text_file.write("{},{},{}\n".format(i, j, sim))
self.logger("Saving edges...")
def getMostSimilarInU(self, concepts, threshold=0.9):
best = 0
bestRow = None
session = -1
counter = 0
items = []
uWithSigma = np.matmul(self.u, np.diag(self.sigma))
similarities = cosine_similarity([np.multiply(concepts, self.sigma)],
uWithSigma)[0]
for similarity in similarities:
if similarity > best:
best = similarity
bestRow = -1
session = self.reversedWl[counter]
if similarity > threshold:
#print("{} SIM:{}".format(self.reversedWl[counter], similarity))
items.append({
'item': self.reversedWl[counter],
'score': similarity
})
counter += 1
return session, bestRow, best, items
def getHighestIndex(self, vector, nr):
highest = []
for i in range(len(vector)):
highest.append({'index': i, 'val': (vector[i] * self.sigma[i])})
newlist = sorted(highest, key=lambda k: k['val'], reverse=True)
indexes = []
for i in range(nr):
indexes.append(newlist[i]['index'])
return indexes
def selectMaxFromVFromColumns(self, columns):
best = -1
bestItem = None
counter = 0
for item in self.v:
score = 0
for c in columns:
score += item[c] * self.sigma[c]
if score > best:
best = score
bestItem = self.reversedWl[counter]
counter += 1
return bestItem, score
def getWordlist(self):
return self.wl
def getSigma(self):
return self.sigma.tolist()
def visualize(self):
x1 = self.u[:, 0]
y1 = self.u[:, 1]
z1 = self.u[:, 2]
fig = plt.figure()
ax = Axes3D(fig)
ax.scatter(x1, y1, z1)
plt.show()
'''
x1 = np.multiply(self.u[:,0], self.sigma[0])
y1 = np.multiply(self.u[:,1], self.sigma[1])
z1 = np.multiply(self.u[:,2], self.sigma[2])
x2 = np.multiply(self.v[:,0], self.sigma[0])
y2 = np.multiply(self.v[:,1], self.sigma[1])
z2 = np.multiply(self.v[:,2], self.sigma[2])
'''
'''
x1 = self.u[:,0]
y1 = self.u[:,1]
z1 = self.u[:,2]
x2 = self.v[:,0]
y2 = self.v[:,1]
z2 = self.v[:,2]
fig = plt.figure()
ax = Axes3D(fig)
ax.scatter(x1, y1, z1)
ax.scatter(x2, y2, z2)
plt.show()
'''
'''
y = self.sigma
x = range(len(self.sigma))
plt.plot(x, y)
plt.show()
'''
def getItemFromV(self, itemKey):
if itemKey in self.wl:
index = self.wl[itemKey]
return np.multiply(self.v[index], self.sigma).tolist(), True
else:
return None, False
def runSvdOnJsonGraph(self, n_components, n_iter):
self.logger.log("Loading data...")
self.data = self.textFileReader.read_json(self.dataFolder)
self.logger.log("Generating wordlist...")
items, wl, reversedWl = self.wordList.getWordlistFromGraph(self.data)
self.wl = wl
self.reversedWl = reversedWl
matrix, wasCached = self.cache.lazyCache(
self.sessionName + "device_device_adjecency.matrix",
self.getOneHotMatrixFromGraph, {
'data': self.data,
'keys': items
})
if wasCached:
self.logger.log("Loaded matrix from cache")
self._runSvd(matrix, n_components, n_iter)
def _runSvd(self, matrix, n_components, n_iter):
#Load the factorization from disk
self.u, uWasCached = self.cache.loadNPIfExists(self.sessionName +
"u.bin")
self.sigma, sigmaWasCached = self.cache.loadNPIfExists(
self.sessionName + "sigma.bin")
self.vt, vtWasCached = self.cache.loadNPIfExists(self.sessionName +
"vt.bin")
self.v, vWasCached = self.cache.loadNPIfExists(self.sessionName +
"v.bin")
#If any of the cached matrices was missing
if not uWasCached or not sigmaWasCached or not vtWasCached and not vWasCached:
self.logger(
"Factorization was not present, calculating... (Might take a while)"
)
self.logger(
"Fitting the randomized_svd with {} iterations and {} components"
.format(n_iter, n_components))
U, Sigma, VT = randomized_svd(matrix,
n_components=n_components,
n_iter=n_iter)
self.sigma = Sigma
self.u = U
self.vt = VT
self.v = np.transpose(self.vt)
self.u.dump(self.sessionName + "u.bin")
self.sigma.dump(self.sessionName + "sigma.bin")
self.vt.dump(self.sessionName + "vt.bin")
self.v.dump(self.sessionName + "v.bin")
else:
self.logger("Loaded factorization from disk")
self.logger(matrix.shape)
self.logger(self.vt.shape)
self.logger(self.sigma.shape)
self.logger(self.u.shape)
self.logger(self.v.shape)
def runSvdOnCsv(self, n_components, n_iter):
self.logger.log("Loading data...")
self.data = self.textFileReader.read_folder(self.dataFolder)
self.logger.log("Generating wordlist...")
items, wl, reversedWl = self.wordList.getWordlist(self.data)
self.wl = wl
self.reversedWl = reversedWl
matrix, wasCached = self.cache.lazyCache(
self.sessionName + "session_device_adjecency.matrix",
self.getOneHotSquareMatrixSession, {
'data': self.data,
'wordlist': wl
})
if wasCached:
self.logger.log("Loaded matrix from cache")
self._runSvd(matrix, n_components, n_iter)
def runSvdOnCsvMatrix(self, n_components, n_iter):
self.logger.log("Loading data...")
data = self.textFileReader.read_folder(self.dataFolder, '*.csv')
m = np.matrix(data[1:], dtype=np.float32)
m = np.delete(m, [0], axis=1)
matrix = normalize(m, axis=1, norm='max')
U, Sigma, VT = randomized_svd(matrix,
n_components=n_components,
n_iter=n_iter)
self.U = U
self.Sigma = Sigma
self.VT = VT
def request(cls, resource_url, **params):
request_url = ApiResource.__request_url(resource_url, **params)
data = None
if ApiResource.enable_mock and request_url in ApiResource.mock_results:
# Load the mocked value
data = ApiResource.mock_results[request_url]
return data
if cls.enable_cache:
# Check if we have a cache hit first
data = Cache.get(request_url)
if data:
return data
if ApiResource.enable_mock:
# We have not mocked out a request for this resource, raise an error
raise NotFoundException(f"Mocking is currently enabled, but Robinhood request has not been mocked: {request_url}")
headers = {}
if cls.authenticated:
access_token = ApiResource.authenticate()
headers['Authorization'] = 'Bearer ' + access_token
attempts = 3
while True:
attempts -= 1
try:
start_time = time()
#print_req(request_url)
response = requests.get(request_url, headers=headers)
except requests.exceptions.ConnectionError:
# Happens occasionally, retry
if attempts > 0:
logger.warn("Warning: Connection error, retrying")
else:
raise ApiInternalErrorException(0, "Repeated connection errors when trying to call Robinhood")
continue
if response.status_code == 200:
data = response.json()
if cls.enable_cache:
# Cache response. Only successful calls are cached.
Cache.set(request_url, data, cls.cache_timeout)
return data
elif response.status_code == 400:
message = "{} (request URL: {})".format(response.text, request_url)
raise ApiBadRequestException(message)
elif response.status_code == 401:
if cls.authenticated:
if attempts > 0:
# Credentials may have expired, try reauthenticating
access_token = ApiResource.authenticate(force_reauth=True)
headers['Authorization'] = 'Bearer ' + access_token
continue
else:
raise ApiUnauthorizedException("Authentication credentials were not accepted")
else:
raise ApiUnauthorizedException("This API endpoint requires authentication: {}".format(cls.endpoint_path))
elif response.status_code == 403:
if attempts > 0:
# Credentials may have expired, try reauthenticating
access_token = ApiResource.authenticate(force_reauth=True)
headers['Authorization'] = 'Bearer ' + access_token
continue
else:
raise ApiForbiddenException("Not authorized to access this resource: {}".format(request_url))
elif response.status_code == 404:
return None
elif response.status_code > 500:
# Internal server error, retry if possible
if attempts <= 0:
raise ApiInternalErrorException(response.status_code, response.text)
else:
raise ApiCallException(response.status_code, response.text)
def test_cache(tmpdir):
cache = Cache(tmpdir)
cached = cache_decorator(cache)
num_calls = 0
@cached
def get_num_calls():
"""This function returns number of times it was called,
so we can determine if the result was restored from cache
or was it recalculated.
"""
nonlocal num_calls
num_calls += 1
return num_calls
# first time it should be one
assert get_num_calls() == 1
# and second time, again: one (as long as cache system works)
assert get_num_calls() == 1
# test caching with args
last_result = 1
def accumulative_mul(a):
"""Mul a*b, where b is the result of the previous multiplication,
or one if there was no previous multiplication.
"""
nonlocal last_result
last_result *= a
return last_result
assert accumulative_mul(5) == 5
assert accumulative_mul(5) == 5 * 5
last_result = 1
cached_accumulative_mul = cached(accumulative_mul)
assert cached_accumulative_mul(5) == 5
assert cached_accumulative_mul(5) == 5
last_result = 10 # this should change nothing
assert cached_accumulative_mul(5) == 5
# unless the cache is deleted
cached_accumulative_mul.clean(5)
assert cached_accumulative_mul(5) == 50
assert cached_accumulative_mul(5) == 50
# handle keywords:
base = 5
@cached
def calc(a=2):
return base**a
assert calc() == 25
assert calc(a=1) == 5
base = 1
assert calc() == 25
assert calc(a=1) == 5
assert calc(a=2) == 1
elif name == 'get-back':
return choose_best_match(ontology, term, history[-1], history[:-1])
elif name:
node = candidates[name][0]
return node
def simplify(name):
return name.lower().replace(' type ', ' ').replace(' form ', ' ').replace(',', '')
def compare(term, name):
return ratio(simplify(term), simplify(name))
cache = Cache('.ontology_cache')
class Ontology:
def __init__(self, obo_path):
self.ontology = obonet.read_obo(obo_path)
# initialize cached propertry with side effects
self.by_name
@property
@lru_cache()
def by_name(self):
nodes_by_name = {}
def __init__(self, dataFolder):
self.dataFolder = dataFolder
self.textFileReader = textFile.TextFile()
self.cache = Cache()
self.calcDistribution()
def authenticate(force_reauth=False):
if ApiResource.enable_mock:
# We are about to make an authentication request, which
# should not occur when we are in mock mode
raise Exception("Attempting to make authentication request when mocking enabled")
if not (ApiResource.username and ApiResource.password and ApiResource.device_token and ApiResource.oauth_client_id):
raise RobinhoodCredentialsException("Attempting to call authenticated endpoint, but one or more Robinhood credentials are missing for this server.")
# If authentication has already failed, do not try again
if ApiResource.permanent_auth_failure:
raise ApiResource.permanent_auth_failure
# Use locking to make sure that we are not trying to authenticate
# from several thrads at once
ApiResource.auth_lock.acquire()
try:
# We should check the cache's value before our local instance,
# as another process may have already reauthenticated and set the value
access_token = Cache.get('auth_access_token') or ApiResource.auth_access_token
auth_expiration = Cache.get('auth_expiration') or ApiResource.auth_expiration
if not force_reauth:
if access_token and auth_expiration and datetime.now() < auth_expiration:
return access_token
refresh_token = Cache.get('auth_refresh_token') or ApiResource.auth_refresh_token
attempts = 3
while True:
attempts -= 1
if refresh_token and not force_reauth:
data = {
'grant_type': 'refresh_token',
'refresh_token': refresh_token,
'client_id': ApiResource.oauth_client_id,
'device_token': ApiResource.device_token
}
else:
data = {
'grant_type': 'password',
'expires_in': AUTH_DURATION.total_seconds(),
'username': ApiResource.username,
'password': ApiResource.password,
'client_id': ApiResource.oauth_client_id,
'device_token': ApiResource.device_token,
'scope': 'internal'
}
try:
response = requests.post(AUTH_INFO['url'], headers=AUTH_INFO['headers'], data=json.dumps(data))
except requests.exceptions.ConnectionError as e:
# Occasional error, retry if possible
if attempts > 0:
sleep(1)
continue
raise e
if response.status_code == 200:
data = response.json()
access_token = data['access_token']
refresh_token = data['refresh_token']
auth_refresh_duration = AUTH_DURATION / 2
auth_expiration = datetime.now() + auth_refresh_duration
# Set the access token to expire early to allow the refresh token to be utilized
Cache.set('auth_access_token', access_token, auth_refresh_duration.total_seconds())
Cache.set('auth_refresh_token', refresh_token, AUTH_DURATION.total_seconds())
Cache.set('auth_expiration', auth_expiration)
# Define in local memory (we may not have a cache available to us)
ApiResource.auth_expiration = auth_expiration
ApiResource.auth_access_token = access_token
ApiResource.auth_refresh_token = refresh_token
return access_token
if response.status_code >= 500:
if attempts > 0:
sleep(1)
continue
raise ApiInternalErrorException(response.status_code, response.text)
if response.status_code == 429:
raise ApiThrottledException(response.text)
# If we reach this point we've likely received an authentication error
# Remove cached credentials and force reauthentication
force_reauth = True
Cache.delete('auth_access_token')
Cache.delete('auth_refresh_token')
Cache.delete('auth_expiration')
ApiResource.auth_access_token = None
ApiResource.auth_refresh_token = None
ApiResource.auth_expiration = None
if response.status_code == 401:
try:
response_data = response.json()
if 'error' in response_data and response_data['error'] == 'invalid_grant':
# Refresh token is no longer valid
# Remove it and re-attempt authentication with username/password
refresh_token = None
continue
except ValueError:
# Response is not valid JSON, let remaining error logic handle it
pass
# Error codes other than these are considered to be permanent errors,
# due to invalid credentials or other issues with user-provided credentials.
if response.status_code == 403:
error = ApiForbiddenException("Authentication is required for this endpoint, but credentials are expired or invalid.")
else:
request_details = "\n\tRequest URL: {}\n\tRequest headers: {}\n\tRequest data: {}".format(
auth_url, auth_request_headers, data)
error = ApiCallException(response.status_code, response.text + request_details)
ApiResource.permanent_auth_failure = error
raise error
finally:
ApiResource.auth_lock.release()
class Distribution:
dataFolder = None
textFileReader = None
cache = None
wordlist = None
def __init__(self, dataFolder):
self.dataFolder = dataFolder
self.textFileReader = textFile.TextFile()
self.cache = Cache()
self.calcDistribution()
def _readDataFromDisk(self):
return self.textFileReader.read_folder(self.dataFolder)
def _readSortedList(self, data):
items = {}
for row in data:
for col in row:
if col not in items:
items[col] = 0
items[col] += 1
itemList = []
for item in items.keys():
itemList.append({'item': item, 'val': items[item]})
return sorted(itemList, key=lambda k: k['val'], reverse=True)
def _readWordList(self):
wordlist = {}
for i in range(len(self.sortedList)):
item = self.sortedList[i]
wordlist[item['item']] = i
return wordlist
def calcDistribution(self):
data, wasCached = self.cache.lazyCache("distribution.pkl",
self._readDataFromDisk, {})
if wasCached:
print("Loaded distribution from cache")
items = {}
for row in data:
for col in row:
if col not in items:
items[col] = 0
items[col] += 1
itemList = []
for item in items.keys():
itemList.append({'item': item, 'val': items[item]})
self.sortedList, sortedWasCached = self.cache.lazyCache(
"sortedDist.pkl", self._readSortedList, {'data': data})
if sortedWasCached:
print("Loaded sortedList from cache")
self.wordlist, wordListWasCached = self.cache.lazyCache(
"wordListDist.pkl", self._readWordList, {})
if wordListWasCached:
print("Loaded wordlist from cache")
def getDistribution(self):
return self.sortedList, self.wordlist
class Dropbox(Strategy):
"""Backup strategy for Dropbox."""
NAME = 'Dropbox'
TYPE = 'dropbox'
cache = None
dbx = None
def start_backup(self):
"""
Start backup.
"""
self.cache = Cache(self.alias)
self.dbx = dropbox.Dropbox(self.config.get('token'))
# Get files recursively
self.get_children()
def get_children(self, path=''):
"""
Get items in directory.
@param string path (optional)
"""
res = self.dbx.files_list_folder(path=path)
for entry in res.entries:
if self.check_if_excluded(entry.path_display):
continue
elif isinstance(entry, dropbox.files.FolderMetadata):
self.get_children(entry.path_display)
else:
destination = os.path.join(self.backup_path,
entry.path_display.strip('/'))
self.logger.info(entry.path_display)
content_hash = self.cache.get(entry.path_display)
if not os.path.isfile(
destination) or content_hash != entry.content_hash:
self.cache.set(entry.path_display, entry.content_hash)
self.download(entry.path_display, destination)
def download(self, dropbox_path, destination):
"""
Download file.
@param string dropbox_path
@param string destination
"""
parent = os.path.dirname(destination)
if not os.path.exists(parent):
os.makedirs(parent)
with open(destination, 'wb+') as fout:
_metadata, res = self.dbx.files_download(path=dropbox_path)
fout.write(res.content)
def check_if_excluded(self, path):
"""
Check if path is excluded.
@param string path
@return boolean
"""
for pattern in self.config.get('exclude'):
if re.match(pattern, path):
return True
return False
if not gene.preferred_isoform:
continue
sequence = getattr(gene.preferred_isoform, trait)
sequences.append(sequence)
names.append(gene.name)
series = Series(sequences, names)
if subset is not None:
series = series[series.index.isin(subset)]
return series
manager = MutationImportManager()
cache = Cache('active_driver_data')
cache.reset('cull_limit', 0)
cached = cache_decorator(cache)
@cached
def prepare_active_driver_data(mutation_source: str,
site_type=None,
mutation_query=None,
sites=None):
if sites is None:
sites = export_and_load(sites_ac)
if site_type:
sites = sites[sites['type'].str.contains(site_type)]
from typing import Optional
from discord import Client
from discord.utils import get
import utils
from db.database import PostgreSQL
from guild_data.guild import Guild
from helpers.cache import Cache
from user_data.user import User
USER_CACHE = Cache()
GUILD_CACHE = Cache()
def clear_cache():
USER_CACHE.clear_all()
GUILD_CACHE.clear_all()
def get_user(db: PostgreSQL,
user_id: int,
create: bool = True) -> Optional[User]:
user = USER_CACHE.get(user_id)
if not user:
if create or db.get_row_data("users", {'id': user_id}):
user = User(db, user_id)
USER_CACHE[user_id] = user
else:
return None
return user
def ptm_muts_frequency(site_type, mode):
groups, significances = ptm_muts_enrichment_frequency(site_type, mode)
return groups
@muts_cases
def ptm_muts_frequency_significance(site_type, mode):
groups, significances = ptm_muts_enrichment_frequency(site_type, mode)
return p_value_annotations(groups, significances)
MODES = ['occurrences', 'distinct']
muts_cases = cases(source=[MC3Mutation, InheritedMutation],
site_type=site_types_with_any,
mode=MODES).set_mode('product')
cached = cache_decorator(Cache('.enrichment_plot_cache'))
@cached
def ptm_muts_enrichment(source, site_type, mode, repeats=1000000):
boxes = {}
sources_and_filters = {
MC3Mutation: ('TCGA', None),
InheritedMutation:
('ClinVar (pathogenic, likely pathogenic, drug response)',
InheritedMutation.significance_filter('strict'))
}
name, filters = sources_and_filters[source]
observed, expected, region, p = ptm_on_random(source=source,
site_type=site_type.name,
mode=mode,
def test_setget(self):
cache = Cache()
cache['Test'] = 24
self.assertIn('Test', cache)
self.assertEqual(24, cache['Test'])
class GoogleDrive(Strategy):
"""Backup strategy for Google Drive."""
NAME = 'Google Drive'
TYPE = 'googledrive'
API_URL = 'https://www.googleapis.com/drive/v3/files'
cache = None
def add(self):
"""Add Google Drive account."""
self.alias = super().add()
# Show instructions
self.show_instructions()
# Parse credentials
credentials_str = input('Paste content of credentials file: ')
self.config.set('credentials', json.loads(
credentials_str)['installed'])
# Get access code
code = self.request_code()
token = self.request_token(code)
self.config.set('token', token)
def start_backup(self):
"""Start backup."""
# Set cache
self.cache = Cache(self.alias)
# Backup
self.get_children()
# Cleanup
self.cleanup()
def cleanup(self):
"""Delete files that have been removed from Drive."""
all_cached = self.cache.get()
for id, item in list(all_cached.items()):
if item['last_seen'] is not util.startup_time and 'path' in item:
# Delete file
Path(item['path']).unlink(missing_ok=True)
# Remove item from cache
self.cache.delete(id)
def build_auth_uri(self):
"""Build auth URI for requesting token.
@return string
"""
auth_uri = self.config.get('credentials.auth_uri')
auth_uri += '?response_type=code'
auth_uri += '&redirect_uri=' + \
quote_plus(self.config.get('credentials.redirect_uris.0'))
auth_uri += '&client_id=' + \
quote_plus(self.config.get('credentials.client_id'))
auth_uri += '&scope=https://www.googleapis.com/auth/drive.readonly'
auth_uri += '&access_type=offline'
auth_uri += '&approval_prompt=auto'
return auth_uri
def request_code(self):
"""Request code from auth URI to obtain token.
@return string
"""
# Build auth uri
auth_uri = self.build_auth_uri()
# Try opening in browser
webbrowser.open(auth_uri, new=1)
print()
print('If your browser does not open, go to this website:')
print(auth_uri)
print()
# Return code
return input('Enter code: ')
def execute_request(self, url, headers={}, params={}, method='GET', is_retry=False):
"""Call Drive API.
@param string url
@param dict headers
@param dict params
@param string method
@param bool is_retry
"""
if self.config.get('token.access_token'):
# Set Authorization-Header
auth_header = {
'Authorization': 'Bearer {}'.format(self.config.get('token.access_token'))
}
headers.update(auth_header)
# Execute request
if method == 'GET':
res = requests.get(url, headers=headers, params=params)
else:
res = requests.post(url, headers=headers, data=params)
# Permission error
if res.status_code == 401:
# Maybe the token is expired
if not is_retry:
# Refresh token
self.config.set('token', self.request_token())
# Re-try request
return self.execute_request(url, headers, params, method, True)
else:
# This is already a retry, don't try again
raise Exception('Failed to refresh token')
return {
'status': res.status_code,
'headers': res.headers,
'body': res.json()
}
def request_token(self, code=''):
"""Request access token.
@param string code (optional)
@return dict
"""
headers = {
'Content-Type': 'application/x-www-form-urlencoded'
}
params = {
'client_id': self.config.get('credentials')['client_id'],
'client_secret': self.config.get('credentials')['client_secret'],
'redirect_uri': self.config.get('credentials')['redirect_uris'][0],
}
if code:
params['grant_type'] = 'authorization_code'
params['code'] = code
else:
params['grant_type'] = 'refresh_token'
params['refresh_token'] = self.config.get('token')['refresh_token']
res = self.execute_request(self.config.get('credentials')[
'token_uri'], headers, params, 'POST')
if res['status'] == 200:
if self.config.get('token'):
res['body']['refresh_token'] = self.config.get('token')[
'refresh_token']
self.config.set('token', res['body'])
return res['body']
else:
raise Exception('Error getting token: ' + str(res['body']))
def show_instructions(self):
"""Print instructions on how to set up Google Cloud Project."""
print()
print('If you already have an OAuth-Client-ID, download the JSON')
print('Otherwise, here\'s how to get credentials:')
print('1. Go to https://console.developers.google.com/')
print('2. Choose or create a project')
print('3. Activate Photos API here: https://console.developers.google.com/apis/library/photoslibrary.googleapis.com')
print('4. Open https://console.developers.google.com/apis/credentials/consent')
print('5. Choose "External"')
print('6. Enter a name, support email and contact email')
print('7. Click "Save and continue"')
print('8. Click "Add or remove scopes"')
print('9. Select ".../auth/drive.readonly"')
print('10. Select ".../auth/photoslibrary.readonly"')
print('11. Click "Save and continue"')
print('12. Enter yourself as a test user')
print('13. Click "Save and continue"')
print(
'14. [Open credentials page](https://console.developers.google.com/apis/credentials)')
print('15. Click on "Create Credentials" -> OAuth-Client-ID -> Desktop Application')
print('16. Download the Client ID JSON')
print()
def check_if_excluded(self, path):
"""Check if file is to be excluded from download.
@param string path
@return boolean
"""
for pattern in self.config.get('exclude'):
if re.match(pattern, path):
return True
return False
def is_folder(self, item):
"""Check if item is a Google Folder.
@param GoogleDriveFile item
@return boolean
"""
return item['mimeType'] == 'application/vnd.google-apps.folder'
def is_google_doc(self, item):
"""Check if item is a Google Doc.
@param GoogleDriveFile item
@return boolean
"""
return item['mimeType'] == 'application/vnd.google-apps.document'
def is_google_sheet(self, item):
"""Check if item is a Google Spreadsheet.
@param GoogleDriveFile item
@return boolean
"""
return item['mimeType'] == 'application/vnd.google-apps.spreadsheet'
def is_google_slides(self, item):
"""Check if item is a Google Slidedeck.
@param GoogleDriveFile item
@return boolean
"""
return item['mimeType'] == 'application/vnd.google-apps.presentation'
def get_children(self, item_id='root', parents=[], page_token=''):
"""Traverse Drive recursively and initiate file download.
@param string item_id (optional)
@param list parents (optional)
@param string page_token (optional)
"""
path_server = '/' + '/'.join(parents).strip('/')
path = os.path.join(self.backup_path, path_server.strip('/'))
params = {
'q': "'" + item_id + "' in parents",
'fields': 'nextPageToken,files(id,name,mimeType,modifiedTime,trashed)',
'pageSize': '100'
}
if page_token:
params['pageToken'] = page_token
# Build param-string
params_str = ''
for key, param in params.items():
params_str = params_str + key + '=' + param + '&'
params_str = params_str[:-1].replace(',', '%2C').replace(' ', '+')
# Send request
res = self.execute_request(self.API_URL + '?' + params_str)
items = res['body']['files'] if res['status'] == 200 else []
for item in items:
url = self.API_URL + '/' + item['id'] + '?alt=media'
path_item = os.path.join(path_server, item['name'])
filename = item['name']
# Excluded or trashed
if self.check_if_excluded(path_item):
continue
if item['trashed']:
continue
# Folders
if self.is_folder(item):
self.get_children(item['id'], parents + [item['name']])
continue
# Google Docs
if self.is_google_doc(item):
url = self.API_URL + '/' + \
item['id'] + '/export?mimeType=application/pdf'
filename = item['name'] + '_converted.pdf'
# Google Spreadsheets
elif self.is_google_sheet(item):
url = self.API_URL + '/' + \
item['id'] + '/export?mimeType=application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
filename = item['name'] + '.xlsx'
# Google Slides
elif self.is_google_slides(item):
url = self.API_URL + '/' + \
item['id'] + '/export?mimeType=application/pdf'
filename = item['name'] + '_converted.pdf'
# Remember last seen in cache
self.cache.set(item['id'] + '.last_seen', util.startup_time)
# Move if moved
if self.check_if_moved_and_move(item, path, filename):
continue
# Download
if not self.is_backed_up(item, path, filename):
try:
self.download(url, path, filename)
# Add to cache
self.cache.set(item['id'] + '.modified',
item['modifiedTime'])
self.cache.set(item['id'] + '.path',
os.path.join(path, filename))
except Exception as e:
self.logger.error(e)
if 'nextPageToken' in res['body']:
self.get_children(item_id, parents, res['body']['nextPageToken'])
def check_if_moved_and_move(self, item, path, filename):
"""Check if source was simply moved and move if so.
To determine whether the item has moved check the modified time.
We can't use 'md5Checksum' here, because Google Docs don't have one.
@param GoogleDriveFile item
@param string path
@param string filename
@return boolean
"""
move_source = self.cache.get(item['id'])
if move_source and 'modified' in move_source:
move_target = os.path.join(path, filename)
if move_source['modified'] == item['modifiedTime'] \
and move_source['path'] != move_target:
# Create folder if not exists
if not os.path.exists(path):
os.makedirs(path)
self.logger.info('Moving {} to {}'.format(
move_source['path'], move_target))
os.rename(move_source['path'], move_target)
self.cache.set(item['id'] + '.' + 'path', move_target)
return True
return False
def is_backed_up(self, item, path, filename):
"""Check if file exists and is newer than on Drive.
@param GoogleDriveFile item
@param string path
@param string filename
@return boolean
"""
if os.path.isfile(os.path.join(path, filename)):
mtime_ts = os.path.getmtime(os.path.join(path, filename))
mtime_date = datetime.utcfromtimestamp(mtime_ts).isoformat()
if item['modifiedTime'] < mtime_date:
return True
return False
def download(self, url, path, filename):
"""Download item.
@param string url
@param string path
@param string filename
"""
# Create folder if not exists
if not os.path.exists(path):
os.makedirs(path)
headers = {
'Authorization': 'Bearer {}'.format(self.config.get('token')['access_token'])
}
# Download file
self.logger.info('Downloading {}...'.format(
os.path.join(path, filename)))
http = urllib3.PoolManager()
res = http.request('GET', url, headers=headers, preload_content=False)
if res.status == 200:
self.logger.info('Downloaded.')
with open(os.path.join(path, filename), 'wb') as out:
while True:
data = res.read(128)
if not data:
break
out.write(data)
res.release_conn()
else:
raise Exception(
'Download failed ({}) -> {}'.format(res.status, str(res.data)))
