def load_worker(self):
while True:
try:
if self.__terminating:
return
new_length = self.__ethereum_utils.get_length(self.__account)
print('load', self.__blockchain_length, new_length)
if new_length > self.__blockchain_length:
self.__store_event.wait()
with self.__lock:
with self.__app.app_context():
for k, v in self.__ethereum_utils.get_history(
self.__account, self.__blockchain_length,
self.__storage):
print('loading:', k, v)
self.load_key_value(k, v)
self.__blockchain_length = new_length
KeyLookupTable.query.session.commit()
Settings().blockchain_length = new_length
Settings().blockchain = self.__blockchain
Settings().write()
socketio.emit('refresh password')
except BadFunctionCallOutput:
break
except Exception as e:
print(e)
finally:
self.__loaded = True
time.sleep(self.__load_interval)
def __init__(self, account: Address, password: str):
"""
Load the ethereum storage for `account`. `password` should also be provided.
**Assumptions**:
1. Singleton class EthereumUtils has been initialized before
2. Contracts has been initialized (EthereumUtils.init_contracts)
3. Storage contract for `account` has been created (EthereumUtils.new_storage)
"""
self.__cache_dict: Dict[str, Tuple[str, bool]] = {}
self.__change_set: Set[str] = set()
self.__delete_set: Set[str] = set()
self.__ethereum_utils = EthereumUtils()
self.__account = account
self.__password = password
try:
self.__storage = self.__ethereum_utils.get_storage(account)
except TypeError:
raise TypeError
self.__lock = Lock()
self.__terminating = False
self.__store_interval = 15
self.__load_interval = 5
self.__store_event = Event()
self.__blockchain_length = Settings().blockchain_length
self.__blockchain = Settings().blockchain
# load blockchain from disk
for k, v in self.__blockchain:
if v == '':
del self.__cache_dict[k]
else:
self.__cache_dict[k] = (v, True)
# make up for the missing entries (delete entries that have not sync'ed)
for k in self.__cache_dict:
if not k.startswith('__') and KeyLookupTable.query.get(k) is None:
new_entry = KeyLookupTable(key=k, meta_data='', hidden=False)
KeyLookupTable.query.session.add(new_entry)
KeyLookupTable.query.session.commit()
self.__load_thread = Thread(target=self.load_worker, daemon=True)
self.__store_thread = Thread(target=self.store_worker, daemon=True)
self.__load_thread.start()
self.__store_thread.start()
self.__loaded = False
self.__app = current_app._get_current_object()
def startup():
if app.config['QUEUE']:
SessionKey(app.config['QUEUE'].get())
if app.config['USE_ETHEREUM']:
ethereum_utils = EthereumUtils(Web3(IPCProvider(get_ipc('./ethereum_private/data', 'geth.ipc'))))
storage_factory_abi = json.load(open('./ethereum_private/contracts/storage_factory.abi.json'))
storage_abi = json.load(open('./ethereum_private/contracts/storage.abi.json'))
ethereum_utils.init_contracts(get_env()['ETH_STORAGE'], storage_factory_abi, storage_abi)
Settings(app.config['SETTINGS_FILE'])
def initializing_worker(app):
with app.app_context():
settings = Settings()
current_app.config['INIT_STATE'] = 1
socketio.emit('state change', 1)
if current_app.config['USE_ETHEREUM']:
ethereum_account = initialize_ethereum_account(ethereum_pass)
current_app.config['STORAGE'] = EthereumStorage(
ethereum_account, ethereum_pass)
settings.ethereum_address = ethereum_account
settings.write()
else:
current_app.config['STORAGE'] = LocalStorage('chain')
storage = current_app.config['STORAGE']
storage.add(MASTER_KEY, settings.master_password_hash)
storage.add(MASTER_SALT_KEY, settings.master_password_hash_salt)
while True:
if storage.get(MASTER_KEY, True)[1] and storage.get(
MASTER_SALT_KEY, True)[1]:
break
time.sleep(0.1)
current_app.config['INIT_STATE'] = 2
socketio.emit('state change', 2)
def config_connection_tortoise():
settings = Settings()
if settings.ENVIRONMENT == "PROD":
logger.info("Using POSTGRES PROD database")
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
return {
"connections": {
"default": {
"engine": "tortoise.backends.asyncpg",
"credentials": {
"database": settings.DATABASE_NAME,
"host": settings.DATABASE_HOST,
"password": settings.DATABASE_PASS,
"port": settings.DATABASE_PORT,
"user": settings.DATABASE_USER,
"ssl": ctx
}
}
},
"apps": {
"models": {
"models": settings.MODELS_TORTOISE,
"default_connection": "default",
}
}
}
else:
logger.info("Using SQLITE DEV database")
return {
"connections": {
"default": settings.DATABASE_URL
},
"apps": {
"models": {
"models": settings.MODELS_TORTOISE,
"default_connection": "default",
}
}
}
def verify():
master_password_in_memory = request.decrypted_data.decode()
master_pass = MasterPassword.verify(master_password_in_memory)
ethereum_pass = master_pass.ethereum_pass
del request.decrypted_data
del master_password_in_memory
if master_pass:
current_app.config['MASTER_PASSWORD'] = master_pass
if current_app.config['USE_ETHEREUM']:
settings = Settings()
current_app.config['STORAGE'] = EthereumStorage(
settings.ethereum_address, ethereum_pass)
else:
current_app.config['STORAGE'] = LocalStorage('chain')
return jsonify(message='Success')
error_respond.master_password_wrong()
def setUpClass(cls):
Settings('db/tmp/config.json')
class EthereumStorage:
def __init__(self, account: Address, password: str):
"""
Load the ethereum storage for `account`. `password` should also be provided.
**Assumptions**:
1. Singleton class EthereumUtils has been initialized before
2. Contracts has been initialized (EthereumUtils.init_contracts)
3. Storage contract for `account` has been created (EthereumUtils.new_storage)
"""
self.__cache_dict: Dict[str, Tuple[str, bool]] = {}
self.__change_set: Set[str] = set()
self.__delete_set: Set[str] = set()
self.__ethereum_utils = EthereumUtils()
self.__account = account
self.__password = password
try:
self.__storage = self.__ethereum_utils.get_storage(account)
except TypeError:
raise TypeError
self.__lock = Lock()
self.__terminating = False
self.__store_interval = 15
self.__load_interval = 5
self.__store_event = Event()
self.__blockchain_length = Settings().blockchain_length
self.__blockchain = Settings().blockchain
# load blockchain from disk
for k, v in self.__blockchain:
if v == '':
del self.__cache_dict[k]
else:
self.__cache_dict[k] = (v, True)
# make up for the missing entries (delete entries that have not sync'ed)
for k in self.__cache_dict:
if not k.startswith('__') and KeyLookupTable.query.get(k) is None:
new_entry = KeyLookupTable(key=k, meta_data='', hidden=False)
KeyLookupTable.query.session.add(new_entry)
KeyLookupTable.query.session.commit()
self.__load_thread = Thread(target=self.load_worker, daemon=True)
self.__store_thread = Thread(target=self.store_worker, daemon=True)
self.__load_thread.start()
self.__store_thread.start()
self.__loaded = False
self.__app = current_app._get_current_object()
def add(self, k: str, v: str):
"""
Add a new entry with key `k` and value `v` into the database. If the entry with key `k` exists,
update its value with `v`. **This will not immediately write the underlying database.**
"""
with self.__lock:
if k in self.__cache_dict and self.__cache_dict[k][1]:
self.__change_set.add(k)
elif k in self.__delete_set:
self.__delete_set.remove(k)
self.__change_set.add(k)
self.__cache_dict[k] = (v, False)
socketio.emit('persistence change', k)
def delete(self, k: str):
"""
Delete an entry in database with key `k`. If the key does not exist, an exception `KeyError` will be thrown.
**This will not immediately write the underlying database.**
"""
with self.__lock:
if k in self.__cache_dict:
if not self.__cache_dict[k][1]:
if k in self.__change_set:
# changed in cache
self.__delete_set.add(k)
del self.__cache_dict[k]
self.__change_set.remove(k)
else:
# new in cache, not changed in cache
del self.__cache_dict[k]
else:
self.__delete_set.add(k)
del self.__cache_dict[k]
else:
raise KeyError(k)
def get(
self,
k: str,
check_persistence: bool = False
) -> Union[Optional[str], Tuple[Optional[str], bool]]:
"""
Get an existing entry with key `k` in the database. If the entry with key `k` exists, return its value.
If the key does not exist, return `None`. If `check_persistence` is `True`,
returns a tuple like `(value, True)`, where the second element shows whether this key-value pair has been
`store`d into the underlying database. If `check_persistence` is `True` and the key does not exist, return
`(None, None)`.
"""
result = self.__cache_dict.get(k, (None, None))
if check_persistence:
return result
else:
return result[0]
def get_all(self) -> dict:
"""
Return all keys with their values and persistence in the database. The returned value should have a structure
like this:
{
key: (value, persistence)
}
"""
return self.__cache_dict
def __get_all_add(self) -> Generator[Tuple[str, str], None, None]:
return ((k, v[0]) for k, v in self.__cache_dict.items() if not v[1])
def __get_all_del(self) -> Generator[str, None, None]:
return (k for k in self.__delete_set)
def store(self):
"""
Synchronize the changes with underlying database.
"""
self.__ethereum_utils.unlock_account(self.__account,
self.__password,
duration=60)
# FIXME: use async add
# TODO: how to determine if a key is really stored? only update persistence if transaction mined?
add_list = []
with self.__lock:
for k, v in self.__get_all_add():
print('adding:', k, v)
add_list.append(
(k, v,
self.__ethereum_utils.add_async(self.__account, k, v)))
for k in self.__get_all_del():
print('deleting:', k)
add_list.append(
(None, None,
self.__ethereum_utils.add_async(self.__account, k)))
self.__change_set = set()
self.__delete_set = set()
return add_list
def estimate_cost(self, args: dict) -> int:
"""
Estimates the cost of the storage operation with arguments `args`.
"""
# FIXME: it returns gas count (gas count * gas price = cost in wei)
key = args['key']
value = args['value']
return self.__ethereum_utils.estimate_add_cost(self.__account, key,
value)
def calculate_total_cost(self) -> int:
"""
Calculates the cost of currently cached storage operations.
"""
# FIXME: it returns gas count (gas count * gas price = cost in wei)
s = 0
for k, v in self.__get_all_add():
s += self.__ethereum_utils.estimate_add_cost(self.__account, k, v)
for k in self.__get_all_del():
s += self.__ethereum_utils.estimate_add_cost(self.__account, k)
return s
def balance(self) -> int:
"""
Returns the balance (remaining storage space) of current user.
"""
# FIXME: it returns wei
return self.__ethereum_utils.get_balance(self.__account)
def get_constructor_arguments(self) -> Address:
"""
Returns the arguments list to pass to the constructor.
"""
# TODO: is it necessary to return password?
return self.__account
def size(self) -> int:
return len(self.__cache_dict)
def __setitem__(self, key, value):
self.add(key, value)
def __delitem__(self, key):
self.delete(key)
def __getitem__(self, item):
self.get(item)
def __len__(self):
return self.size()
def store_worker(self):
while True:
try:
self.__store_event.set()
add_list = self.store()
if self.__terminating:
# Terminating, hopefully someone will mine our transaction :)
return
finished = set()
while len(finished) < len(add_list):
for k, v, h in add_list:
if h not in finished and self.__ethereum_utils.get_transaction_receipt(
h):
finished.add(h)
if k:
with self.__lock:
if self.__cache_dict.get(k)[0] == v:
self.__cache_dict[k] = (v, True)
socketio.emit('persistence change', k)
time.sleep(0.01)
self.__store_event.clear()
time.sleep(self.__store_interval)
except Exception as e:
print(e)
time.sleep(self.__store_interval)
def load_key_value(self, k: str, v: str):
self.__blockchain.append((k, v))
if v == '':
del self.__cache_dict[k]
if not k.startswith('__'):
KeyLookupTable.query.filter_by(key=k).delete()
else:
self.__cache_dict[k] = (v, True)
if not k.startswith('__'):
old_entry = KeyLookupTable.query.get(k)
if old_entry:
old_entry.meta_data = ''
else:
new_entry = KeyLookupTable(key=k,
meta_data='',
hidden=False)
KeyLookupTable.query.session.add(new_entry)
def load_worker(self):
while True:
try:
if self.__terminating:
return
new_length = self.__ethereum_utils.get_length(self.__account)
print('load', self.__blockchain_length, new_length)
if new_length > self.__blockchain_length:
self.__store_event.wait()
with self.__lock:
with self.__app.app_context():
for k, v in self.__ethereum_utils.get_history(
self.__account, self.__blockchain_length,
self.__storage):
print('loading:', k, v)
self.load_key_value(k, v)
self.__blockchain_length = new_length
KeyLookupTable.query.session.commit()
Settings().blockchain_length = new_length
Settings().blockchain = self.__blockchain
Settings().write()
socketio.emit('refresh password')
except BadFunctionCallOutput:
break
except Exception as e:
print(e)
finally:
self.__loaded = True
time.sleep(self.__load_interval)
def terminate(self):
self.__terminating = True
self.__load_thread.join()
self.__store_thread.join()
def __del__(self):
self.terminate()
@property
def loaded(self):
return self.__loaded
def verify_with_account():
if not current_app.config['USE_ETHEREUM']:
abort(404)
data = json.loads(request.decrypted_data.decode())
master_password_in_memory = data['password']
account = data['account']
del request.decrypted_data
del data
ethereum_pass = MasterPassword.generate_ethereum_password(
master_password_in_memory)
storage = None
try:
storage = EthereumStorage(account, ethereum_pass)
except TypeError:
error_respond.blockchain_account_wrong()
while not storage.loaded:
time.sleep(0.1)
master_pass_hash = storage.get(MASTER_KEY)
master_salt = storage.get(MASTER_SALT_KEY)
if master_pass_hash and master_salt:
settings = Settings()
settings.master_password_hash = master_pass_hash
settings.master_password_hash_salt = master_salt
master_pass = MasterPassword.verify(master_password_in_memory)
del master_password_in_memory
if master_pass:
current_app.config['MASTER_PASSWORD'] = master_pass
current_app.config['STORAGE'] = storage
settings.ethereum_address = account
settings.write()
current_app.config['INIT_STATE'] = 2
socketio.emit('state change', 2)
return jsonify(message='Success')
else:
settings.master_password_hash = ''
settings.master_password_hash_salt = ''
settings.write()
storage.terminate()
error_respond.master_password_wrong()
storage.terminate()
error_respond.blockchain_account_wrong()