def token(dm: Dimensigon,
dimension_id_or_name: str,
applicant=None,
expire_time=None):
dm.create_flask_instance()
with dm.flask_app.app_context():
if dimension_id_or_name is not None:
dim = Dimension.query.get(dimension_id_or_name)
if dim is None:
dim = Dimension.query.filter_by(name=dimension_id_or_name)
if dim is None:
exit(f"{dimension_id_or_name} is not a valid dimension")
else:
count = Dimension.query.count()
if count == 1:
dim = Dimension.query.all()[0]
else:
exit("No dimension specified. Please specify a dimension")
if dimension_id_or_name:
dm.flask_app.config['SECRET_KEY'] = dim.id
else:
if dm.flask_app.config['SECRET_KEY'] != dim.id:
exit('Secret key mismatch')
join_user = User.get_by_name('join')
if not join_user:
exit("Unable to create join token. Populate database first.")
print(
create_access_token(
join_user.id,
expires_delta=dt.timedelta(
minutes=expire_time or defaults.JOIN_TOKEN_EXPIRE_TIME),
additional_claims={'applicant': applicant}))
def gate_list(dm: Dimensigon):
dm.create_flask_instance()
with dm.flask_app.app_context():
dprint([
f"{g.ip or g.dns}:{g.port}{' (hidden)' if g.hidden else ''}"
for g in Gate.query.filter_by(
server_id=Server.get_current().id).all()
])
def gate_port(dm: Dimensigon, port):
dm.create_flask_instance()
with dm.flask_app.app_context():
db.engine.execute(sql.text(
f"UPDATE D_gate set port = :port WHERE main.D_gate.server_id = :server_id"
),
port=port,
server_id=Server.get_current().id)
db.session.commit()
def run(dm: Dimensigon):
# check if there is a dimension
result = dm.engine.execute(Dimension.__table__.select())
count = len(result.fetchall())
if count == 0:
exit("No dimension created. Create or join to a dimension")
try:
dm.start()
except RuntimeError as e:
print("\nError: %s\n" % e, file=sys.stderr)
sys.stderr.flush()
sys.exit(1)
def populate_initial_data(dm: Dimensigon):
from dimensigon.domain.entities import ActionTemplate, Locker, Server, User, Parameter
with session_scope(session=dm.get_session()) as session:
gates = dm.config.http_conf.get('binds', None)
SchemaChanges.set_initial(session)
dm.server_id = Server.set_initial(session, gates)
Locker.set_initial(session, unlock=True)
ActionTemplate.set_initial(session)
User.set_initial(session)
Parameter.set_initial(session)
def setup_db(dm: Dimensigon):
"""Ensure database is ready to fly."""
dm.engine = create_engine(dm.config.db_uri)
if not os.path.exists(dm.config.db_uri[len(defaults.DB_PREFIX):]):
_LOGGER.info(
f"Creating database {dm.config.db_uri[len(defaults.DB_PREFIX):]}")
db.Model.metadata.create_all(dm.engine)
time.sleep(5)
else:
_LOGGER.debug(
f"Database {dm.config.db_uri[len(defaults.DB_PREFIX):]} already exists"
)
dm.get_session = scoped_session(sessionmaker(bind=dm.engine))
migrate_schema(dm)
populate_initial_data(dm)
def gate_delete(dm: Dimensigon, ip_or_dns, port, hidden=True):
dm.create_flask_instance()
with dm.flask_app.app_context():
ip, dns = None, None
try:
ip = ipaddress.ip_address(ip_or_dns)
except:
dns = ip_or_dns
query = Gate.query
if ip or dns:
query = query.filter_by(ip=ip, dns=dns)
if port:
query = query.filter_by(port=port)
if hidden:
query = query.filter_by(hidden=hidden)
[g.delete() for g in query.all()]
db.session.commit()
def gate_create(dm: Dimensigon, ip_or_dns, port, hidden=True):
dm.create_flask_instance()
with dm.flask_app.app_context():
ip, dns = None, None
try:
ip = ipaddress.ip_address(ip_or_dns)
except:
dns = ip_or_dns
g = Gate(server=Server.get_current(),
ip=ip,
dns=dns,
port=port,
hidden=hidden)
db.session.add(g)
try:
db.session.commit()
except exc.IntegrityError:
exit(f"{ip or dns}:{port} already exists")
else:
print(
f"{g.ip or g.dns}:{g.port}{' (hidden)' if g.hidden else ''} created succesfully"
)
def catalog(dm: Dimensigon, ip, port, http=False):
import dimensigon.dshell.network as dshell_ntwrk
dm.create_flask_instance()
dm.set_catalog_manager()
with dm.flask_app.app_context():
print("Updating catalog...")
catalog_datamark = Catalog.max_catalog(str)
resp = dshell_ntwrk.request(
'get',
dshell_ntwrk.generate_url(
'api_1_0.catalog',
view_data=dict(data_mark=catalog_datamark),
ip=ip,
port=port,
scheme='http' if http else 'https'),
auth=get_root_auth())
if resp.ok:
try:
dm.catalog_manager.catalog_update(resp.msg)
except Exception as e:
exit(f"Unable to upgrade data. Exception: {e}")
else:
exit(f"Unable to get catalog from {resp.url}: {resp}")
def setup_dm(run_config: RuntimeConfig) -> Dimensigon:
dm = Dimensigon()
# set dimensigon configuration
_setup_dimensigon_config(run_config, dm.config)
# set http configuration. Before setup_db to get ip binds
_setup_http_config(run_config, dm.config)
# set database uri
setup_database_uri(run_config, dm.config)
# set database to allow queries
setup_db(dm)
# set flask configuration
_setup_flask_config(run_config, dm)
return dm
def migrate_schema(dm: Dimensigon):
progress_path = dm.config.path(PROGRESS_FILE)
with session_scope(session=dm.get_session()) as session:
res = (session.query(SchemaChanges).order_by(
SchemaChanges.change_id.desc()).first())
current_version = getattr(res, "schema_version", None)
if current_version is None:
# first time running database
sc = SchemaChanges(schema_version=SCHEMA_VERSION)
session.add(sc)
current_version = SCHEMA_VERSION
if current_version == SCHEMA_VERSION:
# Clean up if old migration left file
if os.path.isfile(progress_path):
_LOGGER.warning("Found existing migration file, cleaning up")
os.remove(dm.config.path(PROGRESS_FILE))
return
with open(progress_path, "w"):
pass
_LOGGER.warning("Database is about to upgrade. Schema version: %s",
current_version)
try:
for version in range(current_version, SCHEMA_VERSION):
new_version = version + 1
_LOGGER.info("Upgrading db schema to version %s", new_version)
_apply_update(dm.engine, new_version, current_version)
session.add(SchemaChanges(schema_version=new_version))
_LOGGER.info("Upgrade to version %s done", new_version)
finally:
os.remove(dm.config.path(PROGRESS_FILE))
def locker_list(dm: Dimensigon):
dm.create_flask_instance()
with dm.flask_app.app_context():
dprint([l.to_dict() for l in Locker.query.all()])
def new(dm: Dimensigon, name: str):
dm.create_flask_instance()
with dm.flask_app.app_context():
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.x509.oid import NameOID
import datetime
Server.set_initial()
count = Dimension.query.count()
if count > 0:
exit("Only one dimension can be created")
dim_name = name or coolname.generate_slug(2)
dim = generate_dimension(dim_name)
private_key = serialization.load_pem_private_key(
dim.private.save_pkcs1(), password=None, backend=default_backend())
dim.current = count == 0
db.session.add(dim)
now = get_now()
subject = issuer = x509.Name([
x509.NameAttribute(NameOID.COUNTRY_NAME, u"LU"),
x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, u"California"),
x509.NameAttribute(NameOID.LOCALITY_NAME, u"San Francisco"),
x509.NameAttribute(NameOID.COMMON_NAME, dim_name),
x509.NameAttribute(NameOID.ORGANIZATION_NAME, u"KnowTrade S.L."),
])
cert = x509.CertificateBuilder().subject_name(subject) \
.issuer_name(issuer) \
.not_valid_before(now) \
.not_valid_after(now + datetime.timedelta(days=365 * 10)) \
.serial_number(x509.random_serial_number()) \
.public_key(private_key.public_key()) \
.add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=False, ) \
.sign(private_key=private_key, algorithm=hashes.SHA256(),
backend=default_backend()
)
ssl_dir = os.path.join(dm.config.config_dir, defaults.SSL_DIR)
os.makedirs(ssl_dir, exist_ok=True)
with open(os.path.join(ssl_dir, defaults.CERT_FILE), "wb") as f:
f.write(cert.public_bytes(serialization.Encoding.PEM))
with open(os.path.join(ssl_dir, defaults.KEY_FILE), 'wb') as file:
file.write(
private_key.private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.TraditionalOpenSSL,
encryption_algorithm=serialization.NoEncryption(),
))
os.chmod(os.path.join(ssl_dir, defaults.KEY_FILE), 0o600)
db.session.commit()
user = User.get_by_name('root')
if user is None:
User.set_initial()
user = User.get_by_name('root')
p = False
p2 = True
while p != p2:
p = prompt_toolkit.prompt("Password for root user: "******"Re-type same password: "******"New dimension created successfully")
print("")
print("----- JOIN TOKEN (valid for {} minutes) -----".format(
defaults.JOIN_TOKEN_EXPIRE_TIME))
token(dm, dim.id)
print("---------------- END TOKEN --------------------")
def join(dm: Dimensigon,
server: str,
token: str,
port: int = None,
ssl: bool = True,
verify: bool = False):
def str_resp(resp: requests.Response):
try:
return resp.json()
except ValueError:
return resp.text
logger = logging.getLogger('dm.join')
dm.create_flask_instance()
dm.set_catalog_manager()
with dm.flask_app.app_context():
Server.set_initial()
db.session.commit()
protocol = "https" if ssl else "http"
resp = None
times = 5
for i in range(times):
try:
resp = requests.get(
f"{protocol}://{server}:{port}/api/v1.0/join/public",
headers={'Authorization': 'Bearer ' + token},
verify=verify,
timeout=20)
except requests.exceptions.ConnectionError as e:
logger.error(f"Unable to contact to {server}.")
except requests.exceptions.Timeout as e:
logger.error(
f"Timeout of 20s reached while trying to contact to {server}."
)
except Exception as e:
logger.exception(f"Error trying to get dimensigon public key.")
else:
if resp.status_code != 200:
if resp.status_code in (401, 422):
logger.error(
f"Error on authentication: {str_resp(resp)}")
sys.exit(9)
else:
logger.error(
f"Error trying to get dimensigon public key: {str_resp(resp)}"
)
else:
break
if i < times - 1:
d = int(random.random() * 25 * (i + 1))
logger.info(f"Retrying in {d} seconds.")
time.sleep(d)
else:
sys.exit(1)
pub_key = rsa.PublicKey.load_pkcs1(resp.content)
# Generate Public and Private Temporal Keys
tmp_pub, tmp_priv = rsa.newkeys(2048)
symmetric_key = generate_symmetric_key()
s = Server.get_current()
data = s.to_json(add_gates=True)
data = pack_msg2(data=data,
pub_key=pub_key,
priv_key=tmp_priv,
symmetric_key=symmetric_key,
add_key=True)
data.update(my_pub_key=tmp_pub.save_pkcs1().decode('ascii'))
logger.info("Joining to dimension...")
resp = None
times = 5
for i in range(times):
try:
resp = requests.post(
f"{protocol}://{server}:{port}/api/v1.0/join",
json=data,
headers={'Authorization': 'Bearer ' + token},
verify=verify,
timeout=45)
from dimensigon.web import errors
if resp.ok:
break
else:
logger.error(
f"Error while trying to join. {str_resp(resp)}")
except Exception as e:
logger.exception(f"Error while trying to join.")
resp = None
if i < times - 1:
d = int(random.random() * 25 * (i + 1))
logger.info(f"Retrying in {d} seconds.")
time.sleep(d)
if resp is None:
resp_data = {}
elif resp.status_code != 200:
db.session.rollback()
logger.info(
f"Error while trying to join the dimension: {str_resp(resp)}")
resp_data = {}
else:
resp_data = unpack_msg2(resp.json(),
pub_key=pub_key,
priv_key=tmp_priv,
symmetric_key=symmetric_key)
logger.log(1, resp_data)
if 'Dimension' in resp_data:
json_dim = resp_data.pop('Dimension')
dim = Dimension.query.get(json_dim.get('id'))
if not dim:
dim = Dimension.from_json(json_dim)
dim.current = True
db.session.add(dim)
keyfile_content = base64.b64decode(
resp_data.pop('keyfile').encode())
with open(dm.config.http_conf['keyfile'], 'wb') as fh:
fh.write(keyfile_content)
del keyfile_content
certfile_content = base64.b64decode(
resp_data.pop('certfile').encode())
with open(dm.config.http_conf['certfile'], 'wb') as fh:
fh.write(certfile_content)
del certfile_content
logger.info('Updating Catalog...')
reference_server_id = resp_data.pop('me')
# remove catalog
for c in Catalog.query.all():
db.session.delete(c)
try:
dm.catalog_manager.db_update_catalog(
resp_data['catalog']) # implicit commit
except Exception as e:
logger.exception(f"Unable to upgrade catalog.")
sys.exit(3)
else:
logger.info('Catalog updated.')
# set reference server as a neighbour
reference_server = Server.query.get(reference_server_id)
if not reference_server:
db.session.rollback()
logger.info(
f"Server id {reference_server_id} not found in catalog."
)
sys.exit(4)
Route(destination=reference_server, cost=0)
# update_route_table_cost(True)
Parameter.set("join_server", f'{reference_server.id}')
resp = None
times = 5
for i in range(times):
try:
resp = requests.post(
f"{protocol}://{server}:{port}/api/v1.0/join/acknowledge/{s.id}",
headers={'Authorization': 'Bearer ' + token},
verify=verify,
timeout=120)
from dimensigon.web import errors
if resp.ok:
break
elif resp.status_code == 404:
if resp.json().get('error',
{}).get('type',
None) == 'EntityNotFound':
break
else:
logger.error(
f"Error while trying to join. Error: {resp.status_code}, "
f"{resp.content}")
except Exception as e:
logger.exception(f"Error while trying to join.")
resp = None
if i < times - 1:
d = int(random.random() * 25 * (i + 1))
logger.info(f"Retrying in {d} seconds.")
time.sleep(d)
if resp.ok:
logger.info('Joined to the dimension.')
else:
logger.error("Unable to confirm join.")
db.session.commit()
else:
logger.error(f"No dimension in response data.")
sys.exit(5)