def get_client(url=None, auth_token=None, username=None, password=None):
"""Return Mergin client."""
if auth_token is not None:
try:
mc = MerginClient(url, auth_token=auth_token)
except ClientError as e:
click.secho(str(e), fg="red")
return None
# Check if the token has expired or is just about to expire
delta = mc._auth_session["expire"] - datetime.now(timezone.utc)
if delta.total_seconds() > 5:
return mc
if username and password:
auth_token = get_token(url, username, password)
if auth_token is None:
return None
mc = MerginClient(url, auth_token=f"Bearer {auth_token}")
else:
click.secho(
"Missing authorization data.\n"
"Either set environment variables (MERGIN_USERNAME and MERGIN_PASSWORD) "
"or specify --username / --password options.\n"
"Note: if --username is specified but password is missing you will be prompted for password.",
fg="red",
)
return None
return mc
def get_token(url, username, password):
"""Get authorization token for given user and password."""
mc = MerginClient(url)
if not mc.is_server_compatible():
click.secho(str(
"This client version is incompatible with server, try to upgrade"),
fg="red")
return None
try:
session = mc.login(username, password)
except LoginError as e:
click.secho("Unable to log in: " + str(e), fg="red")
return None
return session["token"]
def login(url, login, password):
"""Fetch new authentication token. If no URL is specified, the public Mergin instance will be used."""
c = MerginClient(url)
click.echo("Mergin URL: " + c.url)
if not c.is_server_compatible():
click.secho(str('This client version is incompatible with server, try to upgrade'), fg='red')
return
try:
session = c.login(login, password)
except LoginError as e:
click.secho('Unable to log in: ' + str(e), fg='red')
return
print('export MERGIN_URL="%s"' % c.url)
print('export MERGIN_AUTH="%s"' % session['token'])
def _init_client():
url = os.environ.get('MERGIN_URL')
auth_token = os.environ.get('MERGIN_AUTH')
if auth_token is None:
click.secho(
"Missing authorization token: please run 'login' first and set MERGIN_AUTH env variable",
fg='red')
return None
return MerginClient(url, auth_token='Bearer {}'.format(auth_token))
def create_mergin_client():
""" Create instance of MerginClient"""
_check_has_password()
try:
return MerginClient(config.mergin_url,
login=config.mergin_username,
password=config.mergin_password,
plugin_version=f"DB-sync/{__version__}")
except LoginError as e:
# this could be auth failure, but could be also server problem (e.g. worker crash)
raise DbSyncError(
f"Unable to log in to Mergin: {str(e)} \n\n" +
"Have you specified correct credentials in configuration file?")
except ClientError as e:
# this could be e.g. DNS error
raise DbSyncError("Mergin client error: " + str(e))
def _init_client():
url = os.environ.get('MERGIN_URL')
auth_token = os.environ.get('MERGIN_AUTH')
if auth_token is None:
click.secho("Missing authorization token: please run 'login' first and set MERGIN_AUTH env variable", fg='red')
return None
mc = MerginClient(url, auth_token='Bearer {}'.format(auth_token))
# check whether the token has not expired already (normally it expires in 12 hours)
from datetime import datetime, timezone
delta = mc._auth_session['expire'] - datetime.now(timezone.utc)
if delta.total_seconds() < 0:
click.secho("Access token has expired: please run 'login' again and set MERGIN_AUTH env variable", fg='red')
return None
return mc
def _print_unhandled_exception():
""" Outputs details of an unhandled exception that is being handled right now """
click.secho("Unhandled exception!", fg="red")
for line in traceback.format_exception(*sys.exc_info()):
click.echo(line)
@click.group(
epilog=
f"Copyright (C) 2019-2021 Lutra Consulting\n\n(mergin-py-client v{__version__} / pygeodiff v{GeoDiff().version()})"
)
@click.option(
"--url",
envvar="MERGIN_URL",
default=MerginClient.default_url(),
help=f"Mergin server URL. Default is: {MerginClient.default_url()}",
)
@click.option("--auth-token",
envvar="MERGIN_AUTH",
help="Mergin authentication token string")
@click.option("--username", envvar="MERGIN_USERNAME")
@click.option("--password",
cls=OptionPasswordIfUser,
prompt=True,
hide_input=True,
envvar="MERGIN_PASSWORD")
@click.pass_context
def cli(ctx, url, auth_token, username, password):
"""
Command line interface for the Mergin client module.
def mc():
assert SERVER_URL and API_USER and USER_PWD
#assert SERVER_URL and SERVER_URL.rstrip('/') != 'https://public.cloudmergin.com' and API_USER and USER_PWD
return MerginClient(SERVER_URL, login=API_USER, password=USER_PWD)
def dbsync_init(from_gpkg=True):
""" Initialize the dbsync so that it is possible to do two-way sync between Mergin and a database """
# let's start with various environment checks to make sure
# the environment is set up correctly before doing any work
if os.path.exists(config.project_working_dir):
raise DbSyncError("The project working directory already exists: " +
config.project_working_dir)
print("Connecting to the database...")
try:
conn = psycopg2.connect(config.db_conn_info)
except psycopg2.Error as e:
raise DbSyncError("Unable to connect to the database: " + str(e))
if _check_schema_exists(conn, config.db_schema_base):
raise DbSyncError("The base schema already exists: " +
config.db_schema_base)
if from_gpkg:
if _check_schema_exists(conn, config.db_schema_modified):
raise DbSyncError("The 'modified' schema already exists: " +
config.db_schema_modified)
else:
if not _check_schema_exists(conn, config.db_schema_modified):
raise DbSyncError("The 'modified' schema does not exist: " +
config.db_schema_modified)
_check_has_password()
print("Logging in to Mergin...")
try:
mc = MerginClient(config.mergin_url,
login=config.mergin_username,
password=config.mergin_password)
except LoginError:
raise DbSyncError(
"Unable to log in to Mergin: have you specified correct credentials in configuration file?"
)
# download the Mergin project
print("Download Mergin project " + config.mergin_project_name + " to " +
config.project_working_dir)
mc.download_project(config.mergin_project_name, config.project_working_dir)
_check_has_working_dir()
gpkg_full_path = os.path.join(config.project_working_dir,
config.mergin_sync_file)
if from_gpkg:
if not os.path.exists(gpkg_full_path):
raise DbSyncError("The input GPKG file does not exist: " +
gpkg_full_path)
else:
if os.path.exists(gpkg_full_path):
raise DbSyncError("The output GPKG file exists already: " +
gpkg_full_path)
# check there are no pending changes on server (or locally - which should never happen)
status_pull, status_push, _ = mc.project_status(config.project_working_dir)
if status_pull['added'] or status_pull['updated'] or status_pull['removed']:
raise DbSyncError(
"There are pending changes on server - need to pull them first: " +
str(status_pull))
if status_push['added'] or status_push['updated'] or status_push['removed']:
raise DbSyncError(
"There are pending changes in the local directory - that should never happen! "
+ str(status_push))
if from_gpkg:
# we have an existing GeoPackage in our Mergin project and we want to initialize database
# COPY: gpkg -> modified
_geodiff_make_copy("sqlite", "", gpkg_full_path, config.db_driver,
config.db_conn_info, config.db_schema_modified)
# COPY: modified -> base
_geodiff_make_copy(config.db_driver, config.db_conn_info,
config.db_schema_modified, config.db_driver,
config.db_conn_info, config.db_schema_base)
else:
# we have an existing schema in database with tables and we want to initialize geopackage
# within our a Mergin project
# COPY: modified -> base
_geodiff_make_copy(config.db_driver, config.db_conn_info,
config.db_schema_modified, config.db_driver,
config.db_conn_info, config.db_schema_base)
# COPY: modified -> gpkg
_geodiff_make_copy(config.db_driver, config.db_conn_info,
config.db_schema_modified, "sqlite", "",
gpkg_full_path)
# upload gpkg to mergin (client takes care of storing metadata)
mc.push_project(config.project_working_dir)
def dbsync_push():
""" Take changes in the 'modified' schema in the database and push them to Mergin """
tmp_dir = tempfile.gettempdir()
tmp_changeset_file = os.path.join(tmp_dir, 'dbsync-push-base2our')
if os.path.exists(tmp_changeset_file):
os.remove(tmp_changeset_file)
_check_has_working_dir()
_check_has_sync_file()
_check_has_password()
try:
mc = MerginClient(config.mergin_url,
login=config.mergin_username,
password=config.mergin_password)
status_pull, status_push, _ = mc.project_status(
config.project_working_dir)
except LoginError as e:
# this could be auth failure, but could be also server problem (e.g. worker crash)
raise DbSyncError("Mergin log in error: " + str(e))
except ClientError as e:
raise DbSyncError("Mergin client error: " + str(e))
# check there are no pending changes on server (or locally - which should never happen)
if status_pull['added'] or status_pull['updated'] or status_pull['removed']:
raise DbSyncError(
"There are pending changes on server - need to pull them first: " +
str(status_pull))
if status_push['added'] or status_push['updated'] or status_push['removed']:
raise DbSyncError(
"There are pending changes in the local directory - that should never happen! "
+ str(status_push))
conn = psycopg2.connect(config.db_conn_info)
if not _check_schema_exists(conn, config.db_schema_base):
raise DbSyncError("The base schema does not exist: " +
config.db_schema_base)
if not _check_schema_exists(conn, config.db_schema_modified):
raise DbSyncError("The 'modified' schema does not exist: " +
config.db_schema_modified)
# get changes in the DB
_geodiff_create_changeset(config.db_driver, config.db_conn_info,
config.db_schema_base, config.db_schema_modified,
tmp_changeset_file)
if os.path.getsize(tmp_changeset_file) == 0:
print("No changes in the database.")
return
# summarize changes
summary = _geodiff_list_changes_summary(tmp_changeset_file)
_print_changes_summary(summary)
# write changes to the local geopackage
print("Writing DB changes to working dir...")
gpkg_full_path = os.path.join(config.project_working_dir,
config.mergin_sync_file)
_geodiff_apply_changeset("sqlite", "", gpkg_full_path, tmp_changeset_file)
# write to the server
try:
mc.push_project(config.project_working_dir)
except ClientError as e:
# TODO: should we do some cleanup here? (undo changes in the local geopackage?)
raise DbSyncError("Mergin client error on push: " + str(e))
print("Pushed new version to Mergin: " + _get_project_version())
# update base schema in the DB
print("Updating DB base schema...")
_geodiff_apply_changeset(config.db_driver, config.db_conn_info,
config.db_schema_base, tmp_changeset_file)
print("Push done!")
def dbsync_status():
""" Figure out if there are any pending changes in the database or in Mergin """
_check_has_working_dir()
_check_has_sync_file()
_check_has_password()
# get basic information
mp = MerginProject(config.project_working_dir)
if mp.geodiff is None:
raise DbSyncError(
"Mergin client installation problem: geodiff not available")
status_push = mp.get_push_changes()
if status_push['added'] or status_push['updated'] or status_push['removed']:
raise DbSyncError(
"Pending changes in the local directory - that should never happen! "
+ str(status_push))
project_path = mp.metadata["name"]
local_version = mp.metadata["version"]
print("Working directory " + config.project_working_dir)
print("Mergin project " + project_path + " at local version " +
local_version)
print("")
print("Checking status...")
# check if there are any pending changes on server
try:
mc = MerginClient(config.mergin_url,
login=config.mergin_username,
password=config.mergin_password)
server_info = mc.project_info(project_path, since=local_version)
except LoginError as e:
# this could be auth failure, but could be also server problem (e.g. worker crash)
raise DbSyncError("Mergin log in error: " + str(e))
except ClientError as e:
raise DbSyncError("Mergin client error: " + str(e))
print("Server is at version " + server_info["version"])
status_pull = mp.get_pull_changes(server_info["files"])
if status_pull['added'] or status_pull['updated'] or status_pull['removed']:
print("There are pending changes on server:")
_print_mergin_changes(status_pull)
else:
print("No pending changes on server.")
print("")
conn = psycopg2.connect(config.db_conn_info)
if not _check_schema_exists(conn, config.db_schema_base):
raise DbSyncError("The base schema does not exist: " +
config.db_schema_base)
if not _check_schema_exists(conn, config.db_schema_modified):
raise DbSyncError("The 'modified' schema does not exist: " +
config.db_schema_modified)
# get changes in the DB
tmp_dir = tempfile.gettempdir()
tmp_changeset_file = os.path.join(tmp_dir, 'dbsync-status-base2our')
if os.path.exists(tmp_changeset_file):
os.remove(tmp_changeset_file)
_geodiff_create_changeset(config.db_driver, config.db_conn_info,
config.db_schema_base, config.db_schema_modified,
tmp_changeset_file)
if os.path.getsize(tmp_changeset_file) == 0:
print("No changes in the database.")
else:
print("There are changes in DB")
# summarize changes
summary = _geodiff_list_changes_summary(tmp_changeset_file)
_print_changes_summary(summary)
def dbsync_pull():
""" Downloads any changes from Mergin and applies them to the database """
_check_has_working_dir()
_check_has_sync_file()
_check_has_password()
try:
mc = MerginClient(config.mergin_url,
login=config.mergin_username,
password=config.mergin_password)
status_pull, status_push, _ = mc.project_status(
config.project_working_dir)
except LoginError as e:
# this could be auth failure, but could be also server problem (e.g. worker crash)
raise DbSyncError("Mergin log in error: " + str(e))
except ClientError as e:
# this could be e.g. DNS error
raise DbSyncError("Mergin client error: " + str(e))
if not status_pull['added'] and not status_pull[
'updated'] and not status_pull['removed']:
print("No changes on Mergin.")
return
if status_push['added'] or status_push['updated'] or status_push['removed']:
raise DbSyncError(
"There are pending changes in the local directory - that should never happen! "
+ str(status_push))
gpkg_basefile = os.path.join(config.project_working_dir, '.mergin',
config.mergin_sync_file)
gpkg_basefile_old = gpkg_basefile + "-old"
# make a copy of the basefile in the current version (base) - because after pull it will be set to "their"
shutil.copy(gpkg_basefile, gpkg_basefile_old)
tmp_dir = tempfile.gettempdir()
tmp_base2our = os.path.join(tmp_dir, 'dbsync-pull-base2our')
tmp_base2their = os.path.join(tmp_dir, 'dbsync-pull-base2their')
# find out our local changes in the database (base2our)
_geodiff_create_changeset(config.db_driver, config.db_conn_info,
config.db_schema_base, config.db_schema_modified,
tmp_base2our)
needs_rebase = False
if os.path.getsize(tmp_base2our) != 0:
needs_rebase = True
summary = _geodiff_list_changes_summary(tmp_base2our)
_print_changes_summary(summary, "DB Changes:")
try:
mc.pull_project(config.project_working_dir) # will do rebase as needed
except ClientError as e:
# TODO: do we need some cleanup here?
raise DbSyncError("Mergin client error on pull: " + str(e))
print("Pulled new version from Mergin: " + _get_project_version())
# simple case when there are no pending local changes - just apply whatever changes are coming
_geodiff_create_changeset("sqlite", "", gpkg_basefile_old, gpkg_basefile,
tmp_base2their)
# summarize changes
summary = _geodiff_list_changes_summary(tmp_base2their)
_print_changes_summary(summary, "Mergin Changes:")
if not needs_rebase:
print("Applying new version [no rebase]")
_geodiff_apply_changeset(config.db_driver, config.db_conn_info,
config.db_schema_base, tmp_base2their)
_geodiff_apply_changeset(config.db_driver, config.db_conn_info,
config.db_schema_modified, tmp_base2their)
else:
print("Applying new version [WITH rebase]")
tmp_conflicts = os.path.join(tmp_dir, 'dbsync-pull-conflicts')
_geodiff_rebase(config.db_driver, config.db_conn_info,
config.db_schema_base, config.db_schema_modified,
tmp_base2their, tmp_conflicts)
_geodiff_apply_changeset(config.db_driver, config.db_conn_info,
config.db_schema_base, tmp_base2their)
os.remove(gpkg_basefile_old)
print("Pull done!")