def with_exclusive_lock():
if not varify_user_lock(repository_path, session_token):
return fail(lock_fail_msg)
#===
data_store = versioned_storage(repository_path)
if not data_store.have_active_commit():
return fail(no_active_commit_msg)
# There is no valid reason for path traversal characters to be in a file path within this system
file_path = request.headers['path']
if any(True for item in re.split(r'\\|/', file_path)
if item in ['..', '.']):
return fail()
#===
tmp_path = cpjoin(repository_path, 'tmp_file')
with open(tmp_path, 'wb') as f:
while True:
chunk = request.body.read(1000 * 1000)
if chunk is None: break
f.write(chunk)
#===
data_store.fs_put_from_file(tmp_path, {'path': file_path})
# updates the user lock expiry
update_user_lock(repository_path, session_token)
return success()
def with_exclusive_lock():
# The commit is locked for a given time period to a given session token,
# a client must hold this lock to use any of push_file(), delete_files() and commit().
# It does not matter if the user lock technically expires while a client is writing
# a large file, as the user lock is locked using flock for the duration of any
# operation and thus cannot be stolen by another client. It is updated to be in
# the future before returning to the client. The lock only needs to survive until
# the client owning the lock sends another request and re acquires the flock.
if not can_aquire_user_lock(repository_path, session_token):
return fail(lock_fail_msg)
# Commits can only take place if the committing user has the latest revision,
# as committing from an outdated state could cause unexpected results, and may
# have conflicts. Conflicts are resolved during a client update so they are
# handled by the client, and a server interface for this is not needed.
data_store = versioned_storage(repository_path)
if data_store.get_head() != request.headers["previous_revision"]:
return fail(need_to_update_msg)
# Should the lock expire, the client which had the lock previously will be unable
# to continue the commit it had in progress. When this, or another client attempts
# to commit again it must do so by first obtaining the lock again by calling begin_commit().
# Any remaining commit data from failed prior commits is garbage collected here.
# While it would technically be possible to implement commit resume should the same
# client resume, I only see commits failing due to a network error and this is so
# rare I don't think it's worth the trouble.
if data_store.have_active_commit(): data_store.rollback()
#------------
data_store.begin()
update_user_lock(repository_path, session_token)
return success()
def list_versions(request: Request) -> Responce:
session_token = request.headers['session_token'].encode('utf8')
repository = request.headers['repository']
#===
current_user = have_authenticated_user(request.remote_addr, repository,
session_token)
if current_user is False: return fail(user_auth_fail_msg)
#===
data_store = versioned_storage(config['repositories'][repository]['path'])
return success({}, {'versions': data_store.get_commit_chain()})
def find_changed(request: Request) -> Responce:
""" Find changes since the revision it is currently holding """
session_token = request.headers['session_token'].encode('utf8')
repository = request.headers['repository']
#===
current_user = have_authenticated_user(request.remote_addr, repository,
session_token)
if current_user is False: return fail(user_auth_fail_msg)
#===
repository_path = config['repositories'][repository]['path']
body_data = request.get_json()
#===
data_store = versioned_storage(repository_path)
head = data_store.get_head()
if head == 'root':
return success({}, {'head': 'root', 'sorted_changes': {'none': []}})
# Find changed items
client_changes = json.loads(body_data['client_changes'])
server_changes = data_store.get_changes_since(
request.headers["previous_revision"], head)
# Resolve conflicts
conflict_resolutions = json.loads(body_data['conflict_resolutions'])
if conflict_resolutions != []:
resolutions = {'server': {}, 'client': {}} # type: ignore
for r in conflict_resolutions:
if len(r['4_resolution']) != 1 or r['4_resolution'][0] not in [
'client', 'server'
]:
return fail(conflict_msg)
resolutions[r['4_resolution'][0]][r['1_path']] = None
client_changes = {
k: v
for k, v in client_changes.items()
if v['path'] not in resolutions['server']
}
server_changes = {
k: v
for k, v in server_changes.items()
if v['path'] not in resolutions['client']
}
sorted_changes = merge_client_and_server_changes(server_changes,
client_changes)
return success({}, {'head': head, 'sorted_changes': sorted_changes})
def test_get_changes_since(self):
file_put_contents(cpjoin(DATA_DIR, 'test 1'), b'test')
file_put_contents(cpjoin(DATA_DIR, 'test 2'), b'test 1')
file_put_contents(cpjoin(DATA_DIR, 'test 3'), b'test 2')
#==================
data_store = versioned_storage(DATA_DIR)
data_store.begin()
data_store.fs_put_from_file(cpjoin(DATA_DIR, 'test 1'),
{'path': '/test/path'})
id1 = data_store.commit('test msg', 'test user')
changes = data_store.get_changes_since('root', data_store.get_head())
self.assertEqual(
changes, {
'/test/path': {
'hash':
'9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08',
'path': '/test/path',
'status': 'new'
}
})
#==================
data_store.begin()
data_store.fs_put_from_file(cpjoin(DATA_DIR, 'test 2'),
{'path': '/another/path'})
data_store.fs_put_from_file(cpjoin(DATA_DIR, 'test 3'),
{'path': '/yet/another/path'})
data_store.commit('test msg', 'test user')
changes = data_store.get_changes_since(id1, data_store.get_head())
self.assertEqual(
changes, {
'/another/path': {
'hash':
'f67213b122a5d442d2b93bda8cc45c564a70ec5d2a4e0e95bb585cf199869c98',
'path': '/another/path',
'status': 'new'
},
'/yet/another/path': {
'hash':
'dec2e4bc4992314a9c9a51bbd859e1b081b74178818c53c19d18d6f761f5d804',
'path': '/yet/another/path',
'status': 'new'
}
})
def with_exclusive_lock():
if not varify_user_lock(repository_path, session_token):
return fail(lock_fail_msg)
try:
data_store = versioned_storage(repository_path)
if not data_store.have_active_commit():
return fail(no_active_commit_msg)
#-------------
for fle in json.loads(body_data['files']):
data_store.fs_delete(fle)
# updates the user lock expiry
update_user_lock(repository_path, session_token)
return success()
except Exception:
return fail() # pylint: disable=broad-except
def with_exclusive_lock():
if not varify_user_lock(repository_path, session_token):
return fail(lock_fail_msg)
#===
data_store = versioned_storage(repository_path)
if not data_store.have_active_commit():
return fail(no_active_commit_msg)
result = {}
if request.headers['mode'] == 'commit':
new_head = data_store.commit(request.headers['commit_message'],
current_user['username'])
result = {'head': new_head}
else:
data_store.rollback()
# Release the user lock
update_user_lock(repository_path, None)
return success(result)
def pull_file(request: Request) -> Responce:
""" Get a file from the server """
session_token = request.headers['session_token'].encode('utf8')
repository = request.headers['repository']
#===
current_user = have_authenticated_user(request.remote_addr, repository,
session_token)
if current_user is False: return fail(user_auth_fail_msg)
#===
data_store = versioned_storage(config['repositories'][repository]['path'])
file_info = data_store.get_file_info_from_path(request.headers['path'])
full_file_path: str = cpjoin(
data_store.get_file_directory_path(file_info['hash']),
file_info['hash'][2:])
return success({'file_info_json': json.dumps(file_info)},
ServeFile(full_file_path))
def test_rollback(self):
file_put_contents(cpjoin(DATA_DIR, 'test 1'), b'test')
file_put_contents(cpjoin(DATA_DIR, 'test 2'), b'test')
file_put_contents(cpjoin(DATA_DIR, 'test 3'), b'test 2')
#==================
data_store = versioned_storage(DATA_DIR)
data_store.begin()
data_store.fs_put_from_file(cpjoin(DATA_DIR, 'test 1'),
{'path': '/test/path'})
data_store.commit('test msg', 'test user')
data_store.begin()
data_store.fs_put_from_file(cpjoin(DATA_DIR, 'test 2'),
{'path': '/another/path'})
data_store.fs_put_from_file(cpjoin(DATA_DIR, 'test 3'),
{'path': '/yet/another/path'})
data_store.rollback()
self.assertEqual(os.listdir(cpjoin(DATA_DIR, 'files')), ['9f'])
self.assertEqual(
os.listdir(cpjoin(DATA_DIR, 'files', '9f')),
['86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08'])