def test_tag_basic(runner):
with runner.isolated_filesystem():
os.mkdir('./.lamby')
file = open('./.lamby/log', 'w+')
log_str = """
{
"test.onnx": [
{
"timestamp": 1550098166,
"message": "Test Commit",
"hash": "testhash"
}
]
}
"""
file.write(log_str)
file.close()
result = runner.invoke(tag, ["testhash", "-t", "TestTag"])
assert result.exit_code == 0
log = deserialize_log()
assert log["test.onnx"][0]["tag"] == "TestTag"
def log(files, all, count):
"""Show information about most recent commits of speicified files"""
lamby_dir = './.lamby'
if not os.path.isdir(lamby_dir):
click.echo('Lamby project has not been initialized in ' + os.getcwd())
sys.exit(1)
if (not all) and len(files) == 0:
click.echo('Please include files you want to see the commit logs for' +
' or include -a tag to see commit logs for all files.')
sys.exit(1)
log = deserialize_log()
if all:
files = log.keys()
for f in files:
if f not in log:
click.echo(f + ' cannot be found in your lamby repository.')
sys.exit(1)
for f in files:
click.echo('\nFile: ' + f + '\n')
for i in range(max([0 - count, 0 - len(log[f])]), 0):
click.echo('\tCommit ID: ' + log[f][i]['hash'])
if 'tag' in log[f][i]:
click.echo('\tTag: ' + log[f][i]['tag'])
date = datetime.datetime.fromtimestamp(
log[f][i]['timestamp']).strftime('%a %b %-d %-H:%M:%S %Y %z')
click.echo('\tDate: ' + date)
click.echo('\tMessage: ' + log[f][i]['message'])
click.echo()
sys.exit(0)
def free():
'''Removes all commit objects that are saved locally'''
lamby_dir = './.lamby'
if not os.path.isdir(lamby_dir):
click.echo('Lamby project has not been initialized in ' + os.getcwd())
sys.exit(1)
config = deserialize_config()
if 'project_id' not in config:
click.echo('No Lamby project detected.')
sys.exit(1)
project_id = deserialize_config()['project_id']
log = deserialize_log()
commits_in_log = set()
for k in log:
for c in log[k]:
commits_in_log.add(c['hash'])
res = get_request('/api/projects/{}'.format(project_id))
res_json = res.json()
for commit_hash in commits_in_log:
if commit_hash not in res_json['commits']:
click.echo('Local repository not in sync with remote repository.')
click.echo('Please use lamby push before freeing commit objects.')
sys.exit(1)
folder = os.path.join(lamby_dir, 'commit_objects')
for the_file in os.listdir(folder):
file_path = os.path.join(folder, the_file)
if os.path.isfile(file_path):
os.unlink(file_path)
def test_checkout_basic(runner):
with runner.isolated_filesystem():
assert runner.invoke(init).exit_code == 0
filename = 'file1.onnx'
create_file(filename, 100)
shutil.copyfile(filename, './copy')
assert runner.invoke(commit, [filename]).exit_code == 0
mutate_file(filename, 100)
assert runner.invoke(commit, [filename]).exit_code == 0
log = deserialize_log()
checkout_hash = log[filename][0]['hash']
result = runner.invoke(checkout, [checkout_hash])
meta = deserialize_meta()
assert result.exit_code == 0
assert cmp_files('./copy', filename)
assert meta['file_head'][filename]['hash'] == checkout_hash
assert meta['file_head'][filename]['index'] == 0
def tag(commits, tag):
"""Tags a specific commit with given tag in the version history."""
lamby_dir = './.lamby'
if not os.path.isdir(lamby_dir):
click.echo('Lamby project has not been initialized in ' + os.getcwd())
sys.exit(1)
if tag is None:
# Not listing and didn't provide a tag
click.echo('Please include a tag using -t <tag>')
sys.exit(1)
if len(commits) == 0:
click.echo('Please indicate commit ID of commit to tag.')
sys.exit(1)
else:
click.echo('Tagging following commit(s):')
log = deserialize_log()
for filename in log.keys():
log_commits = log[filename]
for commit_dict in log_commits:
if commit_dict["hash"] in commits:
click.echo("\t" + commit_dict["hash"][0:6] + ": " +
commit_dict["message"])
commit_dict["tag"] = tag
click.echo('Tag: ' + tag)
serialize_log(log)
sys.exit(0)
def test_commit_no_spec_file(runner):
with runner.isolated_filesystem():
lamby_dir = './.lamby'
os.mkdir(lamby_dir)
os.mkdir(lamby_dir + '/commit_objects')
config_file = open(lamby_dir + '/config', "w+")
config_file.write('{}')
config_file.close()
log_file = open(lamby_dir + '/log', "w+")
log_file.write('{}')
log_file.close()
meta_file = open(lamby_dir + '/meta', "w+")
meta_file.write('''{
\"file_head\": {},
\"latest_commit\": {}
}''')
meta_file.close()
os.mkdir('dir1')
create_file('file1.onnx', 100)
create_file('dir1/file2.onnx', 100)
result = runner.invoke(commit, ['-m', 'Foo Bar!'])
assert result.exit_code == 0
log_file = deserialize_log()
compressed_filename = './.lamby/commit_objects/' + \
log_file['file1.onnx'][0]['hash']
uncompressed_filename = log_file['file1.onnx'][0]['hash']
assert os.path.isfile(compressed_filename)
copy_file(compressed_filename, uncompressed_filename)
assert cmp_files('file1.onnx', uncompressed_filename)
compressed_filename = './.lamby/commit_objects/' + \
log_file['file2.onnx'][0]['hash']
uncompressed_filename = log_file['file2.onnx'][0]['hash']
assert os.path.isfile(compressed_filename)
copy_file(compressed_filename, uncompressed_filename)
assert cmp_files('dir1/file2.onnx', uncompressed_filename)
def checkout(hash):
'''Checks out the binary files associated with a commit hash'''
meta = deserialize_meta()
log = deserialize_log()
results = []
for file_name in log.keys():
for ci, commit in enumerate(log[file_name]):
if commit['hash'].startswith(hash):
results.append((file_name, ci, commit['hash']))
if len(results) == 0:
click.echo('Commit hash not found')
sys.exit(1)
elif len(results) > 1:
for result in results:
click.echo('{} [{}]'.format(result[0], result[2]))
sys.exit(0)
result = results[0]
result_name = result[0]
result_index = result[1]
result_hash = result[2]
if result_name in meta['file_head'] \
and meta['file_head'][result_name]['hash'] == result_hash:
click.echo('Hash is currently head')
else:
fetch_commit(result_hash)
fetch_commit(meta['file_head'][result_name]['hash'])
file_search_results = search_pattern('./**/' + result_name)
# TODO: add check for duplicate filenames
file_path = file_search_results[0]
if not diff_files(
file_path, './.lamby/commit_objects/' +
meta['file_head'][result_name]['hash']):
click.echo('Cannot checkout with uncommitted changes')
sys.exit(1)
copy_file('./.lamby/commit_objects/' + result_hash, file_path)
meta['file_head'][result_name] = {
'hash': result_hash,
'index': result_index
}
serialize_meta(meta)
def test_checkout_uncommitted_changes(runner):
with runner.isolated_filesystem():
assert runner.invoke(init).exit_code == 0
filename = 'file1.onnx'
create_file(filename, 100)
assert runner.invoke(commit, [filename]).exit_code == 0
mutate_file(filename, 100)
assert runner.invoke(commit, [filename]).exit_code == 0
mutate_file(filename, 100)
log = deserialize_log()
result = runner.invoke(checkout, [log[filename][0]['hash']])
assert result.exit_code == 1
assert result.output == 'Cannot checkout with uncommitted changes\n'
def test_commit_basic(runner):
with runner.isolated_filesystem():
lamby_dir = './.lamby'
os.mkdir(lamby_dir)
os.mkdir(lamby_dir + '/commit_objects')
config_file = open(lamby_dir + '/config', "w+")
config_file.write('{}')
config_file.close()
log_file = open(lamby_dir + '/log', "w+")
log_file.write('{}')
log_file.close()
meta_file = open(lamby_dir + '/meta', "w+")
meta_file.write('''{
\"file_head\": {},
\"latest_commit\": {}
}''')
meta_file.close()
filename = 'file1.onnx'
message = 'Foo Bar!'
create_file(filename, 100)
result = runner.invoke(commit, [filename, '-m', message])
assert result.exit_code == 0
log_file = deserialize_log()
compressed_filename = './.lamby/commit_objects/' + \
log_file[filename][0]['hash']
uncompressed_filename = log_file[filename][0]['hash']
assert log_file[filename][0]['message'] == message
assert os.path.isfile(compressed_filename)
copy_file(compressed_filename, uncompressed_filename)
assert cmp_files(filename, uncompressed_filename)
def rename(file_original, file_rename):
"""" Renames a file being tracked by the system in all instances """
if file_rename.split('.')[-1] != 'onnx':
click.echo(file_rename + ' is not an onnx file')
sys.exit(1)
lamby_dir = './.lamby'
if not os.path.isdir(lamby_dir):
click.echo('Lamby project has not been initialized in ' + os.getcwd())
sys.exit(1)
log = deserialize_log()
click.echo('Renaming ' + file_original + ' to ' + file_rename)
if file_original not in log:
click.echo(file_original + ' cannot be found in your lamby repo')
sys.exit(1)
if file_rename in log:
click.echo(file_rename + ' already exists in your lamby repo')
sys.exit(1)
log[file_rename] = log[file_original]
del log[file_original]
serialize_log(log)
meta = deserialize_meta()
if file_original in meta['file_head']:
meta['file_head'][file_rename] = meta['file_head'][file_original]
del meta['file_head'][file_original]
if file_original in meta['latest_commit']:
meta['latest_commit'][file_rename] = meta[
'latest_commit'][file_original]
del meta['latest_commit'][file_original]
serialize_meta(meta)
click.echo('Rename successful')
def test_status_previous(runner):
with runner.isolated_filesystem():
runner.invoke(init)
create_file('f1.onnx', 100)
result = runner.invoke(commit)
assert result.exit_code == 0
create_file('f1.onnx', 200)
result = runner.invoke(commit)
assert result.exit_code == 0
log = deserialize_log()
checkout_hash = log['f1.onnx'][0]['hash']
result = runner.invoke(checkout, [checkout_hash])
assert result.exit_code == 0
result = runner.invoke(status)
assert result.exit_code == 0
def push():
'''Pushes all new commits \
from local repository to remote Lamby repository'''
lamby_dir = './.lamby'
if not os.path.isdir(lamby_dir):
click.echo('Lamby project has not been initialized in ' + os.getcwd())
sys.exit(1)
gc_path = os.path.dirname(os.path.abspath(__file__)) + '/.config'
if not os.path.isfile(gc_path):
click.echo('No authorization found. Please run lamby auth.')
sys.exit(1)
with open(gc_path) as global_config:
gc_json = json.load(global_config)
log = deserialize_log()
payload = {'log': log}
header = {'x-auth': gc_json['api_key']}
config = deserialize_config()
if 'project_id' not in config:
click.echo('No Lamby project detected.')
sys.exit(1)
project_id = deserialize_config()['project_id']
try:
res = requests.post(os.getenv('LAMBY_WEB_URI') +
'/api/projects/status/{}'.format(project_id),
json=payload,
headers=header)
except requests.exceptions.ConnectionError:
click.echo('Could not reach lamby web. Aborting push.')
sys.exit(1)
except requests.exceptions.Timeout:
click.echo('Connection timed out. Aborting push.')
sys.exit(1)
except requests.exceptions.TooManyRedirects:
click.echo('Too many redirects to reach lamby web. Aborting push.')
sys.exit(1)
except requests.exceptions.RequestException as e:
click.echo(e)
sys.exit(1)
res_json = res.json()
commits_to_upload = res_json['commits_to_upload']
from lamby.filestore import fs
for filename in commits_to_upload:
for commit in commits_to_upload[filename]:
fs.upload_file(lamby_dir + '/commit_objects/' + commit['hash'],
project_id)
payload = res_json
payload['meta'] = deserialize_meta()
try:
res = requests.post(os.getenv('LAMBY_WEB_URI') +
'/api/projects/push/{}'.format(project_id),
json=payload,
headers=header)
except requests.exceptions.ConnectionError:
click.echo('Could not reach lamby web. Aborting push.')
sys.exit(1)
except requests.exceptions.Timeout:
click.echo('Connection timed out. Aborting push.')
sys.exit(1)
except requests.exceptions.TooManyRedirects:
click.echo('Too many redirects to reach lamby web. Aborting push.')
sys.exit(1)
except requests.exceptions.RequestException as e:
click.echo(e)
sys.exit(1)
click.echo(res.json()['message'])
sys.exit(0)
def commit(files, message):
"""Commits changes made to the relevant files to the Lamby system"""
message = "" if message is None else message
lamby_dir = './.lamby'
if not os.path.isdir(lamby_dir):
click.echo('Lamby project has not been initialized in ' + os.getcwd())
sys.exit(1)
files = search_file_type('.', 'onnx') if len(files) == 0 else files
if len(files) == 0:
click.echo('There are no onnx files in the project directory.')
sys.exit(1)
log = deserialize_log()
meta = deserialize_meta()
file_errors = False
for file in files:
basename = os.path.basename(file)
if not os.path.isfile(file):
click.echo(file + ' is not a file')
file_errors = True
if file.split('.')[-1] != 'onnx':
click.echo(file + ' is not an onnx file')
file_errors = True
if basename in log and diff_files(
file, './.lamby/commit_objects/' + log[basename][-1]['hash']):
click.echo(file + ' has no changes to commit')
file_errors = True
if file_errors:
sys.exit(1)
for file in files:
basename = os.path.basename(file)
if basename not in log:
log[basename] = []
commit_record = {}
commit_record["timestamp"] = int(time.time())
commit_record["message"] = str(message)
str_to_hash = basename
str_to_hash += str(commit_record["timestamp"])
str_to_hash += commit_record["message"]
str_to_hash += file_sha256(file)
if len(log[basename]) > 0:
str_to_hash += log[basename][-1]["hash"]
str_to_hash = str_to_hash.encode("utf-8")
hash_gen = hashlib.sha256(str_to_hash).hexdigest()
commit_record["hash"] = hash_gen
log[basename].append(commit_record)
meta['file_head'][basename] = {
'hash': hash_gen,
'index': len(log[basename]) - 1
}
meta['latest_commit'][basename] = hash_gen
copy_file(file, './.lamby/commit_objects/' + hash_gen)
serialize_log(log)
serialize_meta(meta)
click.echo('Committed the following files:')
for file in files:
click.echo('\t' + os.path.basename(file))
click.echo("Commit message: \"" + message + "\"")