def test_tag_basic(runner): with runner.isolated_filesystem(): os.mkdir('./.lamby') file = open('./.lamby/log', 'w+') log_str = """ { "test.onnx": [ { "timestamp": 1550098166, "message": "Test Commit", "hash": "testhash" } ] } """ file.write(log_str) file.close() result = runner.invoke(tag, ["testhash", "-t", "TestTag"]) assert result.exit_code == 0 log = deserialize_log() assert log["test.onnx"][0]["tag"] == "TestTag"
def log(files, all, count): """Show information about most recent commits of speicified files""" lamby_dir = './.lamby' if not os.path.isdir(lamby_dir): click.echo('Lamby project has not been initialized in ' + os.getcwd()) sys.exit(1) if (not all) and len(files) == 0: click.echo('Please include files you want to see the commit logs for' + ' or include -a tag to see commit logs for all files.') sys.exit(1) log = deserialize_log() if all: files = log.keys() for f in files: if f not in log: click.echo(f + ' cannot be found in your lamby repository.') sys.exit(1) for f in files: click.echo('\nFile: ' + f + '\n') for i in range(max([0 - count, 0 - len(log[f])]), 0): click.echo('\tCommit ID: ' + log[f][i]['hash']) if 'tag' in log[f][i]: click.echo('\tTag: ' + log[f][i]['tag']) date = datetime.datetime.fromtimestamp( log[f][i]['timestamp']).strftime('%a %b %-d %-H:%M:%S %Y %z') click.echo('\tDate: ' + date) click.echo('\tMessage: ' + log[f][i]['message']) click.echo() sys.exit(0)
def free(): '''Removes all commit objects that are saved locally''' lamby_dir = './.lamby' if not os.path.isdir(lamby_dir): click.echo('Lamby project has not been initialized in ' + os.getcwd()) sys.exit(1) config = deserialize_config() if 'project_id' not in config: click.echo('No Lamby project detected.') sys.exit(1) project_id = deserialize_config()['project_id'] log = deserialize_log() commits_in_log = set() for k in log: for c in log[k]: commits_in_log.add(c['hash']) res = get_request('/api/projects/{}'.format(project_id)) res_json = res.json() for commit_hash in commits_in_log: if commit_hash not in res_json['commits']: click.echo('Local repository not in sync with remote repository.') click.echo('Please use lamby push before freeing commit objects.') sys.exit(1) folder = os.path.join(lamby_dir, 'commit_objects') for the_file in os.listdir(folder): file_path = os.path.join(folder, the_file) if os.path.isfile(file_path): os.unlink(file_path)
def test_checkout_basic(runner): with runner.isolated_filesystem(): assert runner.invoke(init).exit_code == 0 filename = 'file1.onnx' create_file(filename, 100) shutil.copyfile(filename, './copy') assert runner.invoke(commit, [filename]).exit_code == 0 mutate_file(filename, 100) assert runner.invoke(commit, [filename]).exit_code == 0 log = deserialize_log() checkout_hash = log[filename][0]['hash'] result = runner.invoke(checkout, [checkout_hash]) meta = deserialize_meta() assert result.exit_code == 0 assert cmp_files('./copy', filename) assert meta['file_head'][filename]['hash'] == checkout_hash assert meta['file_head'][filename]['index'] == 0
def tag(commits, tag): """Tags a specific commit with given tag in the version history.""" lamby_dir = './.lamby' if not os.path.isdir(lamby_dir): click.echo('Lamby project has not been initialized in ' + os.getcwd()) sys.exit(1) if tag is None: # Not listing and didn't provide a tag click.echo('Please include a tag using -t <tag>') sys.exit(1) if len(commits) == 0: click.echo('Please indicate commit ID of commit to tag.') sys.exit(1) else: click.echo('Tagging following commit(s):') log = deserialize_log() for filename in log.keys(): log_commits = log[filename] for commit_dict in log_commits: if commit_dict["hash"] in commits: click.echo("\t" + commit_dict["hash"][0:6] + ": " + commit_dict["message"]) commit_dict["tag"] = tag click.echo('Tag: ' + tag) serialize_log(log) sys.exit(0)
def test_commit_no_spec_file(runner): with runner.isolated_filesystem(): lamby_dir = './.lamby' os.mkdir(lamby_dir) os.mkdir(lamby_dir + '/commit_objects') config_file = open(lamby_dir + '/config', "w+") config_file.write('{}') config_file.close() log_file = open(lamby_dir + '/log', "w+") log_file.write('{}') log_file.close() meta_file = open(lamby_dir + '/meta', "w+") meta_file.write('''{ \"file_head\": {}, \"latest_commit\": {} }''') meta_file.close() os.mkdir('dir1') create_file('file1.onnx', 100) create_file('dir1/file2.onnx', 100) result = runner.invoke(commit, ['-m', 'Foo Bar!']) assert result.exit_code == 0 log_file = deserialize_log() compressed_filename = './.lamby/commit_objects/' + \ log_file['file1.onnx'][0]['hash'] uncompressed_filename = log_file['file1.onnx'][0]['hash'] assert os.path.isfile(compressed_filename) copy_file(compressed_filename, uncompressed_filename) assert cmp_files('file1.onnx', uncompressed_filename) compressed_filename = './.lamby/commit_objects/' + \ log_file['file2.onnx'][0]['hash'] uncompressed_filename = log_file['file2.onnx'][0]['hash'] assert os.path.isfile(compressed_filename) copy_file(compressed_filename, uncompressed_filename) assert cmp_files('dir1/file2.onnx', uncompressed_filename)
def checkout(hash): '''Checks out the binary files associated with a commit hash''' meta = deserialize_meta() log = deserialize_log() results = [] for file_name in log.keys(): for ci, commit in enumerate(log[file_name]): if commit['hash'].startswith(hash): results.append((file_name, ci, commit['hash'])) if len(results) == 0: click.echo('Commit hash not found') sys.exit(1) elif len(results) > 1: for result in results: click.echo('{} [{}]'.format(result[0], result[2])) sys.exit(0) result = results[0] result_name = result[0] result_index = result[1] result_hash = result[2] if result_name in meta['file_head'] \ and meta['file_head'][result_name]['hash'] == result_hash: click.echo('Hash is currently head') else: fetch_commit(result_hash) fetch_commit(meta['file_head'][result_name]['hash']) file_search_results = search_pattern('./**/' + result_name) # TODO: add check for duplicate filenames file_path = file_search_results[0] if not diff_files( file_path, './.lamby/commit_objects/' + meta['file_head'][result_name]['hash']): click.echo('Cannot checkout with uncommitted changes') sys.exit(1) copy_file('./.lamby/commit_objects/' + result_hash, file_path) meta['file_head'][result_name] = { 'hash': result_hash, 'index': result_index } serialize_meta(meta)
def test_checkout_uncommitted_changes(runner): with runner.isolated_filesystem(): assert runner.invoke(init).exit_code == 0 filename = 'file1.onnx' create_file(filename, 100) assert runner.invoke(commit, [filename]).exit_code == 0 mutate_file(filename, 100) assert runner.invoke(commit, [filename]).exit_code == 0 mutate_file(filename, 100) log = deserialize_log() result = runner.invoke(checkout, [log[filename][0]['hash']]) assert result.exit_code == 1 assert result.output == 'Cannot checkout with uncommitted changes\n'
def test_commit_basic(runner): with runner.isolated_filesystem(): lamby_dir = './.lamby' os.mkdir(lamby_dir) os.mkdir(lamby_dir + '/commit_objects') config_file = open(lamby_dir + '/config', "w+") config_file.write('{}') config_file.close() log_file = open(lamby_dir + '/log', "w+") log_file.write('{}') log_file.close() meta_file = open(lamby_dir + '/meta', "w+") meta_file.write('''{ \"file_head\": {}, \"latest_commit\": {} }''') meta_file.close() filename = 'file1.onnx' message = 'Foo Bar!' create_file(filename, 100) result = runner.invoke(commit, [filename, '-m', message]) assert result.exit_code == 0 log_file = deserialize_log() compressed_filename = './.lamby/commit_objects/' + \ log_file[filename][0]['hash'] uncompressed_filename = log_file[filename][0]['hash'] assert log_file[filename][0]['message'] == message assert os.path.isfile(compressed_filename) copy_file(compressed_filename, uncompressed_filename) assert cmp_files(filename, uncompressed_filename)
def rename(file_original, file_rename): """" Renames a file being tracked by the system in all instances """ if file_rename.split('.')[-1] != 'onnx': click.echo(file_rename + ' is not an onnx file') sys.exit(1) lamby_dir = './.lamby' if not os.path.isdir(lamby_dir): click.echo('Lamby project has not been initialized in ' + os.getcwd()) sys.exit(1) log = deserialize_log() click.echo('Renaming ' + file_original + ' to ' + file_rename) if file_original not in log: click.echo(file_original + ' cannot be found in your lamby repo') sys.exit(1) if file_rename in log: click.echo(file_rename + ' already exists in your lamby repo') sys.exit(1) log[file_rename] = log[file_original] del log[file_original] serialize_log(log) meta = deserialize_meta() if file_original in meta['file_head']: meta['file_head'][file_rename] = meta['file_head'][file_original] del meta['file_head'][file_original] if file_original in meta['latest_commit']: meta['latest_commit'][file_rename] = meta[ 'latest_commit'][file_original] del meta['latest_commit'][file_original] serialize_meta(meta) click.echo('Rename successful')
def test_status_previous(runner): with runner.isolated_filesystem(): runner.invoke(init) create_file('f1.onnx', 100) result = runner.invoke(commit) assert result.exit_code == 0 create_file('f1.onnx', 200) result = runner.invoke(commit) assert result.exit_code == 0 log = deserialize_log() checkout_hash = log['f1.onnx'][0]['hash'] result = runner.invoke(checkout, [checkout_hash]) assert result.exit_code == 0 result = runner.invoke(status) assert result.exit_code == 0
def push(): '''Pushes all new commits \ from local repository to remote Lamby repository''' lamby_dir = './.lamby' if not os.path.isdir(lamby_dir): click.echo('Lamby project has not been initialized in ' + os.getcwd()) sys.exit(1) gc_path = os.path.dirname(os.path.abspath(__file__)) + '/.config' if not os.path.isfile(gc_path): click.echo('No authorization found. Please run lamby auth.') sys.exit(1) with open(gc_path) as global_config: gc_json = json.load(global_config) log = deserialize_log() payload = {'log': log} header = {'x-auth': gc_json['api_key']} config = deserialize_config() if 'project_id' not in config: click.echo('No Lamby project detected.') sys.exit(1) project_id = deserialize_config()['project_id'] try: res = requests.post(os.getenv('LAMBY_WEB_URI') + '/api/projects/status/{}'.format(project_id), json=payload, headers=header) except requests.exceptions.ConnectionError: click.echo('Could not reach lamby web. Aborting push.') sys.exit(1) except requests.exceptions.Timeout: click.echo('Connection timed out. Aborting push.') sys.exit(1) except requests.exceptions.TooManyRedirects: click.echo('Too many redirects to reach lamby web. Aborting push.') sys.exit(1) except requests.exceptions.RequestException as e: click.echo(e) sys.exit(1) res_json = res.json() commits_to_upload = res_json['commits_to_upload'] from lamby.filestore import fs for filename in commits_to_upload: for commit in commits_to_upload[filename]: fs.upload_file(lamby_dir + '/commit_objects/' + commit['hash'], project_id) payload = res_json payload['meta'] = deserialize_meta() try: res = requests.post(os.getenv('LAMBY_WEB_URI') + '/api/projects/push/{}'.format(project_id), json=payload, headers=header) except requests.exceptions.ConnectionError: click.echo('Could not reach lamby web. Aborting push.') sys.exit(1) except requests.exceptions.Timeout: click.echo('Connection timed out. Aborting push.') sys.exit(1) except requests.exceptions.TooManyRedirects: click.echo('Too many redirects to reach lamby web. Aborting push.') sys.exit(1) except requests.exceptions.RequestException as e: click.echo(e) sys.exit(1) click.echo(res.json()['message']) sys.exit(0)
def commit(files, message): """Commits changes made to the relevant files to the Lamby system""" message = "" if message is None else message lamby_dir = './.lamby' if not os.path.isdir(lamby_dir): click.echo('Lamby project has not been initialized in ' + os.getcwd()) sys.exit(1) files = search_file_type('.', 'onnx') if len(files) == 0 else files if len(files) == 0: click.echo('There are no onnx files in the project directory.') sys.exit(1) log = deserialize_log() meta = deserialize_meta() file_errors = False for file in files: basename = os.path.basename(file) if not os.path.isfile(file): click.echo(file + ' is not a file') file_errors = True if file.split('.')[-1] != 'onnx': click.echo(file + ' is not an onnx file') file_errors = True if basename in log and diff_files( file, './.lamby/commit_objects/' + log[basename][-1]['hash']): click.echo(file + ' has no changes to commit') file_errors = True if file_errors: sys.exit(1) for file in files: basename = os.path.basename(file) if basename not in log: log[basename] = [] commit_record = {} commit_record["timestamp"] = int(time.time()) commit_record["message"] = str(message) str_to_hash = basename str_to_hash += str(commit_record["timestamp"]) str_to_hash += commit_record["message"] str_to_hash += file_sha256(file) if len(log[basename]) > 0: str_to_hash += log[basename][-1]["hash"] str_to_hash = str_to_hash.encode("utf-8") hash_gen = hashlib.sha256(str_to_hash).hexdigest() commit_record["hash"] = hash_gen log[basename].append(commit_record) meta['file_head'][basename] = { 'hash': hash_gen, 'index': len(log[basename]) - 1 } meta['latest_commit'][basename] = hash_gen copy_file(file, './.lamby/commit_objects/' + hash_gen) serialize_log(log) serialize_meta(meta) click.echo('Committed the following files:') for file in files: click.echo('\t' + os.path.basename(file)) click.echo("Commit message: \"" + message + "\"")