def init_git(self, local_git_path, repo_url):
try:
self._local_git_path = local_git_path
self._repo_url = repo_url
self._repo = Git(self._local_git_path, self._repo_url)
self._repo.pull()
return True
except:
return False
def setup():
"""Setup function for tests."""
# global variable shouldn't be used but is quite useful here
# pylint: disable=W0603
global TEMP_DIR
TEMP_DIR = mkdtemp()
if not os.path.exists(FAKE_DIR):
os.mkdir(FAKE_DIR)
Git(os.path.join(FAKE_DIR, 'documents.git')).init()
app = Flask(
__name__,
static_folder=os.path.join(PATH, 'static'),
template_folder=os.path.join(PATH, 'templates'))
app.config.from_object(config)
app.db = SQLAlchemy(app)
nuts = application.Koztumize(app)
application.app = app
application.nuts = nuts
app.config['MODELS'] = os.path.join(TEMP_DIR, 'models')
git = Git(app.config['MODELS'])
git.init()
git.remote('add', '-t', 'models', 'origin', app.config['GIT_REMOTE'])
git.pull()
git.checkout('models')
execute_sql(app, 'db.sql')
import koztumize.routes
import koztumize.tests.document
def setup():
"""Set up the git repository for the all the tests"""
# global variable shouldn't be used but is quite useful here
# pylint: disable=W0603
global TEMP_DIR
TEMP_DIR = mkdtemp()
koztumize.app.config.from_pyfile(
os.environ.get('KOZTUMIZE_CONFIG', 'test/config_test.py'))
koztumize.db_model.init(koztumize.app)
Git.push = lambda *args, **kwargs: None
for repo in ('archive', 'model'):
koztumize.app.config[repo.upper()] = os.path.join(TEMP_DIR, repo)
git = Git(koztumize.app.config[repo.upper()])
git.init()
git.remote(
'add', '-t', repo, 'origin',
koztumize.app.config['GIT_REMOTE'])
git.pull()
git.checkout(repo)
class GitFile:
def __init__(self, path):
self.__path = path
self.__new = True
self.__git = Git(dirname(path))
def read(self):
if exists(self.__path):
with open(self.__path, 'r') as input:
for line in input:
line = line.strip()
if line and not line.startswith('#'):
epoch, sentence = line.split(' ', 1)
yield int(epoch), sentence
self.__new = False
def __header(self):
with open(self.__path, 'w') as output:
output.write('''
# each non-comment line contains a unix epoch, followed by a sentence,
# encoded in ascii, and terminated by a newline character (unix style).
# the epoch is the time at which the sentence was created. obviously there
# may be some delay before it is visible. sentences 'end' when the next one
# is created. the gap is typically of order %d seconds, but is not
# guaranteed (there is a random - stochastic - component as well as the
# possibility of service downtimes, etc).
# blank lines and those starting with a # are comments.
''' % PERIOD)
def __push(self):
try:
self.__git.add(self.__path)
self.__git.commit(self.__path, m='new sentence')
self.__git.push()
except GitException as e:
eprint(e)
def write(self, append, epoch, sentence):
if self.__new:
self.__header()
self.__push()
self.__new = False
if append:
with open(self.__path, 'a') as output:
output.write("%d %s\n" % (epoch, sentence))
self.__push()
def test_basic():
shutil.rmtree("/tmp/brigit_test", ignore_errors=True)
git = Git("/tmp/brigit_test")
with open("/tmp/brigit_test/file_1", "w") as f:
f.write('1')
git.add("/tmp/brigit_test/file_1")
git.commit(message="Adding file_1")
assert "Adding file_1" in git.log()
assert len(list(git.pretty_log())) == 1
with open("/tmp/brigit_test/file_2", "w") as f:
f.write('2')
git.add("/tmp/brigit_test/file_2")
git.commit(message="Adding file_2")
assert "Adding file_2" in git.log()
assert len(list(git.pretty_log())) == 2
git.reset("HEAD~1")
assert len(list(git.pretty_log())) == 1
assert "Untracked files:\n\tfile_2" in git.status()
git.clean("-fdx")
git.branch("newbranch")
with open("/tmp/brigit_test/file_3", "w") as f:
f.write('3')
git.add("/tmp/brigit_test/file_3")
git.commit(message="Adding file_3")
assert "Adding file_3" in git.log()
git.checkout("newbranch")
git.cherryPick("master")
shutil.rmtree("/tmp/brigit_test")
class GitCommandList:
_repo = None
_local_git_path = None
_repo_url = None
# to initialize the repository variable
def init_git(self, local_git_path, repo_url):
try:
self._local_git_path = local_git_path
self._repo_url = repo_url
self._repo = Git(self._local_git_path, self._repo_url)
self._repo.pull()
return True
except:
return False
# to make a commit, without doing a push on the remote repository on git
def commit(self, message='Auto-commit: ' + str(datetime.now())):
try:
for file in local_tree(self._local_git_path):
self._repo.add(file)
self._repo.commit(message)
return True
except:
return False
# To push che files in the local repository.
# A commit is generated automatically
def push(self):
try:
self.commit()
self._repo.pretty_log()
r = self._repo.push()
return '[*] Pushing at: ' + str(datetime.now()) + '\n' + r
except:
return False
# to pull in the local from the remote repository
def pull(self):
try:
self._repo.pretty_log()
r = self._repo.pull()
return '[*] Pulling at: ' + str(datetime.now()) + '\n' + r
except Exception as e:
print(e)
return False
def test_basic():
shutil.rmtree("/tmp/brigit_test", ignore_errors=True)
git = Git("/tmp/brigit_test")
with open("/tmp/brigit_test/file_1", "w") as f:
f.write('1')
git.add("/tmp/brigit_test/file_1")
git.commit(message="Adding file_1")
assert "Adding file_1" in git.log()
assert len(list(git.pretty_log())) == 1
with open("/tmp/brigit_test/file_2", "w") as f:
f.write('2')
git.add("/tmp/brigit_test/file_2")
git.commit(message="Adding file_2")
assert "Adding file_2" in git.log()
assert len(list(git.pretty_log())) == 2
git.reset("HEAD~1")
assert len(list(git.pretty_log())) == 1
assert "Untracked files:\n\tfile_2" in git.status()
shutil.rmtree("/tmp/brigit_test")
def __init__(self, path):
self.__path = path
self.__new = True
self.__git = Git(dirname(path))
class GitCheckpoints(CustomCheckpoints):
"""
A Checkpoints that commits checkpoints for files to a git repo.
"""
root_dir = Unicode(config=True)
env = os.environ[
'DEPLOY_ENV'] #need to pass this through to notebooks in jupyterhub config
name = os.environ['USER'].upper()
if 'DEBUG_HOME' in os.environ.keys():
home = os.environ['DEBUG_HOME']
else:
home = os.environ['HOME']
print('Init Git wrapper in {}'.format(home))
git = Git(home)
try:
user, pw, email, = os.environ['GIT_USER'], os.environ[
'GIT_PASS'], os.environ['GIT_EMAIL']
repo_url = os.environ['GIT_URL']
except KeyError:
print('GIT Env Variables not set properly. Assuming local Git only')
traceback.print_exc()
user, pw, email, repo_url = '', '', '', ''
N_CHECKPOINTS = 10
try:
git.status()
init = True
branch = git("rev-parse", '--abbrev-ref', 'HEAD').replace('\n', '')
except GitException:
init = False
#default to USER-ENV for branch name.
#when used with JupyterHub, assume users are only using branch-names they are meant to
#TODO: restrict branch naming scheme
branch = '{}-{}'.format(name, env)
if not init:
repo = 'http://{}:{}@{}'.format(user, pw, repo_url)
git.init()
if user and pw and repo_url:
git.remote('add', 'origin', repo)
print("Git checkpoints connecting to remote repo ...")
try:
print(
"Assume branch already exists, fetch, link branches and checkout"
)
git.fetch('origin', branch)
git.branch(branch, 'origin/{}'.format(branch))
git.checkout(branch)
except GitException:
print("Create branch if not existing in remote")
git.checkout('-b', branch)
#Create a .gitignore to ignore hidden folders
with open(home + '/.gitignore', 'w') as f:
f.write('.*\n!/.gitignore')
try:
git.add('.')
git.commit('-m', 'Init checkpoints for existing untracked files')
try:
git.push('--set-upstream', 'origin', branch)
except: #this might fail with no remote
pass
git.push()
except GitException: #might only have gitignore in the repo
pass
#Set git config
git.config('user.name', name)
git.config('user.email', email)
git.config('push.default', 'matching')
print('GitCheckpoints initialised')
def _root_dir_default(self):
try:
return self.parent.root_dir
except AttributeError:
return getcwd()
# ContentsManager-dependent checkpoint API
def create_checkpoint(self, contents_mgr, path):
"""Create a checkpoint."""
path = self.checkpoint_path('', path)
self.log.debug(
"Creating checkpoint %s",
path,
)
self.git.add(path)
try:
self.git.commit('-m', 'Checkpoint {}'.format(path), path)
except GitException: #no changes, perhaps?
pass
stats = [i for i in self.git.pretty_log('-1')][0]
checkpoint_id, datetime = stats['hash'], stats['datetime']
self.git.push('--set-upstream', 'origin', self.branch)
self.git.push()
return self.checkpoint_model(checkpoint_id, datetime)
def restore_checkpoint(self, contents_mgr, checkpoint_id, path):
"""Restore a checkpoint."""
path = self.checkpoint_path('', path)
self.log.debug(
"Restoring checkpoint %s -> %s",
checkpoint_id,
path,
)
#Commit any current changes locally
try:
self.git.add(path)
self.git.commit(
'-m',
'Committing changes and restoring to {}'.format(checkpoint_id),
path)
except GitException:
pass
#Restore file to commit hash
self.git.checkout(checkpoint_id, path)
# ContentsManager-independent checkpoint API
def rename_checkpoint(self, checkpoint_id, old_path, new_path):
"""Rename a checkpoint from old_path to new_path."""
old_cp_path = self.checkpoint_path(checkpoint_id, old_path)
new_cp_path = self.checkpoint_path(checkpoint_id, new_path)
self.log.debug(
"Renaming checkpoint %s -> %s",
old_cp_path,
new_cp_path,
)
try:
self.git.add(old_cp_path, new_cp_path)
self.git.commit(
'-m', 'Renaming {} -> {}'.format(old_cp_path, new_cp_path),
old_cp_path, new_cp_path)
except GitException: #Fresh notebooks with no save
self.git.add(new_cp_path)
self.git.commit(
'-m', 'Renaming unsaved notebook -> {}'.format(new_cp_path),
new_cp_path)
self.git.push()
def delete_checkpoint(self, checkpoint_id, path):
"""delete a file's checkpoint"""
#in our Git context, this will just be committing the deleted file
cp_path = self.checkpoint_path(checkpoint_id, path)
self.log.debug(
"Deleting checkpoint %s @ %s",
cp_path,
checkpoint_id,
)
self.git.add(cp_path)
self.git.commit('-m', 'Deleting {}'.format(cp_path), cp_path)
self.git.push()
def list_checkpoints(self, path):
"""
list the latest checkpoints for a given file.
N_CHECKPOINTS is set globally above
"""
path = self.checkpoint_path('', path)
self.log.debug('List checkpoints for {}'.format(path))
try:
commit_log = self.git.pretty_log('-{}'.format(self.N_CHECKPOINTS),
'--', path)
stats = [(i['hash'], i['datetime']) for i in commit_log]
except:
traceback.print_exc()
stats = []
return [
self.checkpoint_model(checkpoint_id, datetime)
for (checkpoint_id, datetime) in stats
]
# Checkpoint-related utilities
def checkpoint_path(self, checkpoint_id, path):
"""find the path to a checkpoint"""
return os.path.join(self.home, path.strip('/'))
def checkpoint_model(self, checkpoint_id, datetime):
"""construct the info dict for a given checkpoint"""
local = pytz.timezone(PYTZ_TIMEZONE)
local_dt = local.localize(datetime, is_dst=True)
last_modified = local_dt.astimezone(pytz.utc)
info = dict(
id=checkpoint_id,
last_modified=last_modified,
)
return info
# Error Handling
def no_such_checkpoint(self, path, checkpoint_id):
raise HTTPError(
404, u'Checkpoint does not exist: %s@%s' % (path, checkpoint_id))
