def _get_repo(self, name):
root = repository_path(name)
if os.path.isdir(root):
repo = GitRepoManager()
repo.open_repo(root)
return repo
def _add_repository(self, dest, repository_identifier, creator, create_repo):
repository_identifier = format_repository_identifier(repository_identifier)
# sys.exit()
proot = os.path.join(paths.repository_dataset_dir, repository_identifier)
if not os.path.isdir(proot):
# create new local repo
os.mkdir(proot)
repo = GitRepoManager()
repo.open_repo(proot)
repo.add_ignore('.DS_Store')
self.repo_man = repo
if create_repo:
# add repo to central location
create_github_repo(repository_identifier)
url = 'https://github.com/{}/{}.git'.format(ORG, repository_identifier)
self.debug('Create repo at github. url={}'.format(url))
repo.create_remote(url)
else:
repo = GitRepoManager()
repo.open_repo(proot)
dbexp = dest.get_repository(repository_identifier)
if not dbexp:
dest.add_repository(repository_identifier, creator)
return repo
def _get_repo(self, name):
root = repository_path(name)
if os.path.isdir(root):
repo = GitRepoManager()
repo.open_repo(root)
return repo
def _selected_local_repository_name_changed(self, new):
if new:
root = os.path.join(paths.experiment_dataset_dir, new)
# print root, new, os.path.isdir(root)
if os.path.isdir(root):
repo = GitRepoManager()
repo.open_repo(root)
self._repo = repo
self._refresh_branches()
def _commit_freeze(self, added, msg):
key = lambda x: x[0]
rr = sorted(added, key=key)
for repo, ps in groupby(rr, key=key):
rm = GitRepoManager()
rm.open_repo(repo, paths.repository_dataset_dir)
rm.add_paths(ps)
rm.smart_pull()
rm.commit(msg)
def _selected_local_repository_name_changed(self, new):
if new:
root = os.path.join(paths.experiment_dataset_dir, new)
# print root, new, os.path.isdir(root)
if os.path.isdir(root):
repo = GitRepoManager()
repo.open_repo(root)
self._repo = repo
self._refresh_branches()
def commit_initial_import(repo_identifier, root):
from pychron.git_archive.repo_manager import GitRepoManager
rm = GitRepoManager()
proot = os.path.join(root, repo_identifier)
rm.open_repo(proot)
repo = rm._repo
repo.git.add('.')
repo.git.commit('-m', '<IMPORT> initial')
repo.git.push('--set-upstream', 'origin', 'master')
def fix_import_commit(repo_identifier, root):
from pychron.git_archive.repo_manager import GitRepoManager
rm = GitRepoManager()
proot = os.path.join(root, repo_identifier)
rm.open_repo(proot)
repo = rm._repo
print('========= {} ======'.format(repo_identifier))
txt = repo.git.log('--pretty=oneline')
print(txt)
def __init__(self, path=None, root=None, *args, **kw):
super(BaseGitHistory, self).__init__(*args, **kw)
if root:
from pychron.git_archive.repo_manager import GitRepoManager
self.repo_man = GitRepoManager()
self.repo_man.open_repo(root)
if path:
self._path = path
def fix_import_commit(repo_identifier, root):
from pychron.git_archive.repo_manager import GitRepoManager
rm = GitRepoManager()
proot = os.path.join(root, repo_identifier)
rm.open_repo(proot)
repo = rm._repo
print '========= {} ======'.format(repo_identifier)
txt = repo.git.log('--pretty=oneline')
print txt
def commit_initial_import(repo_identifier, root):
from pychron.git_archive.repo_manager import GitRepoManager
rm = GitRepoManager()
proot = os.path.join(root, repo_identifier)
rm.open_repo(proot)
repo = rm._repo
repo.git.add('.')
repo.git.commit('-m', '<IMPORT> initial')
repo.git.push('--set-upstream', 'origin', 'master')
def fix_meta(dest, repo_identifier, root):
d = os.path.join(root, repo_identifier)
changed = False
with dest.session_ctx():
repo = dest.get_repository(repo_identifier)
for ra in repo.repository_associations:
an = ra.analysis
p = analysis_path(an.record_id, repo_identifier)
obj = dvc_load(p)
if not obj:
print '********************** {} not found in repo'.format(
an.record_id)
continue
print an.record_id, p
if not obj['irradiation']:
obj['irradiation'] = an.irradiation
lchanged = True
changed = True
if not obj['irradiation_position']:
obj['irradiation_position'] = an.irradiation_position_position
lchanged = True
changed = True
if not obj['irradiation_level']:
obj['irradiation_level'] = an.irradiation_level
lchanged = True
changed = True
if not obj['material']:
obj['material'] = an.irradiation_position.sample.material.name
lchanged = True
changed = True
if not obj['project']:
obj['project'] = an.irradiation_position.sample.project.name
lchanged = True
changed = True
if obj['repository_identifier'] != an.repository_identifier:
obj['repository_identifier'] = an.repository_identifier
lchanged = True
changed = True
if lchanged:
print '{} changed'.format(an.record_id)
dvc_dump(obj, p)
if changed:
from pychron.git_archive.repo_manager import GitRepoManager
rm = GitRepoManager()
rm.open_repo(d)
repo = rm._repo
repo.git.add('.')
repo.git.commit('-m', '<MANUAL> fixed metadata')
repo.git.push()
def fix_meta(dest, repo_identifier, root):
d = os.path.join(root, repo_identifier)
changed = False
with dest.session_ctx():
repo = dest.get_repository(repo_identifier)
for ra in repo.repository_associations:
an = ra.analysis
p = analysis_path(an.record_id, repo_identifier)
obj = dvc_load(p)
if not obj:
print('********************** {} not found in repo'.format(an.record_id))
continue
print(an.record_id, p)
if not obj['irradiation']:
obj['irradiation'] = an.irradiation
lchanged = True
changed = True
if not obj['irradiation_position']:
obj['irradiation_position'] = an.irradiation_position_position
lchanged = True
changed = True
if not obj['irradiation_level']:
obj['irradiation_level'] = an.irradiation_level
lchanged = True
changed = True
if not obj['material']:
obj['material'] = an.irradiation_position.sample.material.name
lchanged = True
changed = True
if not obj['project']:
obj['project'] = an.irradiation_position.sample.project.name
lchanged = True
changed = True
if obj['repository_identifier'] != an.repository_identifier:
obj['repository_identifier'] = an.repository_identifier
lchanged = True
changed = True
if lchanged:
print('{} changed'.format(an.record_id))
dvc_dump(obj, p)
if changed:
from pychron.git_archive.repo_manager import GitRepoManager
rm = GitRepoManager()
rm.open_repo(d)
repo = rm._repo
repo.git.add('.')
repo.git.commit('-m', '<MANUAL> fixed metadata')
repo.git.push()
def _get_experiment_repo(self, experiment_id):
repo = self.experiment_repo
path = experiment_path(experiment_id)
if repo is None or repo.path != path:
self.debug('make new repomanager for {}'.format(path))
repo = GitRepoManager()
repo.path = path
repo.open_repo(path)
self.experiment_repo = repo
return repo
def create_repo_for_existing_local(repo_identifier, root, organization='NMGRLData'):
from pychron.git_archive.repo_manager import GitRepoManager
repo = GitRepoManager()
proot = os.path.join(root, repo_identifier)
repo.open_repo(proot)
org = Organization(organization)
if not org.has_repo(repo_identifier):
usr = os.environ.get('GITHUB_USER')
pwd = os.environ.get('GITHUB_PASSWORD')
org.create_repo(repo_identifier, usr, pwd)
url = 'https://github.com/{}/{}.git'.format(organization, repo_identifier)
repo.create_remote(url)
def experiment_has_staged(ps):
if not hasattr(ps, '__iter__'):
ps = (ps,)
changed = []
repo = GitRepoManager()
for p in ps:
pp = os.path.join(paths.experiment_dataset_dir, p)
repo.open_repo(pp)
if repo.has_unpushed_commits():
changed.append(p)
return changed
def _get_repository(self, repository_identifier, as_current=True):
repo = None
if as_current:
repo = self.current_repository
path = repository_path(repository_identifier)
if repo is None or repo.path != path:
self.debug('make new repomanager for {}'.format(path))
repo = GitRepoManager()
repo.path = path
repo.open_repo(path)
if as_current:
self.current_repository = repo
return repo
def activated(self):
repo = GitRepoManager()
repo.open_repo(paths.labbook_dir)
self._repo = repo
self.history_model.repo_man = repo
self.history_model.auto_commit_checkouts = False
# self._preference_binder('pychron.labbook', ('remote',))
if self.remote:
remote = 'https://github.com/{}'.format(self.remote)
self._repo.create_remote(remote, force=True)
self.pull(make=False)
self.make_hierarchy()
self._repo.add(os.path.join(paths.labbook_dir, 'labels.db'))
def activated(self):
repo = GitRepoManager()
repo.open_repo(paths.labbook_dir)
self._repo = repo
self.history_model.repo_man = repo
self.history_model.auto_commit_checkouts = False
# self._preference_binder('pychron.labbook', ('remote',))
if self.remote:
remote = 'https://github.com/{}'.format(self.remote)
self._repo.create_remote(remote, force=True)
self.pull(make=False)
self.make_hierarchy()
self._repo.add(os.path.join(paths.labbook_dir, 'labels.db'))
def __init__(self, path=None, root=None, *args, **kw):
super(GitArchiveHistory, self).__init__(*args, **kw)
if root:
from pychron.git_archive.repo_manager import GitRepoManager
self.repo_man = GitRepoManager()
self.repo_man.open_repo(root)
if path:
self._path = path
def push_repositories(ps, remote='origin', branch='master', quiet=True):
for p in ps:
pp = os.path.join(paths.repository_dataset_dir, p)
# repo = Repo(pp)
repo = GitRepoManager()
repo.open_repo(pp)
if repo.smart_pull(remote=remote, branch=branch, quiet=quiet):
repo.push(remote, branch)
def push_repositories(ps, remote='origin', branch='master', quiet=True):
for p in ps:
pp = repository_path(p)
# repo = Repo(pp)
repo = GitRepoManager()
repo.open_repo(pp)
if repo.smart_pull(remote=remote, branch=branch, quiet=quiet):
repo.push(remote=remote, branch=branch)
def create_repo_for_existing_local(repo_identifier,
root,
organization='NMGRLData'):
from pychron.git_archive.repo_manager import GitRepoManager
repo = GitRepoManager()
proot = os.path.join(root, repo_identifier)
repo.open_repo(proot)
org = Organization(organization)
if not org.has_repo(repo_identifier):
usr = os.environ.get('GITHUB_USER')
pwd = os.environ.get('GITHUB_PASSWORD')
org.create_repo(repo_identifier, usr, pwd)
url = 'https://github.com/{}/{}.git'.format(organization,
repo_identifier)
repo.create_remote(url)
def initialize(self, repository, pull=True):
"""
setup git repos.
repositories are guaranteed to exist. The automated run factory clones the required projects
on demand.
:return:
"""
self.debug('^^^^^^^^^^^^^ Initialize DVCPersister {} pull={}'.format(repository, pull))
self.dvc.initialize()
repository = format_repository_identifier(repository)
self.active_repository = repo = GitRepoManager()
root = repository_path(repository)
repo.open_repo(root)
remote = 'origin'
if repo.has_remote(remote) and pull:
self.info('pulling changes from repo: {}'.format(repository))
self.active_repository.pull(remote=remote, use_progress=False)
def initialize(self, experiment, pull=True):
"""
setup git repos.
repositories are guaranteed to exist. The automated run factory clones the required projects
on demand.
:return:
"""
self.debug('^^^^^^^^^^^^^ Initialize DVCPersister {} pull={}'.format(experiment, pull))
self.dvc.initialize()
experiment = format_experiment_identifier(experiment)
self.experiment_repo = repo = GitRepoManager()
root = os.path.join(paths.experiment_dataset_dir, experiment)
repo.open_repo(root)
remote = 'origin'
if repo.has_remote(remote) and pull:
self.info('pulling changes from experiment repo: {}'.format(experiment))
self.experiment_repo.pull(remote=remote, use_progress=False)
def push_repositories(ps, remote=None):
repo = GitRepoManager()
for p in ps:
pp = os.path.join(paths.repository_dataset_dir, p)
repo.open_repo(pp)
repo.push(remote=remote)
def push_experiments(ps):
repo = GitRepoManager()
for p in ps:
pp = os.path.join(paths.experiment_dataset_dir, p)
repo.open_repo(pp)
repo.push()
def push_repositories(ps, remote=None):
repo = GitRepoManager()
for p in ps:
pp = os.path.join(paths.repository_dataset_dir, p)
repo.open_repo(pp)
repo.push(remote=remote)
def _add_repository(self, dest, repository_identifier, creator,
create_repo):
repository_identifier = format_repository_identifier(
repository_identifier)
# sys.exit()
proot = os.path.join(paths.repository_dataset_dir,
repository_identifier)
if not os.path.isdir(proot):
# create new local repo
os.mkdir(proot)
repo = GitRepoManager()
repo.open_repo(proot)
repo.add_ignore('.DS_Store')
self.repo_man = repo
if create_repo:
# add repo to central location
create_github_repo(repository_identifier)
url = 'https://github.com/{}/{}.git'.format(
ORG, repository_identifier)
self.debug('Create repo at github. url={}'.format(url))
repo.create_remote(url)
else:
repo = GitRepoManager()
repo.open_repo(proot)
dbexp = dest.get_repository(repository_identifier)
if not dbexp:
dest.add_repository(repository_identifier, creator)
return repo
def _repo_manager_default(self):
from pychron.git_archive.repo_manager import GitRepoManager
return GitRepoManager(application=self.application)
class GitArchiveHistory(BaseGitHistory):
checkout_button = Button('Checkout')
diff_button = Button
limit = Int(100, enter_set=True, auto_set=False)
diffable = Property(depends_on='selected')
checkoutable = Property(depends_on='selected')
checkout_event = Event
diff_klass = DiffView
auto_commit_checkouts = True
selected = List
selected_commit = Property(depends_on='selected')
repo_man = Instance('pychron.git_archive.repo_manager.GitRepoManager')
_path = Str
def __init__(self, path=None, root=None, *args, **kw):
super(BaseGitHistory, self).__init__(*args, **kw)
if root:
from pychron.git_archive.repo_manager import GitRepoManager
self.repo_man = GitRepoManager()
self.repo_man.open_repo(root)
if path:
self._path = path
def close(self):
pass
# self.repo_man.close()
def load_history(self, p=None):
if p is None:
p = self._path
if p:
self._path = p
hx = self.repo_man.commits_iter(p, keys=['message', 'committed_date'],
limit=self.limit)
self.items = [GitShaObject(hexsha=a, message=b,
date=datetime.utcfromtimestamp(c),
name=p) for a, b, c in hx]
def _selected_changed(self, new):
if new:
new = new[-1]
if not new.blob:
new.blob = self.repo_man.unpack_blob(new.hexsha, new.name)
def _get_selected_commit(self):
if self.selected:
return self.selected[-1]
def _limit_changed(self):
self.load_history()
def _checkout_button_fired(self):
with open(self._path, 'w') as wfile:
wfile.write(self.selected_commit.blob)
if self.auto_commit_checkouts:
self.repo_man.add(self._path,
msg_prefix='checked out')
self.load_history()
self.selected = self.items[:1]
self.checkout_event = self._path
def _diff_button_fired(self):
a, b = self.selected
d = self.repo_man.diff(a.hexsha, b.hexsha)
if not a.blob:
a.blob = self.repo_man.unpack_blob(a.hexsha, a.name)
if not b.blob:
b.blob = self.repo_man.unpack_blob(b.hexsha, b.name)
ds = '\n'.join([li for li in d.split('\n')
if li[0] in ('-', '+')])
lm = a.message
n = 40
if len(lm) > n:
lm = '{}...'.format(lm[:n])
rm = a.message
if len(rm) > n:
rm = '{}...'.format(rm[:n])
dd = self.diff_klass(left=a.blob, left_date=a.date.strftime('%m-%d-%Y %H:%M:%S'), left_message=lm,
right=b.blob, right_date=b.date.strftime('%m-%d-%Y %H:%M:%S'), right_message=rm,
diff=ds)
dd.edit_traits()
# dd.configure_traits()
def _get_diffable(self):
if self.selected:
return len(self.selected) == 2
def _get_checkoutable(self):
if self.selected:
return len(self.selected) == 1
class GitArchiveHistory(BaseGitHistory):
checkout_button = Button('Checkout')
diff_button = Button
limit = Int(100, enter_set=True, auto_set=False)
diffable = Property(depends_on='selected')
checkoutable = Property(depends_on='selected')
checkout_event = Event
diff_klass = DiffView
auto_commit_checkouts = True
selected = List
selected_commit = Property(depends_on='selected')
repo_man = Instance('pychron.git_archive.repo_manager.GitRepoManager')
_path = Str
def __init__(self, path=None, root=None, *args, **kw):
super(BaseGitHistory, self).__init__(*args, **kw)
if root:
from pychron.git_archive.repo_manager import GitRepoManager
self.repo_man = GitRepoManager()
self.repo_man.open_repo(root)
if path:
self._path = path
def close(self):
pass
# self.repo_man.close()
def load_history(self, p=None):
if p is None:
p = self._path
if p:
self._path = p
hx = self.repo_man.commits_iter(p,
keys=['message', 'committed_date'],
limit=self.limit)
self.items = [
GitSha(hexsha=a,
message=b,
date=datetime.utcfromtimestamp(c),
name=p) for a, b, c in hx
]
def _selected_changed(self, new):
if new:
new = new[-1]
if not new.blob:
new.blob = self.repo_man.unpack_blob(new.hexsha, new.name)
def _get_selected_commit(self):
if self.selected:
return self.selected[-1]
def _limit_changed(self):
self.load_history()
def _checkout_button_fired(self):
with open(self._path, 'w') as wfile:
wfile.write(self.selected_commit.blob)
if self.auto_commit_checkouts:
self.repo_man.add(self._path, msg_prefix='checked out')
self.load_history()
self.selected = self.items[:1]
self.checkout_event = self._path
def _diff_button_fired(self):
a, b = self.selected
d = self.repo_man.diff(a.hexsha, b.hexsha)
if not a.blob:
a.blob = self.repo_man.unpack_blob(a.hexsha, a.name)
if not b.blob:
b.blob = self.repo_man.unpack_blob(b.hexsha, b.name)
ds = '\n'.join([li for li in d.split('\n') if li[0] in ('-', '+')])
lm = a.message
n = 40
if len(lm) > n:
lm = '{}...'.format(lm[:n])
rm = a.message
if len(rm) > n:
rm = '{}...'.format(rm[:n])
dd = self.diff_klass(left=a.blob,
left_date=a.date.strftime('%m-%d-%Y %H:%M:%S'),
left_message=lm,
right=b.blob,
right_date=b.date.strftime('%m-%d-%Y %H:%M:%S'),
right_message=rm,
diff=ds)
dd.edit_traits()
# dd.configure_traits()
def _get_diffable(self):
if self.selected:
return len(self.selected) == 2
def _get_checkoutable(self):
if self.selected:
return len(self.selected) == 1
def clone_from(self, name, root, organization):
url = self.make_url(name, organization)
GitRepoManager.clone_from(url, root)
