def annex_status(collection):
"""Command-line function for running git annex status on collection repository.
@param collection: Collection
@return: message ('ok' if successful)
"""
return dvcs.annex_status(dvcs.repository(collection.path))
def annex_status(collection_path):
"""Command-line function for running git annex status on collection repository.
@param collection_path: Absolute path to collection repo.
@return: message ('ok' if successful)
"""
return dvcs.annex_status(collection_path)
def annex_status(collection):
"""Command-line function for running git annex status on collection repository.
@param collection: Collection
@return: message ('ok' if successful)
"""
return dvcs.annex_status(dvcs.repository(collection.path))
def repo_annex_status(self):
"""Get annex status of collection repo.
"""
if not self._astatus and (os.path.exists(self.git_path)):
astatus = dvcs.annex_status(dvcs.repository(self.path))
if astatus:
self._astatus = astatus
return self._astatus
def repo_annex_status( self ):
"""Get annex status of collection repo.
"""
if not self._astatus and (os.path.exists(os.path.join(self.path, '.git'))):
astatus = dvcs.annex_status(self.path)
if astatus:
self._astatus = astatus
return self._astatus
def annex_info(repo):
"""Info from git-annex info command
@param repo: GitPython.Repository object
@returns: dict
"""
WHITELIST = [f for f in AINFO_FIELDS.keys()]
ainfo = {
AINFO_FIELDS[key]: val
for key, val in dvcs.annex_status(repo).items() if key in WHITELIST
}
return ainfo
def annex_info(repo):
"""Info from git-annex info command
@param repo: GitPython.Repository object
@returns: dict
"""
WHITELIST = [f for f in AINFO_FIELDS.iterkeys()]
ainfo = {
AINFO_FIELDS[key]: val
for key,val in dvcs.annex_status(repo).iteritems()
if key in WHITELIST
}
return ainfo
def test_annex_status():
path = os.path.join(TESTING_BASE_DIR, 'test-repo')
repo = make_repo(path, ['testing'])
annex_init(repo)
status = dvcs.annex_status(repo)
cleanup_repo(path)
found = False
for key in status.iterkeys():
if 'repositories' in key:
for r in status[key]:
if r['here']:
found = True
assert found
def test_annex_status(tmpdir):
path = str(tmpdir / 'test-repo')
repo = make_repo(path, ['testing'])
annex_init(repo)
status = dvcs.annex_status(repo)
cleanup_repo(path)
found = False
for key in status.keys():
if 'repositories' in key:
for r in status[key]:
if r['here']:
found = True
assert found
def update( base_dir, collection_path ):
"""Gets a bunch of status info for the collection; refreshes if forced
timestamp, elapsed, status, annex_status, syncstatus
@param force: Boolean Forces refresh of status
@returns: dict
"""
start = datetime.now()
repo = dvcs.repository(collection_path)
status = dvcs.repo_status(repo, short=True)
annex_status = dvcs.annex_status(repo)
timestamp = datetime.now()
syncstatus = sync_status(collection_path, git_status=status, timestamp=timestamp, force=True)
elapsed = timestamp - start
text = write(base_dir, collection_path, timestamp, elapsed, status, annex_status, syncstatus)
return loads(text)
def sync(user_name, user_mail, collection):
"""Sync repo with bare clone on hub server; replaces git-annex-sync.
Git-annex has a "sync" command for communicating annex changes between
repositories, but it is designed to be used between non-bare repositories.
Normally Git does not support pushing to non-bare repositories, and
git-annex does some trickery involving "synced/BRANCH" branches to make
this work.
Reference: http://git-annex.branchable.com/sync/
When git-annex-sync is used between a non-bare repo and a bare repo
(e.g. between a local repo and our hub server running Gitolite),
the "synced/master" branches do not get merged in to master and syncing
no longer works. Therefore it is necessary to sync manually.
If you think you want to use git-annex-sync, remember that we tried this
in commit 1857a7aa3f and it did not work and we reverted to manual syncing.
@param user_name: Username for use in changelog, git log
@param user_mail: User email address for use in changelog, git log
@param collection: Collection
@return: message ('ok' if successful)
"""
repo = dvcs.repository(collection.path, user_name, user_mail)
logging.debug('repo: %s' % repo)
drive_label = storage.drive_label(repo.working_dir)
dvcs.annex_set_description(repo,
dvcs.annex_status(repo),
drive_label=drive_label)
dvcs.remote_add(repo, collection.git_url, config.GIT_REMOTE_NAME)
# list remotes
logging.debug('remotes')
for remote in dvcs.remotes(repo):
logging.debug('- %s %s' % (remote['name'], remote['target']))
# sync
logging.debug('git annex sync')
out = repo.git.annex('sync')
logging.debug(out)
return 0, 'ok'
def clone(user_name, user_mail, identifier, dest_path):
"""Command-line function for cloning an existing collection.
Clones existing collection object from workbench server.
@param user_name: Username for use in changelog, git log
@param user_mail: User email address for use in changelog, git log
@param identifier: Identifier
@param dest_path: str
@return: message ('ok' if successful)
"""
git_url = '{}:{}.git'.format(config.GITOLITE, identifier.id)
repo = git.Repo.clone_from(git_url, dest_path)
logging.debug(' git clone {}'.format(git_url))
if repo:
logging.debug(' OK')
else:
logging.error(' COULD NOT CLONE!')
return 1, 'could not clone'
if os.path.exists(identifier.path_abs('git')):
logging.debug(' .git/ is present')
else:
logging.error(' .git/ IS MISSING!')
return 1, '.git/ is missing'
# git annex init if not already existing
if not os.path.exists(identifier.path_abs('annex')):
logging.debug(' git annex init')
repo.git.annex('init')
#
repo.git.checkout('master')
dvcs.git_set_configs(repo, user_name, user_mail)
dvcs.annex_set_configs(repo, user_name, user_mail)
drive_label = storage.drive_label(repo.working_dir)
dvcs.annex_set_description(repo,
dvcs.annex_status(repo),
drive_label=drive_label)
dvcs.remote_add(repo, git_url, config.GIT_REMOTE_NAME)
return 0, 'ok'
def sync(user_name, user_mail, collection):
"""Sync repo with bare clone on hub server; replaces git-annex-sync.
Git-annex has a "sync" command for communicating annex changes between
repositories, but it is designed to be used between non-bare repositories.
Normally Git does not support pushing to non-bare repositories, and
git-annex does some trickery involving "synced/BRANCH" branches to make
this work.
Reference: http://git-annex.branchable.com/sync/
When git-annex-sync is used between a non-bare repo and a bare repo
(e.g. between a local repo and our hub server running Gitolite),
the "synced/master" branches do not get merged in to master and syncing
no longer works. Therefore it is necessary to sync manually.
If you think you want to use git-annex-sync, remember that we tried this
in commit 1857a7aa3f and it did not work and we reverted to manual syncing.
@param user_name: Username for use in changelog, git log
@param user_mail: User email address for use in changelog, git log
@param collection: Collection
@return: message ('ok' if successful)
"""
repo = dvcs.repository(collection.path, user_name, user_mail)
logging.debug('repo: %s' % repo)
drive_label = storage.drive_label(repo.working_dir)
dvcs.annex_set_description(repo, dvcs.annex_status(repo), drive_label=drive_label)
dvcs.remote_add(repo, collection.git_url, config.GIT_REMOTE_NAME)
# list remotes
logging.debug('remotes')
for remote in dvcs.remotes(repo):
logging.debug('- %s %s' % (remote['name'], remote['target']))
# sync
logging.debug('git annex sync')
out = repo.git.annex('sync')
logging.debug(out)
return 0,'ok'
def clone(user_name, user_mail, identifier, dest_path):
"""Command-line function for cloning an existing collection.
Clones existing collection object from workbench server.
@param user_name: Username for use in changelog, git log
@param user_mail: User email address for use in changelog, git log
@param identifier: Identifier
@param dest_path: str
@return: message ('ok' if successful)
"""
git_url = '{}:{}.git'.format(config.GITOLITE, identifier.id)
repo = git.Repo.clone_from(git_url, dest_path)
logging.debug(' git clone {}'.format(git_url))
if repo:
logging.debug(' OK')
else:
logging.error(' COULD NOT CLONE!')
return 1,'could not clone'
if os.path.exists(identifier.path_abs('git')):
logging.debug(' .git/ is present')
else:
logging.error(' .git/ IS MISSING!')
return 1,'.git/ is missing'
# git annex init if not already existing
if not os.path.exists(identifier.path_abs('annex')):
logging.debug(' git annex init')
repo.git.annex('init')
#
repo.git.checkout('master')
dvcs.git_set_configs(repo, user_name, user_mail)
dvcs.annex_set_configs(repo, user_name, user_mail)
drive_label = storage.drive_label(repo.working_dir)
dvcs.annex_set_description(repo, dvcs.annex_status(repo), drive_label=drive_label)
dvcs.remote_add(repo, git_url, config.GIT_REMOTE_NAME)
return 0,'ok'
def create(user_name, user_mail, identifier, templates, agent=''):
"""Command-line function for creating a new collection.
Clones a blank collection object from workbench server, adds files, commits.
- clones new repo from gitolite server
# Easier to have Gitolite create repo then clone (http://sitaramc.github.com/gitolite/repos.html)
# than to add existing to Gitolite (http://sitaramc.github.com/gitolite/rare.html#existing).
local requests CID from workbench API
background:collection init: $ collection -cCID -oinit]
background:collection init: $ git clone git@mits:ddr-ORG-C
$ git clone git@mits:ddr-densho-1
Cloning into 'ddr-densho-1'...
Initialized empty Git repository in /home/git/repositories/ddr-densho-1.git/
warning: You appear to have cloned an empty repository.
background:entity init: $ git annex init
background:entity init: $ git add changelog control ead.xml .gitignore
background:entity init: $ git commit
@param user_name: Username for use in changelog, git log
@param user_mail: User email address for use in changelog, git log
@param identifier: Identifier
@param templates: List of metadata templates (absolute paths).
@param agent: (optional) Name of software making the change.
@return: message ('ok' if successful)
"""
gitolite = dvcs.Gitolite(config.GITOLITE)
gitolite.initialize()
if identifier.id in gitolite.collections():
raise Exception("'%s' already exists -- clone instead." % identifier.id)
git_url = '{}:{}.git'.format(config.GITOLITE, identifier.id)
repo = git.Repo.clone_from(git_url, identifier.path_abs())
logging.debug(' git clone {}'.format(git_url))
if repo:
logging.debug(' OK')
else:
logging.error(' COULD NOT CLONE!')
if os.path.exists(identifier.path_abs('git')):
logging.debug(' .git/ is present')
else:
logging.error(' .git/ IS MISSING!')
# there is no master branch at this point
dvcs.remote_add(repo, git_url, config.GIT_REMOTE_NAME)
dvcs.git_set_configs(repo, user_name, user_mail)
dvcs.annex_set_configs(repo, user_name, user_mail)
git_files = []
# copy template files to collection
for src in templates:
if os.path.exists(src):
dst = os.path.join(identifier.path_abs(), os.path.basename(src))
logging.debug('cp %s, %s' % (src, dst))
shutil.copy(src, dst)
if os.path.exists(dst):
git_files.append(dst)
else:
logging.error('COULD NOT COPY %s' % src)
# instantiate now that we have collection dir and some templates
object_class = identifier.object_class()
collection = object_class(identifier.path_abs())
# add control, .gitignore, changelog
control = collection.control()
gitignore = collection.gitignore()
# prep log entries
changelog_messages = ['Initialized collection {}'.format(collection.id)]
if agent:
changelog_messages.append('@agent: %s' % agent)
commit_message = dvcs.compose_commit_message(changelog_messages[0], agent=agent)
write_changelog_entry(collection.changelog_path,
changelog_messages,
user_name, user_mail)
if os.path.exists(control.path):
git_files.append(control.path_rel)
else:
logging.error(' COULD NOT CREATE control')
if os.path.exists(collection.gitignore_path):
git_files.append(collection.gitignore_path_rel)
else:
logging.error(' COULD NOT CREATE .gitignore')
if os.path.exists(collection.changelog_path):
git_files.append(collection.changelog_path_rel)
else:
logging.error(' COULD NOT CREATE changelog')
# add files and commit
repo = commit_files(repo, commit_message, git_files, [])
# master branch should be created by this point
# git annex init
logging.debug(' git annex init')
repo.git.annex('init')
if os.path.exists(os.path.join(collection.path, '.git', 'annex')):
logging.debug(' .git/annex/ OK')
else:
logging.error(' .git/annex/ IS MISSING!')
# manual version of git-annex-sync - see notes for DDR.commands.sync.
logging.debug('git push %s git-annex' % config.GIT_REMOTE_NAME)
repo.git.checkout('git-annex')
repo.git.push(config.GIT_REMOTE_NAME, 'git-annex')
logging.debug('git push %s master' % config.GIT_REMOTE_NAME)
repo.git.checkout('master')
repo.git.push(config.GIT_REMOTE_NAME, 'master')
logging.debug('OK')
drive_label = storage.drive_label(repo.working_dir)
dvcs.annex_set_description(repo, dvcs.annex_status(repo), drive_label=drive_label)
return 0,'ok'
def create(user_name, user_mail, identifier, templates, agent=''):
"""Command-line function for creating a new collection.
Clones a blank collection object from workbench server, adds files, commits.
- clones new repo from gitolite server
# Easier to have Gitolite create repo then clone (http://sitaramc.github.com/gitolite/repos.html)
# than to add existing to Gitolite (http://sitaramc.github.com/gitolite/rare.html#existing).
local requests CID from workbench API
background:collection init: $ collection -cCID -oinit]
background:collection init: $ git clone git@mits:ddr-ORG-C
$ git clone git@mits:ddr-densho-1
Cloning into 'ddr-densho-1'...
Initialized empty Git repository in /home/git/repositories/ddr-densho-1.git/
warning: You appear to have cloned an empty repository.
background:entity init: $ git annex init
background:entity init: $ git add changelog control ead.xml .gitignore
background:entity init: $ git commit
@param user_name: Username for use in changelog, git log
@param user_mail: User email address for use in changelog, git log
@param identifier: Identifier
@param templates: List of metadata templates (absolute paths).
@param agent: (optional) Name of software making the change.
@return: message ('ok' if successful)
"""
gitolite = dvcs.Gitolite(config.GITOLITE)
gitolite.initialize()
if identifier.id in gitolite.collections():
raise Exception("'%s' already exists -- clone instead." % identifier.id)
git_url = '{}:{}.git'.format(config.GITOLITE, identifier.id)
repo = git.Repo.clone_from(git_url, identifier.path_abs())
logging.debug(' git clone {}'.format(git_url))
if repo:
logging.debug(' OK')
else:
logging.error(' COULD NOT CLONE!')
if os.path.exists(identifier.path_abs('git')):
logging.debug(' .git/ is present')
else:
logging.error(' .git/ IS MISSING!')
# there is no master branch at this point
dvcs.remote_add(repo, git_url, config.GIT_REMOTE_NAME)
dvcs.git_set_configs(repo, user_name, user_mail)
dvcs.annex_set_configs(repo, user_name, user_mail)
git_files = []
# copy template files to collection
for src in templates:
if os.path.exists(src):
dst = os.path.join(identifier.path_abs(), os.path.basename(src))
logging.debug('cp %s, %s' % (src, dst))
shutil.copy(src, dst)
if os.path.exists(dst):
git_files.append(dst)
else:
logging.error('COULD NOT COPY %s' % src)
# instantiate now that we have collection dir and some templates
object_class = identifier.object_class()
collection = object_class(identifier.path_abs())
# add control, .gitignore, changelog
control = collection.control()
gitignore = collection.gitignore()
# prep log entries
changelog_messages = ['Initialized collection {}'.format(collection.id)]
if agent:
changelog_messages.append('@agent: %s' % agent)
commit_message = dvcs.compose_commit_message(changelog_messages[0], agent=agent)
write_changelog_entry(collection.changelog_path,
changelog_messages,
user_name, user_mail)
if os.path.exists(control.path):
git_files.append(control.path_rel)
else:
logging.error(' COULD NOT CREATE control')
if os.path.exists(collection.gitignore_path):
git_files.append(collection.gitignore_path_rel)
else:
logging.error(' COULD NOT CREATE .gitignore')
if os.path.exists(collection.changelog_path):
git_files.append(collection.changelog_path_rel)
else:
logging.error(' COULD NOT CREATE changelog')
# add files and commit
repo = commit_files(repo, commit_message, git_files, [])
# master branch should be created by this point
# git annex init
logging.debug(' git annex init')
repo.git.annex('init')
if os.path.exists(os.path.join(collection.path, '.git', 'annex')):
logging.debug(' .git/annex/ OK')
else:
logging.error(' .git/annex/ IS MISSING!')
# manual version of git-annex-sync - see notes for DDR.commands.sync.
logging.debug('git push %s git-annex' % config.GIT_REMOTE_NAME)
repo.git.checkout('git-annex')
repo.git.push(config.GIT_REMOTE_NAME, 'git-annex')
logging.debug('git push %s master' % config.GIT_REMOTE_NAME)
repo.git.checkout('master')
repo.git.push(config.GIT_REMOTE_NAME, 'master')
logging.debug('OK')
drive_label = storage.drive_label(repo.working_dir)
dvcs.annex_set_description(repo, dvcs.annex_status(repo), drive_label=drive_label)
return 0,'ok'
